From 8d634bc5de3729b0c61bdcbb2c6a00fb0dc2346c Mon Sep 17 00:00:00 2001 From: andyhrc <12491949+andyhrc@users.noreply.github.com> Date: Thu, 15 Oct 2026 10:04:20 +0800 Subject: [PATCH 01/21] batch: STRATA_PIPELINE_ACTIVE_ROWS=1 - a pipelined group's window runs its active slots only, so an idle slot keeps its conversation (opt-in) A pipelined group's batch window spans its slots up to its last active one, and an idle slot below an active one runs as a pad row that writes its state. A conversation that finished in such a slot is gone before its next turn. With STRATA_PIPELINE_ACTIVE_ROWS=1 (a layer split with --batch and more than one group) the window lists only the active slots (strata::program::pipeline_rows::pack, Verifier::batch_launch_rows on CUDA and SYCL): there is no pad row, an idle slot keeps its state, and INFO says slot_padding_safe=1. BT/BDONE name the packed rows' own slots. After a prompt's restore the engine says CACHE_SOURCE : where the reused prefix came from. Unset, every path is the one before. Measured on 3 x RTX A4000 16 GB (layer split, --batch 6 in 3 groups, IQ3_S, 200K context), the same binary with the variable off / on, tools/serve_load.py 6 clients x 2048 tokens, temperature 0, one run each: 177.4 / 173.5 tok/s total, first token p50 2.4 / 3.7 s; one client 67.3 / 66.7 tok/s. tests/core/pipeline_rows_test.cpp (registered in CMake): every active mask of 2, 4 and 8 slots, groups of 1, 2 and 4. Built on CUDA (sm_86, Linux); the SYCL change was not built here (no oneAPI on this machine). Co-Authored-By: Claude Opus 5.5 --- CMakeLists.txt | 3 ++ docs/BATCHING.md | 40 ++++++++++++++++ include/strata/core/verify.hpp | 2 + include/strata/program/pipeline_rows.hpp | 16 +++++++ src/core/verify.cpp | 9 +++- src/program/generate.cpp | 42 +++++++++++++---- sycl/include/strata/core/verify.hpp | 2 + sycl/src/core/verify.cpp | 12 +++-- tests/core/pipeline_rows_test.cpp | 59 ++++++++++++++++++++++++ 9 files changed, 169 insertions(+), 16 deletions(-) create mode 100644 include/strata/program/pipeline_rows.hpp create mode 100644 tests/core/pipeline_rows_test.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 369d699c070..a49688b32d9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1771,6 +1771,9 @@ if(STRATA_BUILD_TESTS OR STRATA_BUILD_CONVERSATION_TESTS) add_executable(batch_groups_test src/program/batch_groups_test.cpp) target_include_directories(batch_groups_test PRIVATE include) add_test(NAME batch_groups_test COMMAND batch_groups_test) + add_executable(pipeline_rows_test tests/core/pipeline_rows_test.cpp) + target_include_directories(pipeline_rows_test PRIVATE include) + add_test(NAME pipeline_rows_test COMMAND pipeline_rows_test) endif() if(STRATA_CPU_X86) diff --git a/docs/BATCHING.md b/docs/BATCHING.md index 70c09fb51df..e6b7fa4d3d6 100644 --- a/docs/BATCHING.md +++ b/docs/BATCHING.md @@ -386,6 +386,44 @@ one cold 64K prompt joining four streams: `STRATA_BATCH_DECODE_SHARE=2.0` 2.4 s s and 13.8 s. Pieces of one chunk (`PIECE=8192`) cost the long prompt the most. Not measured: other GPU pairs, three or more stages, HIP and SYCL (built for CUDA only), sampled decoding, prompts with pictures. +## Keeping conversations in pipelined slots (opt-in) + +With a layer split, `--batch` runs the slots in pipeline groups (`--batch-groups`). A group's batch window spans its +slots up to its last active one, and an idle slot below an active one runs as a pad row that writes its state. So in +a pipelined group a finished slot is not a reliable conversation cache, and the server places requests by the groups +alone. Agent clients (Claude Code, Codex) send long conversations - tens of thousands of tokens - in many short turns, +with short side requests in between, and whatever is lost between two turns is read again. + +What that cost on one machine: the engine log of 3 x RTX A4000 16 GB (layer split, `--batch` 3-6 in 3 groups, +Qwen3.8-Flash-Next IQ3_S, 200K context), two people working with Claude Code and Codex, 2026-10-09 to 2026-10-14. +Of 3,027 prompts of 8K tokens or more, 538 reused less than half of their previous turn, and 40.7M prompt tokens +were read again. By prompt lengths (the log has no content, so this is a heuristic): + +| Why the previous turn was gone | Prompts | Tokens read again | +| --- | ---: | ---: | +| it was in a batch slot that another conversation took | 170 | 16.4M | +| it ran on the solo path, and the next request (often a short side request) replaced the main session | 244 | 15.0M | +| the client changed the history (not addressed here) | 63 | 6.1M | +| no earlier turn (a first turn, or a restart) | 66 | 3.1M | + +At this machine's ~2,900 tok/s for long prompts, the first two rows are about three hours of prompt reading in six +days. What follows addresses them: the engine keeps idle slots and parks the solo path's conversation, the server +places requests by what the slots hold, and a placement policy (`serve/slot_policy.py`) decides which history to give +up. Each part is opt-in. + +### Engine: only the active slots in a window (`STRATA_PIPELINE_ACTIVE_ROWS=1`) + +With `STRATA_PIPELINE_ACTIVE_ROWS=1` (a layer split with `--batch` and more than one group), a group's window lists +only its active slots (`include/strata/program/pipeline_rows.hpp`, `Verifier::batch_launch_rows`): there is no pad +row, an idle slot keeps its conversation, and `INFO` says `slot_padding_safe=1`. After restoring a prompt's prefix the +engine says where it came from: `CACHE_SOURCE ` (-1: the main session, a checkpoint or the +conversation cache). + +Measured on the machine above (the same binary with the variable off / on, `tools/serve_load.py`, 6 clients x 2048 +tokens at temperature 0, one run each): 177.4 / 173.5 tok/s in total (-2.2%), first token p50 2.4 / 3.7 s; one client +67.3 / 66.7 tok/s. With all six slots busy the windows hold the same rows either way; runs of this configuration with +other builds measured 173.9-178.2 tok/s, so this is close to the spread between runs, but it was not repeated. + ## The VRAM expert tier keeps adapting in batch windows Batch windows count which experts they route to, and every `--adapt-every` windows (default 4) the engine swaps the @@ -707,6 +745,8 @@ On top of `GEN` / `GENI`: | `BYIELD ` | in | the prompt being read gives way at its next chunk boundary; its part read waits in `` (the admission's own, or a free slot for a solo request) | | `YIELDED ` | out | before the `DONE cancel` of a read that gave way: the request is sent again later and goes on from there | | `INFO ... batch_slots=N` | out | the slots the engine runs (only with `--batch`) | +| `INFO ... slot_padding_safe=1` | out | `STRATA_PIPELINE_ACTIVE_ROWS=1`: a pipelined group's window runs its active slots only; an idle slot keeps its conversation | +| `CACHE_SOURCE ` | out | `STRATA_PIPELINE_ACTIVE_ROWS=1`, after a prompt's restore: the idle slot its reused prefix came from (-1: the main session, a checkpoint or the conversation cache) and how long it is | `tools/batch_test.py` drives the engine directly: the same prompts alone, then together, compared token by token, and the aggregate rate. diff --git a/include/strata/core/verify.hpp b/include/strata/core/verify.hpp index 78609014a57..a024c14acbc 100644 --- a/include/strata/core/verify.hpp +++ b/include/strata/core/verify.hpp @@ -243,6 +243,8 @@ class Verifier { // a window in flight from one thread (batch_poll does not block). Stage k can then run group g while stage k+1 // runs group g-1. Rows of group `base` use hand-off rows [base, base + S), so groups never share a hand-off row. bool batch_launch(int base, int S, const int32_t* tokens, const int64_t* pos, std::string& err); + // STRATA_PIPELINE_ACTIVE_ROWS=1: packed active rows bind actual slot states; hand-off storage still starts at base. + bool batch_launch_rows(int base, int S, const int* rows, const int32_t* tokens, const int64_t* pos, std::string& err); /// 1 = this stage's window and commit are done (the last stage's picks are in batch_out), 0 = still running, /// -1 = an error (err). Serves every layer that has rung so far. int batch_poll(PoolMultiFn pool, void* user, std::string& err); diff --git a/include/strata/program/pipeline_rows.hpp b/include/strata/program/pipeline_rows.hpp new file mode 100644 index 00000000000..99969726573 --- /dev/null +++ b/include/strata/program/pipeline_rows.hpp @@ -0,0 +1,16 @@ +// STRATA_PIPELINE_ACTIVE_ROWS=1: active-row packing; unlisted slot state is never submitted to the verifier. +#pragma once +#include +#include +namespace strata::program::pipeline_rows { +template +int pack(const Slots& slots, int base, int count, int* rows, int32_t* tokens, int64_t* pos) { + int n = 0; + for (int b = base; b < base + count; ++b) { + const auto& slot = slots[(std::size_t) b]; + if (!slot.active) continue; + rows[n] = b; tokens[n] = slot.x; pos[n] = slot.p; ++n; + } + return n; +} +} // namespace strata::program::pipeline_rows diff --git a/src/core/verify.cpp b/src/core/verify.cpp index c8eef3f30e9..3ca2cf22ff4 100644 --- a/src/core/verify.cpp +++ b/src/core/verify.cpp @@ -4569,10 +4569,15 @@ bool Verifier::sample_rows(int S, std::string& err) { } bool Verifier::batch_launch(int base, int S, const int32_t* tokens, const int64_t* pos, std::string& err) { - const OnDevice on_device(device_); - if (b_running_) { err = "verify: batch_launch while this stage is busy"; return false; } int rows[8] = {}; for (int t = 0; t < S && t < 8; ++t) rows[t] = base + t; + return batch_launch_rows(base, S, rows, tokens, pos, err); +} + +bool Verifier::batch_launch_rows(int base, int S, const int* rows, const int32_t* tokens, + const int64_t* pos, std::string& err) { + const OnDevice on_device(device_); + if (b_running_) { err = "verify: batch_launch while this stage is busy"; return false; } if (pre_launch_ && !pre_launch_(cs_, err)) return false; if (!stage_batch(rows, S, base, tokens, pos, err)) return false; cudaError_t le = cudaGraphLaunch(exec_bm_[bkey(rows, S, base)], cs_); diff --git a/src/program/generate.cpp b/src/program/generate.cpp index 824bd19dbe6..a171b30c64f 100644 --- a/src/program/generate.cpp +++ b/src/program/generate.cpp @@ -100,6 +100,7 @@ #include "strata/spec/suffix_drafter.hpp" #include "strata/spec/spec_start.hpp" #include "strata/program/batch_groups.hpp" +#include "strata/program/pipeline_rows.hpp" #include "strata/kernels/cvec.hpp" #include "strata/core/progress.hpp" #include "strata/core/device.hpp" @@ -12128,6 +12129,9 @@ int main(int argc, char** argv) { } } } + const char* active_rows_env = std::getenv("STRATA_PIPELINE_ACTIVE_ROWS"); + const bool pipeline_active_rows = o.batch > 0 && o.batch_groups > 1 && !stages.empty() && + active_rows_env != nullptr && std::strcmp(active_rows_env, "1") == 0; // what the server's Monitor tab shows (servers before 0.1.8 skip unknown lines until READY) { size_t free_b = 0, total_b = 0; @@ -12174,7 +12178,8 @@ int main(int argc, char** argv) { strata::prefill::Prefill::pipe_setting(), o.batch > 0 ? (" batch_slots=" + std::to_string(o.batch) + " slot_cache=" + std::to_string(o.prompt_cache > 0 ? 1 : 0) + - " batch_groups=" + std::to_string(o.batch_groups)).c_str() : ""); + " batch_groups=" + std::to_string(o.batch_groups) + + (pipeline_active_rows ? " slot_padding_safe=1" : "")).c_str() : ""); } // issue #29: a request whose heartbeat (tokens, prompt chunks, verify windows) stops for this long is stuck on // a flag nobody will raise - end the engine with where it was, so the server starts it again instead of the @@ -12896,11 +12901,12 @@ int main(int argc, char** argv) { int32_t tok[strata::kernels::kVerifyMaxT] = {}; int64_t pos[strata::kernels::kVerifyMaxT] = {}; int64_t since = 0; ///< tick it started waiting (fairness) - int S = 0; ///< slots in its window: up to its last active one (idle ones cost rows) + int S = 0; ///< legacy span, or packed active-row count int active = 0; ///< live slots at launch; S can also include idle padding rows Clock::time_point started; ///< the group window's first stage launch // STRATA_PIPED_MTP (pmtp): S counts ROWS - each of the group's slots' token, then that slot's proposals - at - // hand-off rows [hbase, hbase + S) + // hand-off rows [hbase, hbase + S). STRATA_PIPELINE_ACTIVE_ROWS=1: the packed slot IDs, immutable while in + // flight int rows[strata::kernels::kVerifyMaxT] = {}; int hbase = 0; }; @@ -13122,14 +13128,15 @@ int main(int argc, char** argv) { const int32_t* outb = vk.batch_out(); if (pmtp && !pmtp_verdict(G, vk, outb)) return false; for (int t = 0; t < G.S && !pmtp; ++t) { - BSlot& sl = bs[(size_t) (gi * GS + t)]; + const int slot = pipeline_active_rows ? G.rows[t] : gi * GS + t; + BSlot& sl = bs[(size_t) slot]; if (!sl.active) continue; if (vk.nonfinite_in(t)) { // #879: the slot ends, nothing emitted std::fprintf(stderr, "strata batch: non-finite logits (#879) in slot %d at position %lld: it " "ends (BDONE nonfinite) and the next request reads its prompt from token 0\n", - gi * GS + t, (long long) sl.p); + slot, (long long) sl.p); const double ms = std::chrono::duration(Clock::now() - sl.t0).count(); - std::printf("BDONE %d %lld nonfinite %.1f\n", gi * GS + t, (long long) sl.produced, ms); + std::printf("BDONE %d %lld nonfinite %.1f\n", slot, (long long) sl.produced, ms); sl.active = false; sl.cached = false; sl.checks.clear(); @@ -13137,7 +13144,7 @@ int main(int argc, char** argv) { continue; } const int32_t y = outb[t]; - std::printf("BT %d %d\n", gi * GS + t, (int) y); + std::printf("BT %d %d\n", slot, (int) y); ++sl.produced; ++bt_rows; ++pipe_rows; @@ -13147,7 +13154,7 @@ int main(int argc, char** argv) { sl.ids.push_back(sl.x); if (fin != nullptr) { const double ms = std::chrono::duration(Clock::now() - sl.t0).count(); - std::printf("BDONE %d %lld %s %.1f\n", gi * GS + t, (long long) sl.produced, fin, ms); + std::printf("BDONE %d %lld %s %.1f\n", slot, (long long) sl.produced, fin, ms); sl.active = false; // a pipelined slot is a cache again (#857: a request left alone in its slot goes back to the solo path with its // drafts): its state is final once its last window has left the last stage. A later group window whose pad @@ -13223,6 +13230,9 @@ int main(int argc, char** argv) { } } if (G.S == 0) { G.inflight = false; continue; } // every slot of the group went idle + } else if (pipeline_active_rows) { + G.S = strata::program::pipeline_rows::pack(bs, pick * GS, GS, G.rows, G.tok, G.pos); + G.active = G.S; } else { G.active = 0; for (int t = 0; t < GS; ++t) if (bs[(size_t) (pick * GS + t)].active) { @@ -13251,8 +13261,12 @@ int main(int argc, char** argv) { PGroup& G = pg[(size_t) pick]; strata::core::progress().busy.store(true); if (batch_telemetry_enabled) stage_in_flight.launch(k, Clock::now()); - if (!(pmtp ? stage_verifier(k).batch_launch_rows(G.rows, G.S, G.hbase, G.tok, G.pos, err) - : stage_verifier(k).batch_launch(pick * GS, G.S, G.tok, G.pos, err))) { + const bool launched = pmtp + ? stage_verifier(k).batch_launch_rows(G.rows, G.S, G.hbase, G.tok, G.pos, err) + : pipeline_active_rows + ? stage_verifier(k).batch_launch_rows(pick * GS, G.S, G.rows, G.tok, G.pos, err) + : stage_verifier(k).batch_launch(pick * GS, G.S, G.tok, G.pos, err); + if (!launched) { std::printf("ERR %s\n", err.c_str()); return false; } @@ -14030,6 +14044,7 @@ int main(int argc, char** argv) { // active slot's checkpoints too (its windows write its cells from p on, so the ones below a checkpoint still // hold its prompt; the groups have finished before a request is read) int slot_source = -1; + int observed_slot_source = -1; int64_t slot_tokens = 0; bool resumed_from0 = false; // a prompt read parked in a slot (BYIELD) that had started at token 0 // --prefill-pipe: the plan of the segment a read goes on in (a BYIELD park, a periodic checkpoint), and @@ -14321,6 +14336,7 @@ int main(int argc, char** argv) { } live_imgs.clear(); resume = slot_tokens; + observed_slot_source = slot_source; from_live = true; std::fprintf(stderr, "strata batch: slot %d gave back %lld tokens of this conversation (%s) in " "%.1f ms\n", slot_source, (long long) slot_tokens, @@ -14424,6 +14440,12 @@ int main(int argc, char** argv) { std::chrono::duration(Clock::now() - t0).count(), conversations.size(), conversations.bytes()); } + // Optional protocol metadata, emitted only after a successful restore. + // It describes the source, never a batch execution-slot reservation. + if (pipeline_active_rows) { + std::printf("CACHE_SOURCE %d %lld\n", observed_slot_source, (long long) resume); + std::fflush(stdout); + } if (want_cvec != cvec_cached) { live_ok = false; checks.clear(); diff --git a/sycl/include/strata/core/verify.hpp b/sycl/include/strata/core/verify.hpp index b38249f4dc1..7d0c30dab61 100644 --- a/sycl/include/strata/core/verify.hpp +++ b/sycl/include/strata/core/verify.hpp @@ -168,6 +168,8 @@ class Verifier { // a window in flight from one thread (batch_poll does not block). Stage k can then run group g while stage k+1 // runs group g-1. Rows of group `base` use hand-off rows [base, base + S), so groups never share a hand-off row. bool batch_launch(int base, int S, const int32_t* tokens, const int64_t* pos, std::string& err); + // STRATA_PIPELINE_ACTIVE_ROWS=1: packed active rows bind actual slot states; hand-off storage still starts at base. + bool batch_launch_rows(int base, int S, const int* rows, const int32_t* tokens, const int64_t* pos, std::string& err); /// 1 = this stage's window and commit are done (the last stage's picks are in batch_out), 0 = still running, /// -1 = an error (err). Serves every layer that has rung so far. int batch_poll(PoolMultiFn pool, void* user, std::string& err); diff --git a/sycl/src/core/verify.cpp b/sycl/src/core/verify.cpp index a20208624a8..c84e45cedd5 100644 --- a/sycl/src/core/verify.cpp +++ b/sycl/src/core/verify.cpp @@ -3387,12 +3387,16 @@ catch (sycl::exception const &exc) { std::exit(1); } -bool Verifier::batch_launch(int base, int S, const int32_t *tokens, - const int64_t *pos, std::string &err) try { - const OnDevice on_device(device_); - if (b_running_) { err = "verify: batch_launch while this stage is busy"; return false; } +bool Verifier::batch_launch(int base, int S, const int32_t* tokens, const int64_t* pos, std::string& err) { int rows[8] = {}; for (int t = 0; t < S && t < 8; ++t) rows[t] = base + t; + return batch_launch_rows(base, S, rows, tokens, pos, err); +} + +bool Verifier::batch_launch_rows(int base, int S, const int* rows, const int32_t* tokens, + const int64_t* pos, std::string& err) try { + const OnDevice on_device(device_); + if (b_running_) { err = "verify: batch_launch while this stage is busy"; return false; } if (!stage_batch(rows, S, base, tokens, pos, err)) return false; dpct::err0 le = DPCT_CHECK_ERROR( (cs_)->ext_oneapi_graph(*exec_bm_[bkey(rows, S, base)])); diff --git a/tests/core/pipeline_rows_test.cpp b/tests/core/pipeline_rows_test.cpp new file mode 100644 index 00000000000..f0f0898441e --- /dev/null +++ b/tests/core/pipeline_rows_test.cpp @@ -0,0 +1,59 @@ +// include/strata/program/pipeline_rows.hpp: a pipelined group's window lists its active slots only, in order, with +// their tokens and positions; idle slots are untouched; the packed rows stay as they were when a slot changes later. +// Header-only, CPU. Checks do not depend on assert (they hold in Release builds too). +#include "strata/program/pipeline_rows.hpp" + +#include +#include +#include + +namespace { +int failures = 0; +void check(bool ok, const char* what, int count, int mask, int base) { + if (!ok) { + std::fprintf(stderr, "FAIL: %s (slots %d, active mask %d, base %d)\n", what, count, mask, base); + ++failures; + } +} +struct Slot { + bool active; + int32_t x; + int64_t p; + bool cached; +}; +} // namespace + +int main() { + for (int count : {2, 4, 8}) + for (int mask = 0; mask < (1 << count); ++mask) { + std::array slots{}; + for (int b = 0; b < count; ++b) slots[(size_t) b] = {(mask & (1 << b)) != 0, 100 + b, 10000 + b, true}; + const auto before = slots; + for (int gs : {1, 2, 4}) + for (int base = 0; base + gs <= count; base += gs) { + int rows[8]{}; + int32_t tokens[8]{}; + int64_t pos[8]{}; + const int n = strata::program::pipeline_rows::pack(slots, base, gs, rows, tokens, pos); + int k = 0; + for (int b = base; b < base + gs; ++b) + if (slots[(size_t) b].active) { + check(k < n && rows[k] == b && tokens[k] == slots[(size_t) b].x && + pos[k] == slots[(size_t) b].p, "active slot packed in order", count, mask, base); + ++k; + } + check(k == n, "only the active slots", count, mask, base); + bool untouched = true; + for (int b = 0; b < count; ++b) untouched = untouched && slots[(size_t) b].cached == before[(size_t) b].cached; + check(untouched, "idle slots keep their cache flag", count, mask, base); + if (n > 0) { // a slot that changes after the window was packed does not move its rows + const int first = rows[0]; + slots[(size_t) first].active = !slots[(size_t) first].active; + check(rows[0] == first, "packed rows are the window's own copy", count, mask, base); + slots = before; + } + } + } + if (failures == 0) std::printf("pipeline_rows_test: all checks passed\n"); + return failures == 0 ? 0 : 1; +} From da5a46295f767ca582ad27220ce4057085c78af3 Mon Sep 17 00:00:00 2001 From: andyhrc <12491949+andyhrc@users.noreply.github.com> Date: Thu, 15 Oct 2026 10:04:20 +0800 Subject: [PATCH 02/21] batch: park=B keeps the main session's conversation in an idle slot before a different request replaces it (with STRATA_PIPELINE_ACTIVE_ROWS=1) A request alone runs on the solo path, whose state is the main session. When the next request is another conversation - an agent client's short side request between two turns of a long one - the main session is overwritten and the long conversation's next turn reads its whole history again. New request key park=B on GEN/BGEN (INFO slot_park=1, only with STRATA_PIPELINE_ACTIVE_ROWS=1): when the prompt continues less than half of what the main session holds, the engine first copies the main session's conversation (copy_to_slot, with its last checkpoint) into idle slot B - never the slot it restores from or is admitted to - and says PARKED B (0: the copy failed and the slot holds nothing usable). A park= it does not use is logged. Without the key nothing changes. Measured on 3 x RTX A4000 (0.1.43.1 and 0.1.44.1): parking 36-37K tokens took 414-451 ms in five runs; the next turn of a 37,359-token conversation after a side request reused 37,352 tokens (0 without park=) with the same greedy output as before the side request. Co-Authored-By: Claude Opus 5.5 --- docs/BATCHING.md | 11 ++++++++++ src/program/generate.cpp | 45 +++++++++++++++++++++++++++++++++++++++- 2 files changed, 55 insertions(+), 1 deletion(-) diff --git a/docs/BATCHING.md b/docs/BATCHING.md index e6b7fa4d3d6..613a0ca9e4f 100644 --- a/docs/BATCHING.md +++ b/docs/BATCHING.md @@ -424,6 +424,15 @@ tokens at temperature 0, one run each): 177.4 / 173.5 tok/s in total (-2.2%), fi 67.3 / 66.7 tok/s. With all six slots busy the windows hold the same rows either way; runs of this configuration with other builds measured 173.9-178.2 tok/s, so this is close to the spread between runs, but it was not repeated. +### Engine: keep the solo path's conversation (`park=B`) + +A request alone runs on the solo path, whose state is the engine's main session, not a slot. When the next request is +another conversation - an agent's short side request between two turns of a long one - the main session is +overwritten. With `STRATA_PIPELINE_ACTIVE_ROWS=1` the engine takes `park=B` on `GEN` / `BGEN` (`INFO slot_park=1`): +when the prompt continues less than half of what the main session holds, it first copies the main session's +conversation (and its last checkpoint) into idle slot B - never the slot it restores from or is admitted to - and says +`PARKED B `. Copying 36-37K tokens took 414-451 ms (five runs, the machine above). + ## The VRAM expert tier keeps adapting in batch windows Batch windows count which experts they route to, and every `--adapt-every` windows (default 4) the engine swaps the @@ -747,6 +756,8 @@ On top of `GEN` / `GENI`: | `INFO ... batch_slots=N` | out | the slots the engine runs (only with `--batch`) | | `INFO ... slot_padding_safe=1` | out | `STRATA_PIPELINE_ACTIVE_ROWS=1`: a pipelined group's window runs its active slots only; an idle slot keeps its conversation | | `CACHE_SOURCE ` | out | `STRATA_PIPELINE_ACTIVE_ROWS=1`, after a prompt's restore: the idle slot its reused prefix came from (-1: the main session, a checkpoint or the conversation cache) and how long it is | +| `park=B` (a key of `GEN` / `BGEN`) | in | with `INFO slot_park=1`: before a different conversation replaces the main session's, copy that one into idle slot B | +| `PARKED ` | out | `park=` copied that many tokens into the slot (0: the copy failed, the slot holds nothing usable) | `tools/batch_test.py` drives the engine directly: the same prompts alone, then together, compared token by token, and the aggregate rate. diff --git a/src/program/generate.cpp b/src/program/generate.cpp index a171b30c64f..de60cdf7928 100644 --- a/src/program/generate.cpp +++ b/src/program/generate.cpp @@ -12179,7 +12179,7 @@ int main(int argc, char** argv) { o.batch > 0 ? (" batch_slots=" + std::to_string(o.batch) + " slot_cache=" + std::to_string(o.prompt_cache > 0 ? 1 : 0) + " batch_groups=" + std::to_string(o.batch_groups) + - (pipeline_active_rows ? " slot_padding_safe=1" : "")).c_str() : ""); + (pipeline_active_rows ? " slot_padding_safe=1 slot_park=1" : "")).c_str() : ""); } // issue #29: a request whose heartbeat (tokens, prompt chunks, verify windows) stops for this long is stuck on // a flag nobody will raise - end the engine with where it was, so the server starts it again instead of the @@ -13718,6 +13718,10 @@ int main(int argc, char** argv) { // a parked conversation holding it stays parked) and a later request that resumes from it does not park the // branch it leaves: N queries cost one prefix, not N. Absent = as before. int64_t req_pin = 0; + // STRATA_PIPELINE_ACTIVE_ROWS=1: park=B - when this request replaces the main session with another + // conversation, first copy the outgoing one into idle batch slot B (chosen by the server), so a short side + // request between two turns of a long solo conversation does not cost that conversation its cache + int req_park = -1; // tuning keys (setup's calibration measures settings without restarting the engine): the PCIe share of // the missed experts and the draft-probability floor, for this request only double req_pcie_frac = o.pcie_frac, req_spec_min_p = o.spec_min_p; @@ -13745,6 +13749,7 @@ int main(int argc, char** argv) { else if (key == "ckpt") req_ckpt = std::atoi(tok.c_str() + eq + 1) != 0; else if (key == "prefill") req_prefill_only = std::atoi(tok.c_str() + eq + 1) != 0; else if (key == "pin") req_pin = std::max(0, std::atoll(tok.c_str() + eq + 1)); + else if (key == "park") req_park = std::atoi(tok.c_str() + eq + 1); else if (key == "temperature") req_temperature = fv; else if (key == "top_p") req_top_p = fv; else if (key == "top_k") req_top_k = std::atoi(tok.c_str() + eq + 1); @@ -14205,6 +14210,44 @@ int main(int argc, char** argv) { resume == req_pin && live_ok) for (const ConvCheckpoint& c : checks) if ((int64_t) c.ids.size() == resume && c.pinned) pin_sibling = true; + // park=B (see req_park). Only when this prompt is another conversation (it continues less than + // half of what the main session holds), never into the slot this request restores from or runs in. + if (pipeline_active_rows && req_park >= 0 && req_park < (int) bs.size() && !bs[(size_t) req_park].active && + req_park != admit_slot && req_park != slot_source && live_ok && !live.empty() && + resume * 2 < (int64_t) live.size() && (!from_live || incoming || slot_source >= 0) && !pin_sibling) { + const auto t0 = Clock::now(); + BSlot& sl = bs[(size_t) req_park]; + if (copy_to_slot(req_park, live, err)) { + sl = BSlot{}; + sl.cached = true; + sl.ids = live; + sl.cvec = cvec_cached; + sl.img = !live_imgs.empty(); // pictures: not matched again by tokens alone + const ConvCheckpoint* best = nullptr; + for (const ConvCheckpoint& c : checks) + if (c.ids.size() < live.size() && (best == nullptr || c.ids.size() > best->ids.size()) && + std::equal(c.ids.begin(), c.ids.end(), live.begin())) + best = &c; + if (best != nullptr && !sl.img) sl.checks.push_back(*best); + std::fprintf(stderr, "strata batch: slot %d keeps the previous conversation (%zu tokens, copied in " + "%.1f ms)\n", req_park, live.size(), + std::chrono::duration(Clock::now() - t0).count()); + std::printf("PARKED %d %zu\n", req_park, sl.img ? (size_t) 0 : live.size()); + } else { + std::fprintf(stderr, "strata batch: parking into slot %d failed (%s)\n", req_park, err.c_str()); + err.clear(); + sl.cached = false; // its sessions may be half written + sl.checks.clear(); + std::printf("PARKED %d 0\n", req_park); + } + std::fflush(stdout); + } else if (req_park >= 0) { + std::fprintf(stderr, "strata batch: park=%d not used (rows=%d live_ok=%d live=%zu resume=%lld " + "from_live=%d source=%d admit=%d active=%d)\n", req_park, + pipeline_active_rows ? 1 : 0, live_ok ? 1 : 0, live.size(), (long long) resume, + from_live ? 1 : 0, slot_source, admit_slot, + req_park < (int) bs.size() && bs[(size_t) req_park].active ? 1 : 0); + } // --conversation-cache-park-gain N: a park now would supersede the entry parked_prefix_tokens() reports (the copy // this conversation wrote last), so a conversation that advanced fewer than N tokens since then keeps it: // it still holds this conversation's prefix, and re-parking writes the whole snapshot only for the next From 2b64831ff2790acee8400044735d058eeabe7839 Mon Sep 17 00:00:00 2001 From: andyhrc <12491949+andyhrc@users.noreply.github.com> Date: Thu, 15 Oct 2026 10:04:20 +0800 Subject: [PATCH 03/21] serve: STRATA_SLOT_CACHE_AFFINITY=1 - place batch requests by what the slots hold, with the placement policy as an interface (serve/slot_policy.py) With pipelined groups the server placed requests by the groups alone, because the engine kept no pipelined slot as a cache. On a 3 x RTX A4000 server used by two people with Claude Code and Codex, six days of engine log (3,027 prompts of 8K tokens or more) show 16.4M prompt tokens read again because the previous turn's slot had been taken by another conversation, and 15.0M because the previous turn ran on the solo path and the next request replaced it. With STRATA_SLOT_CACHE_AFFINITY=1 and pipelined groups, the server: - asks serve/slot_policy.place() for the slot, with each slot's tokens, last use (from BDONE), validity, the spreading order, the group size, the observed prompt rate and the engine's slot_padding_safe; without that engine capability it tracks from BADM/BT/BDONE which idle slot a later window of its group has padded over; - waits when the policy answers "protected": once per request, at most slot_policy.PROTECTION_WAIT_S, the control lines given back, re-checking every 0.1 s; a cancel leaves every slot as it was; - tracks what the main (solo) session holds and, with INFO slot_park=1, asks slot_policy.park_destination() for a slot and sends park=B when a request is another conversation of at least slot_policy.PARK_MIN_TOKENS; PARKED updates that slot's state, an unused claim is released. /metrics gains live.cache_events (the last 96: allocation, admission with the tokens reused/read, resume and cache source, park, protection waits, expiries), request keys and client addresses (the socket's; X-Forwarded-For is not used) in live and requests[], live.solo_*, and per-slot cache fields; with STRATA_CACHE_TRACE=1 a per-request snapshot of how long a start each slot shared with the prompt (serve/cache_trace.py, lengths only). These /metrics fields are present with the variables off too (additive); generation is unchanged. serve/slot_policy.py is the interface here: place, park_destination, cache_state, superseded_slots, reliable_flags, configure/settings, PROTECTION_WAIT_S, PARK_MIN_TOKENS. Which history to give up is the policy's choice - our policy is the next commit, and another one can replace it without touching the server. With STRATA_SLOT_CACHE_AFFINITY=1 or STRATA_CACHE_TRACE=1 and no policy, NotImplementedError. Tests: serve/test_slot_alloc.py, serve/test_parallel.py, serve/test_cache_trace.py (those that need a policy come with it). Full `python -m unittest discover -s serve -t .` passes. Co-Authored-By: Claude Opus 5.5 --- docs/BATCHING.md | 34 +++- serve/cache_trace.py | 18 ++ serve/server.py | 338 +++++++++++++++++++++++++++++++++++--- serve/slot_policy.py | 80 +++++++++ serve/test_cache_trace.py | 29 ++++ serve/test_parallel.py | 54 +++++- serve/test_slot_alloc.py | 146 +++++++++++++++- 7 files changed, 675 insertions(+), 24 deletions(-) create mode 100644 serve/cache_trace.py create mode 100644 serve/slot_policy.py create mode 100644 serve/test_cache_trace.py diff --git a/docs/BATCHING.md b/docs/BATCHING.md index 613a0ca9e4f..e171927faf1 100644 --- a/docs/BATCHING.md +++ b/docs/BATCHING.md @@ -185,7 +185,8 @@ about 10-25% speed per request on this card". `--parallel N` is honoured as aske engine copies its state back (50-60 ms for a short conversation) instead of reading the history again - also for a client that drops the reply's thinking from the history (the checkpoint matches up to the new turn). A new conversation takes an empty slot, else the one used longest ago. -- Slots are assigned so that consecutive requests land in different pipeline groups (`--batch-groups`). +- Slots are assigned so that consecutive requests land in different pipeline groups (`--batch-groups`) - with + `STRATA_SLOT_CACHE_AFFINITY=1`, by the conversations they hold (below). - A client that disconnects stops its slot (`BSTOP`); the others go on. ### Long prompts that move from solo to a batch slot @@ -433,6 +434,37 @@ when the prompt continues less than half of what the main session holds, it firs conversation (and its last checkpoint) into idle slot B - never the slot it restores from or is admitted to - and says `PARKED B `. Copying 36-37K tokens took 414-451 ms (five runs, the machine above). +### Server: placement by what the slots hold (`STRATA_SLOT_CACHE_AFFINITY=1`) + +With `"env": {"STRATA_SLOT_CACHE_AFFINITY": "1"}` in the server config and pipelined groups, `pick_slot` asks the +placement policy in `serve/slot_policy.py` instead of placing by the groups alone. The server side is the same for any +policy: + +- The policy gets each slot's state: busy, the tokens it holds, when it was last used (counted from the request's real + end, `BDONE`), whether its state is intact, the spreading order, the group size, the observed prompt rate and + whether the engine runs active slots only (`slot_padding_safe=1`). Without that engine capability the server tracks + from the `BADM` / `BT` / `BDONE` lines which idle slot a later window of its group has padded over, and marks it. +- An answer "protected" makes the admission wait - the control lines given back, the slots re-checked every 0.1 s - + at most `PROTECTION_WAIT_S` once per request, and then ask again allowing an eviction. A cancelled request leaves + every slot as it was. +- With `INFO slot_park=1` the server tracks what the main session holds; when a request is another conversation (the + first half differs) and the held one has at least `PARK_MIN_TOKENS`, it asks the policy for a slot and sends + `park=B`; `PARKED` updates that slot's state. A claim the engine did not use is released. + +The policy's interface is `place`, `park_destination`, `cache_state`, `superseded_slots`, `reliable_flags`, +`configure` / `settings` and the constants `PROTECTION_WAIT_S`, `PARK_MIN_TOKENS` (their docstrings say what they +answer). This part ships them as an interface without a policy: with `STRATA_SLOT_CACHE_AFFINITY=1` (or +`STRATA_CACHE_TRACE=1`) and no policy they raise `NotImplementedError`; with both off nothing changes. + +`/metrics` shows what happened: `live.cache_events` (the last 96: allocation, admission with the tokens reused and +read, resume and cache source, park claims and `parked`, protection waits, expiries), each request's key, client +address, slot and cache source in `requests[]`, `live.peer_ip` and `live.solo_*` (the request on the solo path and +where its cache came from), and for slots the client of the conversation each one holds; with +`STRATA_SLOT_CACHE_AFFINITY=1` also each idle slot's protection and scores and `live.slot_policy`; with +`STRATA_CACHE_TRACE=1` each request's snapshot of how long a start every slot shared with its prompt (lengths only, +`serve/cache_trace.py`). The client address is the socket's (behind a reverse proxy, the proxy's); `X-Forwarded-For` +and request fields are not used. No prompt text or token ids are recorded. + ## The VRAM expert tier keeps adapting in batch windows Batch windows count which experts they route to, and every `--adapt-every` windows (default 4) the engine swaps the diff --git a/serve/cache_trace.py b/serve/cache_trace.py new file mode 100644 index 00000000000..23bb7d506df --- /dev/null +++ b/serve/cache_trace.py @@ -0,0 +1,18 @@ +"""Metadata-only cache prefix inspection; no prompt text or token IDs are recorded.""" +from . import slot_policy + +def snapshot(prompt, held, busy, valid, used, now): + rows=[] + for b,h in enumerate(held): + hi=min(len(prompt),len(h));lo=0 + if h[:hi]==prompt[:hi]:lo=hi + else: + while lo 1 and groups_run > 1 else 0 + self.slot_padding_safe = self.info.get("slot_padding_safe") in (1, "1") self.slot_q = [queue.Queue() for _ in range(self.batch)] self.slot_busy = [False] * self.batch # #1603: who owns a busy flag. slot_claims: slots a request of this server picked or reserved and has not @@ -1172,6 +1178,19 @@ def __init__(self, exe: str, args: list[str], cwd: str | None = None, log: str | # the slot that has its start (the engine checks it again); when the slot was last used self.slot_held: list[list[int]] = [[] for _ in range(self.batch)] self.slot_used = [0.0] * self.batch + # park=: what the engine's main (solo) session holds, as far as this server knows, and slots claimed for a + # park= copy of it until the engine says PARKED + self.main_held: list[int] = [] + self.main_peer = None + self._park_claims: set[int] = set() + self.slot_cache_peer = [None] * self.batch + self._cache_expiry_seen = [None] * self.batch + self._cache_events = collections.deque(maxlen=96) + self._cache_event_lock = threading.Lock() + self.slot_cache_valid = [False] * self.batch + self._slot_native_active = [False] * self.batch + self._slot_steps = [0] * self.batch + self._slot_valid_until = [{} for _ in range(self.batch)] self.slot_live: list[dict | None] = [None] * self.batch # /metrics: the request in each slot # #1012: the admission state belongs to the server, not to one engine process. restart() runs this again # while requests still wait on it (they hold these very objects), so it is made once and kept: new ones @@ -1196,6 +1215,8 @@ def _pump(self): slot_q = self.slot_q line = None for line in proc.stdout: + if self.proc is proc and self.slot_cache_affinity and line.startswith(("BT ", "BDONE ", "BADM ")): + self._observe_cache_line(line) # checked before batch routing: a fatal line is never a slot's own if line.startswith(FATAL_PREFIXES): # release_gpu_waits invalidates the verifier, even if the native @@ -1233,6 +1254,59 @@ def _pump(self): for q in slot_q: q.put(None) + def cache_event(self, kind, **fields): + """Bounded metadata only; never record prompt contents or credentials.""" + tl = getattr(self, "_tl", None) + if kind not in ("expired", "padding_invalidated"): + for key in ("request_key", "peer_ip"): + value = getattr(tl, key, None) + if value is not None: fields.setdefault(key, value) + lock = self.__dict__.setdefault("_cache_event_lock", threading.Lock()) + with lock: + events = self.__dict__.setdefault("_cache_events", collections.deque(maxlen=96)) + events.append({"time": time.time(), "kind": kind, **fields}) + + def cache_events_view(self): + lock = self.__dict__.setdefault("_cache_event_lock", threading.Lock()) + with lock: + return list(getattr(self, "_cache_events", ()))[::-1] + + def _observe_cache_line(self, line: str) -> None: + """Remember observed padding invalidation without treating held tokens as native truth. + + The lower row's BDONE precedes the higher row's BT in that SAME window. Allow + that BT; a later higher-row window pads over the now-idle lower cache. + """ + gs = getattr(self, "slot_gs", 0) + if not gs: + return + f = line.split() + try: + b = int(f[1]) + if not 0 <= b < self.batch: + return + with self.slot_cv: + if f[0] == "BADM": + self._slot_native_active[b] = f[2] == "1" + self.slot_cache_valid[b] = False + elif f[0] == "BDONE": + self._slot_native_active[b] = False + self.slot_cache_valid[b] = f[3] != "nonfinite" + self._slot_valid_until[b] = {x: self._slot_steps[x] + int(self._slot_native_active[x]) + for x in range(b + 1, b - b % gs + gs)} + self.slot_used[b] = time.time() + elif f[0] == "BT": + self._slot_native_active[b] = True + self._slot_steps[b] += 1 + for x in ([] if self.slot_padding_safe else range(b - b % gs, b)): + if (not self._slot_native_active[x] and self.slot_cache_valid[x] and + self._slot_steps[b] > self._slot_valid_until[x].get(b, 0)): + self.slot_cache_valid[x] = False + self.cache_event("padding_invalidated", slot=x, by_slot=b) + self.slot_cv.notify_all() + except (IndexError, ValueError): + return + def death_note(self) -> str: """Why the engine most likely ended, from the end of its log: its own watchdog (issue #29), else RAM.""" if getattr(self, "silent_note", None): # #481: the server ended it, not the OS or the engine itself @@ -1544,6 +1618,11 @@ def _control(self, cancel, on_token, stop_when=None, on_quiet=None, want_lp=Fals self._send("STOP") stopped = True yield False # a token is pending (not a heartbeat) + elif line.startswith("RESUME "): + reused = int(line.split()[1]) + self.reused = reused + if self._ctl_mode == "solo": self.solo_cache_reused_tokens = reused + self.cache_event("resume", reused_tokens=reused, path=self._ctl_mode) elif line.startswith("PP "): f = line.split() if len(f) >= 3 and f[1].isdigit() and f[2].isdigit(): @@ -1559,6 +1638,22 @@ def _control(self, cancel, on_token, stop_when=None, on_quiet=None, want_lp=Fals elif line.startswith("DONE"): self._parse_done(line) self._last_done = line + elif line.startswith("PARKED "): + f = line.split() + if len(f) == 3 and f[1].isdigit() and f[2].isdigit(): + self._on_parked(int(f[1]), int(f[2])) + elif line.startswith("CACHE_SOURCE "): + f = line.split() + if len(f) == 3 and f[1].lstrip("-").isdigit() and f[2].isdigit(): + source, reused = int(f[1]), int(f[2]) + source_tl = self.__dict__.setdefault("_tl", threading.local()) + source_tl.cache_source_slot = source if source >= 0 else None + source_tl.cache_source_kind = "slot" if source >= 0 else "main_or_checkpoint" + if self._ctl_mode == "solo": + self.solo_cache_source_slot = source if source >= 0 else None + self.solo_cache_reused_tokens = reused if source >= 0 else None + self.cache_event("cache_source", slot=source if source >= 0 else None, + reused_tokens=reused, path=self._ctl_mode) elif line.startswith("BADM "): f = line.split() self._ctl_result = ("badm", len(f) >= 3 and f[2] == "1") @@ -1637,6 +1732,7 @@ def _release_slot_when_done(self, slot: int, stream: list[int] | None = None): except EngineDied: pass q, busy, held = self.slot_q[slot], self.slot_busy, self.slot_held # this process's: restart() replaces them + used, affinity = self.slot_used, self.slot_cache_affinity releasing = self.slot_releasing releasing.add(slot) @@ -1662,6 +1758,8 @@ def wait(): break held[slot] = tail[:-1] if stream and tail else [] with self.slot_cv: + if affinity: + used[slot] = time.time() # TTL begins when the request actually finishes busy[slot] = False releasing.discard(slot) self.slot_cv.notify_all() @@ -1825,6 +1923,18 @@ def generate_batched(self, ids, max_new, sampling, cancel, embeddings=None): out: list[int] = [] pending: list[int] = [] prompt, left = list(ids), int(max_new) + tl = self.__dict__.setdefault("_tl", threading.local()) + request_key = getattr(tl, "api_request_key", None) or f"{self.gen}:{time.monotonic_ns()}" + tl.request_key = request_key + if not getattr(tl, "api_request_key", None): + tl.cache_debug = None + if getattr(self, "slot_cache_trace", False) and getattr(tl, "cache_debug", None) is None: + from serve.cache_trace import snapshot + with self.slot_cv: + tl.cache_debug = snapshot(ids, self.slot_held, self.slot_busy, + getattr(self, "slot_cache_valid", [True] * self.batch), self.slot_used, time.time()) + self.cache_event("request_start", prompt_tokens=len(ids), cache_debug=tl.cache_debug) + peer_ip = getattr(self.__dict__.setdefault("_tl", threading.local()), "peer_ip", None) reused0 = None # the first read's reused tokens: a continued leg's count includes this request's own output carry = {} # #1869: the figures of the legs that ended in a hand-back to the solo path (the final DONE adds to them) ok = yield from self._take_control(cancel, len(prompt), new=True) @@ -1837,6 +1947,14 @@ def generate_batched(self, ids, max_new, sampling, cancel, embeddings=None): stop_sent = False yields, solo_again, refused = 0, 0, 0 admit_wait, admit_since, admit_said = False, 0.0, False # #1603: waiting for a slot, since when, said + protected_since = None + park_slot = None # a slot claimed for the engine park= copy of the main session + solo_leg = None # (prompt, len(out)) of a solo leg in flight - what the main session will hold + + def choose_slot(): + nonlocal protected_since + chosen, protected_since = self._admission_slot(prompt, protected_since) + return chosen try: while True: # a request in a slot that is left alone goes back to the solo path if not holding: @@ -1847,7 +1965,17 @@ def generate_batched(self, ids, max_new, sampling, cancel, embeddings=None): with self.slot_cv: alone = not any(self.slot_busy) and self.waiting == 0 if alone and left > 1: - head = f"GENI {left}{keys} {embeddings}" if embeddings else f"GEN {left}{keys}" + if park_slot is not None: + self._release_park(park_slot) + park_slot = None + park = self._claim_park(prompt) + if park: + park_slot = int(park.split("=")[1]) + head = f"GENI {left}{keys}{park} {embeddings}" if embeddings else f"GEN {left}{keys}{park}" + solo_leg = (list(prompt), len(out)) + self.solo_cache_source_slot = None + self.solo_cache_reused_tokens = None + self.solo_request_key = request_key self._last_done = None # (#879: this request's DONE decides below) deadline = getattr(cancel, "_strata_queue_deadline", None) if deadline is not None and time.monotonic() >= deadline: @@ -1859,6 +1987,7 @@ def generate_batched(self, ids, max_new, sampling, cancel, embeddings=None): if started is not None: started() phase = "solo" + self.solo_peer_ip = peer_ip self._ctl_mode, self._ctl_result, self._yielded = "solo", None, None self._ctl_erred = False def others(): @@ -1870,7 +1999,7 @@ def give_way(): # at a PP line, and between two far apart (the engine takes it if (reserved is None and not out and not embeddings and yields < self.YIELDS_MAX and self._shorter_waiting(len(prompt))): with self.slot_cv: # the slot the part read will wait in - reserved = self.pick_slot(prompt) + reserved = self.pick_slot(prompt, allow_eviction=not self.slot_cache_affinity) if reserved is not None: self.slot_busy[reserved] = True self.slot_claims.add(reserved) @@ -1885,11 +2014,18 @@ def give_way(): # at a PP line, and between two far apart (the engine takes it if x is None: # a heartbeat (False: a token, flushed above) give_way() yield None + self.solo_peer_ip = None phase = "none" # its DONE is read while pending: t = pending.pop(0) out.append(t) yield t + if park_slot is not None: # no PARKED line: the engine kept nothing there + self._release_park(park_slot) + park_slot = None + # the main session now holds this leg's conversation (a read that gave way did not finish) + self.main_held = [] if self._yielded is not None else solo_leg[0] + out[solo_leg[1]:] + self.main_peer, solo_leg = peer_ip, None if self._done_finish() == "nonfinite": # #879 (never promoted to a slot) raise EngineNonFinite("the engine's logits became non-finite (#879)") if self._yielded is not None and reserved is not None and self._yielded[0] == reserved: @@ -1922,7 +2058,7 @@ def give_way(): # at a PP line, and between two far apart (the engine takes it if deadline is not None and time.monotonic() >= deadline: raise QueueWaitExpired("the inference queue wait deadline expired; retry later") with self.slot_cv: - slot = self.pick_slot(prompt) + slot = choose_slot() if slot is not None: self.slot_busy[slot] = True self.slot_claims.add(slot) @@ -1937,7 +2073,7 @@ def give_way(): # at a PP line, and between two far apart (the engine takes it self.ctl.release() holding = False with self.slot_cv: - if self.pick_slot(prompt) is None: + if choose_slot() is None: if (ENGINE_STALL_S or 0) > 0 and time.monotonic() - admit_since >= ENGINE_STALL_S: # the stall check of #1317 only watches a request on the control lines; one # waiting here is covered by looking for a slot nobody owns, and by saying so @@ -1948,9 +2084,10 @@ def give_way(): # at a PP line, and between two far apart (the engine takes it f"for a free slot (busy: {busy}, in use: {sorted(self.slot_claims)}, " f"ending: {sorted(self.slot_releasing)}) (issue #1603)", flush=True) self._reap_orphan_slots() - if self.pick_slot(prompt) is None: + if choose_slot() is None: wait_remaining = deadline - time.monotonic() if deadline is not None else 1.0 - self.slot_cv.wait(timeout=min(1.0, max(0, wait_remaining))) + self.slot_cv.wait(timeout=min(0.1 if self.slot_cache_affinity else 1.0, + max(0, wait_remaining))) if cancel.is_set(): return if not holding: @@ -1972,13 +2109,33 @@ def give_way(): # at a PP line, and between two far apart (the engine takes it holding, born = True, self.gen # (the engine it now has the control lines of) while not self.slot_q[slot].empty(): self.slot_q[slot].get_nowait() - head = f"BGENI {slot} {left}{keys} {embeddings}" if embeddings else f"BGEN {slot} {left}{keys}" - live = {"slot": slot, "request_id": getattr(cancel, "_strata_request_id", None), - "state": "reading", "prompt_tokens": len(prompt), "prompt_read": 0, "generated": 0, - "started": time.time(), "first_token": None} deadline = getattr(cancel, "_strata_queue_deadline", None) if deadline is not None and time.monotonic() >= deadline: raise QueueWaitExpired("the inference queue wait deadline expired; retry later") + if park_slot is not None: + self._release_park(park_slot) + park_slot = None + park = self._claim_park(prompt, exclude=(slot,)) + if park: + park_slot = int(park.split("=")[1]) + head = (f"BGENI {slot} {left}{keys}{park} {embeddings}" if embeddings + else f"BGEN {slot} {left}{keys}{park}") + self.main_held = [] # the admission reads on the main session + live = {"slot": slot, "request_id": getattr(cancel, "_strata_request_id", None), + "state": "reading", "prompt_tokens": len(prompt), "prompt_read": 0, "generated": 0, + "started": time.time(), "first_token": None, "peer_ip": peer_ip, "request_key": request_key} + if hasattr(self, "slot_cache_peer"): + self.slot_cache_peer[slot] = peer_ip + plan = getattr(self, "slot_last_plan", None) + if plan is not None: + live.update(cache_match_hint=plan.reused_hint, cache_source_hint=plan.source_hint, + placement_reason=plan.reason) + self.cache_event("allocation", slot=slot, peer_ip=peer_ip, + reason=getattr(plan, "reason", "upstream"), + source_hint=getattr(plan, "source_hint", None), + matched_tokens_hint=getattr(plan, "reused_hint", None)) + tl.executed_slot = slot + tl.cache_match_hint = getattr(plan, "reused_hint", None) self.slot_live[slot] = live self._last_done = None # (#879: this admission's DONE decides below) self._send(f"{head} {','.join(str(int(t)) for t in prompt)}") @@ -2034,7 +2191,18 @@ def note_progress(): self._park_slot(slot) slot = None continue - cont = bool(self._ctl_result and self._ctl_result[1]) + if self._last_done is not None: + admission = self.last or {} + live.update(cache_reused_tokens=admission.get("reused"), + cache_read_tokens=admission.get("prompt_read")) + self.cache_event("admission", slot=slot, peer_ip=peer_ip, + reused_tokens=admission.get("reused"), + read_tokens=admission.get("prompt_read")) + if park_slot is not None: # no PARKED line: the engine kept nothing there + self._release_park(park_slot) + park_slot = None + self.main_held = [] # the main session read this admission, which + cont = bool(self._ctl_result and self._ctl_result[1]) # lives on in its slot phase = "slot" if cont else "none" while pending: t = pending.pop(0) @@ -2133,7 +2301,15 @@ def note_progress(): self.last = {**self.last, "reused": reused0} if carry and isinstance(self.last, dict): self.last = add_carry(carry, self.last) + if park_slot is not None: + self._release_park(park_slot) + if solo_leg is not None: + # the consumer stopped mid-leg (usually right after the last token): the main session holds + # the leg all the same; a read that gave way (YIELDED) went into its slot instead + self.main_held = [] if self._yielded is not None else solo_leg[0] + out[solo_leg[1]:] + self.main_peer = peer_ip if holding: + self.solo_peer_ip = None self.ctl.release() if admit_wait: with self.slot_cv: @@ -2165,17 +2341,87 @@ def note_progress(): self.slot_cv.notify_all() slot_groups = 1 # --batch-groups (set when the engine starts) + slot_padding_safe = False # only native INFO capability enables padding-safe policy + slot_cache_affinity = False # STRATA_SLOT_CACHE_AFFINITY=1: preserve cached histories in pipelined groups SLOT_PREFIX_MIN = 512 # a held prefix shorter than this is not worth unbalancing the pipeline's groups TURN_ID = 248045 # <|im_start|>: the engine's default --turn-token (its turn checkpoint sits at the last one) - def pick_slot(self, prompt: list[int]) -> int | None: + + def _claim_park(self, prompt, exclude=()) -> str: + """With engine INFO slot_park=1: when `prompt` is another conversation than the one the main session holds, + claim an idle slot to keep that one in (engine key park=B). Returns the key text, or "".""" + if not (self.slot_cache_affinity and getattr(self, "slot_gs", 0) and self.slot_padding_safe and + (self.info or {}).get("slot_park") in (1, "1")): + return "" + main = getattr(self, "main_held", None) or [] + if len(main) < slot_policy.PARK_MIN_TOKENS or prompt[:len(main) // 2] == main[:len(main) // 2]: + return "" # short, or this prompt continues it (at least half of it): nothing to keep + with self.slot_cv: + b = slot_policy.park_destination(main, self.slot_busy, self.slot_held, self.slot_used, self.slot_order, + self.slot_gs, time.time(), getattr(self, "slot_cache_valid", None), + self.slot_padding_safe, exclude) + if b is not None: + self.slot_busy[b] = True + self.slot_claims.add(b) + self._park_claims.add(b) + self._park_main, self._park_peer = list(main), getattr(self, "main_peer", None) + # outside slot_cv: cache_event takes its own lock + self.cache_event("park_claim" if b is not None else "park_skip", slot=b, held_tokens=len(main)) + return f" park={b}" if b is not None else "" + + def _release_park(self, b) -> None: + with self.slot_cv: + if b in self._park_claims: + self._park_claims.discard(b) + self.slot_busy[b] = False + self.slot_claims.discard(b) + self.slot_cv.notify_all() + + def _on_parked(self, b: int, n: int) -> None: + """PARKED : the engine copied the main session's conversation (its first ) into + (0: the copy failed and the slot holds nothing usable).""" + main = getattr(self, "_park_main", None) or [] + with self.slot_cv: + if 0 <= b < self.batch: + self.slot_held[b] = list(main[:n]) if n > 0 else [] # the engine may hold a few more tail tokens + self.slot_used[b] = time.time() + if hasattr(self, "slot_cache_valid"): + self.slot_cache_valid[b] = n > 0 + if hasattr(self, "slot_cache_peer"): + self.slot_cache_peer[b] = getattr(self, "_park_peer", None) + self.cache_event("parked", slot=b, held_tokens=n) + self._release_park(b) + + def _admission_slot(self, prompt, protected_since): + """Called under slot_cv. One bounded protection window per request; ctl is released while waiting.""" + if not self.slot_cache_affinity: + return self.pick_slot(prompt), protected_since + now = time.monotonic() + allow = protected_since is not None and now - protected_since >= slot_policy.PROTECTION_WAIT_S + slot = self.pick_slot(prompt, allow_eviction=allow) + if slot is None and getattr(self, "slot_last_plan", None) and self.slot_last_plan.reason == "protected": + if protected_since is None: + protected_since = now + self.cache_event("protection_wait", wait_s=slot_policy.PROTECTION_WAIT_S) + if slot is not None and protected_since is not None: + self.cache_event("protection_end", slot=slot, waited_s=round(now - protected_since, 3)) + return slot, protected_since + + def pick_slot(self, prompt: list[int], allow_eviction: bool = True) -> int | None: """A free slot for `prompt` (the caller holds slot_cv): the one whose held tokens are the longest start of the prompt (the engine then reads only the rest), else an empty one, else the one used longest ago - so the conversations other slots hold stay for their next turns. None: every slot is busy.""" + gs = getattr(self, "slot_gs", 0) + if gs and self.slot_cache_affinity: + self.slot_last_plan = slot_policy.place(prompt, self.slot_busy, self.slot_held, self.slot_used, + self.slot_order, gs, time.time(), valid=getattr(self, "slot_cache_valid", None), + prefill_rate=getattr(self, "prefill_tok_s_mean", None), allow_eviction=allow_eviction, + padding_safe=self.slot_padding_safe) + return self.slot_last_plan.slot + free = [b for b in self.slot_order if not self.slot_busy[b]] if not free: return None - gs = getattr(self, "slot_gs", 0) if gs > 1: # pipelined groups of several slots: the engine keeps no idle slot below a busy one as a conversation cache # (its pad row writes the slot), so nothing is worth keeping free for a next turn. A group's window spans @@ -2216,13 +2462,39 @@ def held_prefix(b): def slots_view(self) -> list[dict]: """/metrics: each batch slot - idle (with the tokens it holds for a next turn), reading or decoding.""" now, view = time.time(), [] + superseded = set() + if self.slot_cache_affinity and getattr(self, "slot_gs", 0): + superseded = slot_policy.superseded_slots(self.slot_held, self.slot_busy, getattr( + self, "slot_cache_valid", [True] * self.batch)) for b in range(self.batch): r = self.slot_live[b] if r is None: - view.append({"slot": b, "state": "idle", "held_tokens": len(self.slot_held[b])}) + row = {"slot": b, "state": "idle", "held_tokens": len(self.slot_held[b])} + if self.slot_cache_affinity and getattr(self, "slot_gs", 0): + validity = getattr(self, "slot_cache_valid", [True] * self.batch)[b] + c = slot_policy.cache_state(len(self.slot_held[b]), self.slot_used[b], now, + validity and b not in superseded) + row["cache_superseded"] = b in superseded + row.update(cache_expired=c["expired"], cache_ttl_s=round(c["ttl_s"], 3), + cache_age_s=round(c["age_s"], 3), cache_valid_hint=c["valid"], + cache_retention_score=round(c["retention"], 6), + cache_overwrite_score=round(c["overwrite_score"], 6)) + if hasattr(self, "slot_cache_peer"): + row["cache_peer_ip"] = self.slot_cache_peer[b] + if row.get("cache_expired") and row["held_tokens"]: + seen = self.__dict__.setdefault("_cache_expiry_seen", [None] * self.batch) + signature = (self.slot_used[b], row["held_tokens"]) + with self.slot_cv: + if seen[b] != signature: + seen[b] = signature + self.cache_event("expired", slot=b, held_tokens=row["held_tokens"], + eligible_at=self.slot_used[b] + row["cache_ttl_s"]) + view.append(row) continue ft = r.get("first_token") - view.append({"slot": b, "state": r["state"], "prompt_tokens": r["prompt_tokens"], + view.append({**{k: r[k] for k in ("request_key", "peer_ip", "cache_reused_tokens", "cache_read_tokens", + "cache_match_hint", "cache_source_hint", "placement_reason") if k in r}, + "slot": b, "state": r["state"], "prompt_tokens": r["prompt_tokens"], "prompt_read": r.get("prompt_read") if r["state"] == "reading" else None, "generated": r["generated"], "elapsed_s": round(now - r["started"], 1), "tok_s": round(r["generated"] / max(1e-6, now - ft), 1) if ft else None}) @@ -4752,6 +5024,7 @@ def metrics(self, all_requests=False) -> dict: "tok_s_mean": round(self._tok_s_mean(), 1) if state == "generating" else None, "prefill_tok_s_mean": getattr(self.engine, "prefill_tok_s_mean", None) if s.get("busy") else None, "tok_s_window_s": RATE_WINDOW_S if state == "generating" else None} + live["peer_ip"] = s.get("peer_ip") if s.get("busy") else None if state == "reading" and progress: live["prompt_read"], live["prompt_total"] = progress par = int(getattr(self.engine, "batch", 0) or 0) @@ -4761,7 +5034,11 @@ def metrics(self, all_requests=False) -> dict: slots = self.engine.slots_view() if hasattr(self.engine, "slots_view") else [] # the requests in flight that are not in a slot: the one alone on the solo path, or waiting their turn in_slots = sum(1 for x in slots if x["state"] != "idle") - live.update(parallel=par, running=running, slots=slots, outside_slots=max(0, running - in_slots), + live.update(solo_peer_ip=getattr(self.engine, "solo_peer_ip", None), + solo_cache_source_slot=getattr(self.engine, "solo_cache_source_slot", None), + solo_cache_reused_tokens=getattr(self.engine, "solo_cache_reused_tokens", None), + solo_request_key=getattr(self.engine, "solo_request_key", None), + parallel=par, running=running, slots=slots, outside_slots=max(0, running - in_slots), waiting=int(getattr(self.engine, "waiting", 0) or 0), admitting=int(getattr(self.engine, "admitting", 0) or 0)) if running and state == "idle": @@ -4770,6 +5047,10 @@ def metrics(self, all_requests=False) -> dict: live["replicas"] = self.engine.replica_view() if hasattr(self.engine, "split_view"): # opt-in split serving: the hand-overs so far (serve/split.py) live["split_serve"] = self.engine.split_view() + if hasattr(self.engine, "cache_events_view"): + live["cache_events"] = self.engine.cache_events_view() + if getattr(self.engine, "slot_cache_affinity", False): # the slot policy tunables in force + live["slot_policy"] = slot_policy.settings() engine = {"model": self.model, "max_context": self.reported_ctx(), "images": self.vision is not None, **dict(getattr(self.engine, "info", {}) or {})} tel = self.telemetry.snapshot() if getattr(self, "telemetry", None) else {"now": {}, "history": {}, "static": {}} @@ -5322,6 +5603,16 @@ def cut(evs): segments = [] timed_out = False trace = getattr(self.request_trace, "record", None) + peer_ip = getattr(self.request_trace, "peer_ip", None) + api_request_key = f"req-{time.time_ns():x}" + if isinstance(self.engine, StrataEngine): + context = self.engine.__dict__.setdefault("_tl", threading.local()) + context.peer_ip = peer_ip + context.api_request_key = api_request_key + context.cache_debug = None + context.executed_slot = None + context.cache_source_slot = None + context.cache_match_hint = None waiting = time.perf_counter() first_engine_token_s = None # this request, from entering the engine queue; includes queue/prefill # #465: with "parallel" (the engine's batch slots) requests run at once: each keeps its own status and rate @@ -5406,7 +5697,7 @@ def replica_selected(index: int) -> None: queued_counted = False self.terminal_requests[request_id].update(state="waiting", admitted_at=time.time()) self._terminal_event_locked(request_id, "admitted") - st.update(busy=True, phase="reading the prompt", prompt_tokens=len(ids), + st.update(busy=True, peer_ip=peer_ip, phase="reading the prompt", prompt_tokens=len(ids), generated=0, started=time.time(), first_token=None, tool=None, tail="", max_tokens=max_new, reasoning_recoveries=0, thinking_budget=budget) if par: @@ -5669,7 +5960,12 @@ def replica_selected(index: int) -> None: "id": request_id, "projection": (sampling or {}).get("experimental_speed_projection") is not False if loaded else None, - "time": started, "duration_s": round(time.time() - started, 1), "finish": finish, + "request_key": api_request_key, + "slot": getattr(getattr(self.engine, "_tl", None), "executed_slot", None), + "cache_source_slot": getattr(getattr(self.engine, "_tl", None), "cache_source_slot", None), + "cache_match_hint": getattr(getattr(self.engine, "_tl", None), "cache_match_hint", None), + "cache_debug": getattr(getattr(self.engine, "_tl", None), "cache_debug", None), + "peer_ip": peer_ip, "time": started, "duration_s": round(time.time() - started, 1), "finish": finish, "prompt_tokens": seen, "reused": last.get("reused"), "output_tokens": n, "first_engine_token_s": first_engine_token_s, # the request's whole prompt, and the tokens read of it (None: an older engine) @@ -7275,6 +7571,7 @@ def _post(self): self._json(503, {"error": {"type": "server_error", "message": str(e)}}) return if path in ("/v1/chat/completions", "/v1/messages", "/v1/responses"): + svc.request_trace.peer_ip = self.client_address[0] self.record = svc.begin_request(path, req) if path == "/v1/responses": self._responses(req) @@ -7346,6 +7643,7 @@ def _post(self): finally: if reserved: svc.release_inference() + svc.request_trace.peer_ip = None svc.drop_embeddings() # the images' file of a request that never got to run() if self.watch_done is not None: self.watch_done.set() diff --git a/serve/slot_policy.py b/serve/slot_policy.py new file mode 100644 index 00000000000..86fc7dda284 --- /dev/null +++ b/serve/slot_policy.py @@ -0,0 +1,80 @@ +"""The placement policy interface for STRATA_SLOT_CACHE_AFFINITY=1: what serve/server.py (pick_slot, _admission_slot, +_claim_park, slots_view) and serve/cache_trace.py call. No policy is here - which idle slot a request runs in, which +history is given up, how long one is protected and where the solo path's conversation is parked are a policy's +choice. Without one, STRATA_SLOT_CACHE_AFFINITY=1 and STRATA_CACHE_TRACE=1 raise NotImplementedError; with both +off nothing here runs (configure() and settings() aside).""" +from __future__ import annotations + +from dataclasses import dataclass + +PROTECTION_WAIT_S = 5.0 # an admission that place() answers "protected" waits this long, once, then may evict +PARK_MIN_TOKENS = 8192 # a shorter main-session conversation is not parked + + +def configure(env) -> dict: + """Called by StrataEngine with the engine's environment: read the policy's tunables, return settings().""" + return settings() + + +def settings() -> dict: + """The tunables in force (/metrics live.slot_policy).""" + return {"protect_wait_s": PROTECTION_WAIT_S, "park_min_tokens": PARK_MIN_TOKENS} + + +def cache_state(tokens: int, last_used: float, now: float, valid: bool = True) -> dict: + """An idle history of `tokens`, last used at `last_used`: a dict with tokens, ttl_s, age_s, expired, valid, + retention, overwrite_score (/metrics live.slots and serve/cache_trace.py show it).""" + raise NotImplementedError("serve/slot_policy.py has no placement policy") + + +@dataclass(frozen=True) +class Placement: + slot: int | None + source_hint: int | None = None + reused_hint: int = 0 + near_tokens: int = 0 + reason: str = "busy" + + +def reliable_flags(busy: list[bool], valid: list[bool] | None, group_size: int, padding_safe: bool) -> list[bool]: + """Whether each slot's held tokens are still a usable cache. Without the native active-row engine, an idle row + below a busy row of its group is a pad row: its state is being overwritten.""" + gs = max(1, group_size) + validity = valid if valid is not None else [True] * len(busy) + return [bool(validity[b]) and (padding_safe or not any(busy[x] for x in range(b + 1, b - b % gs + gs))) + for b in range(len(busy))] + + +def superseded_slots(held: list[list[int]], busy: list[bool], reliable: list[bool]) -> set[int]: + """Idle histories that are the start of another idle slot's history: the conversation moved on in that other + slot, so this copy saves nothing it does not. They count as empty (no protection, no wait).""" + out = set() + for b, h in enumerate(held): + if h and not busy[b]: + for y, other in enumerate(held): + if (y != b and not busy[y] and reliable[y] and + (len(other) > len(h) or (len(other) == len(h) and y < b)) and other[:len(h)] == h): + out.add(b) + break + return out + + +def park_destination(main: list[int], busy: list[bool], held: list[list[int]], used: list[float], + order: list[int], group_size: int, now: float, valid: list[bool] | None = None, + padding_safe: bool = False, exclude=()) -> int | None: + """The idle slot to keep the main session's conversation `main` in before another request replaces it (the + engine's park=B), never one in `exclude`; None: do not park.""" + raise NotImplementedError("serve/slot_policy.py has no placement policy") + + +def place(prompt: list[int], busy: list[bool], held: list[list[int]], used: list[float], + order: list[int], group_size: int, now: float, valid: list[bool] | None = None, + prefill_rate: float | None = None, allow_eviction: bool = True, + padding_safe: bool = False) -> Placement: + """The idle slot `prompt` runs in (the caller holds slot_cv). busy/held/used/valid: each slot's state (held: the + tokens it holds, used: when it was last used, valid: the server's knowledge that its state is intact); order: the + slots in the order that spreads requests over the pipeline groups; group_size: slots per group; prefill_rate: the + observed prompt rate (tok/s); padding_safe: the engine runs a group's active slots only (INFO + slot_padding_safe=1). Placement(None, reason="protected") with allow_eviction=False: the request waits (at most + PROTECTION_WAIT_S, once) and asks again with allow_eviction=True. A busy slot is never returned.""" + raise NotImplementedError("serve/slot_policy.py has no placement policy") diff --git a/serve/test_cache_trace.py b/serve/test_cache_trace.py new file mode 100644 index 00000000000..d2cf27370ab --- /dev/null +++ b/serve/test_cache_trace.py @@ -0,0 +1,29 @@ +import unittest +from .cache_trace import snapshot + +class CacheTrace(unittest.TestCase): + + def test_resume_after_main_source_is_actual_reuse(self): + import queue, threading + from .server import StrataEngine + engine=object.__new__(StrataEngine) + engine.lines=queue.Queue() + engine._ctl_mode='solo' + engine._tl=threading.local() + engine._tl.request_key='request-a' + engine._parse_done=lambda line: None + for line in ['CACHE_SOURCE -1 900', 'RESUME 700', 'DONE']: + engine.lines.put(line) + list(engine._control(threading.Event(), lambda token: None)) + self.assertEqual(engine.solo_cache_reused_tokens,700) + self.assertIsNone(engine.solo_cache_source_slot) + self.assertEqual(engine.cache_events_view()[0]['request_key'],'request-a') + self.assertEqual(engine.cache_events_view()[0]['reused_tokens'],700) + def test_global_expiry_does_not_inherit_request(self): + import threading + from .server import StrataEngine + engine=object.__new__(StrataEngine) + engine._tl=threading.local() + engine._tl.request_key='request-a' + engine.cache_event('expired',slot=0) + self.assertNotIn('request_key',engine.cache_events_view()[0]) diff --git a/serve/test_parallel.py b/serve/test_parallel.py index 627cdc98b95..cd63415b0d7 100644 --- a/serve/test_parallel.py +++ b/serve/test_parallel.py @@ -5,6 +5,7 @@ wait for a slot, /metrics shows the slots, each request's history row is its own, and a conversation's next turn goes back to the slot that holds it.""" import json +import os import sys import tempfile import threading @@ -110,6 +111,8 @@ def reused(ids): # token (the request's own output included), a fre stopped.add(b) continue if line.startswith(("GEN ", "BGEN ")): + if "--source-trace" in args: + print("CACHE_SOURCE 1 3", flush=True) f = line.split() slot = int(f[1]) if f[0] == "BGEN" else None max_new = int(f[2] if f[0] == "BGEN" else f[1]) @@ -321,7 +324,7 @@ class FakeEngineCase(unittest.TestCase): """The real StrataEngine and Service over HTTP, the fake engine behind them.""" def start(self, slots, fit=None, slot_cache=False, fail=False, reuse=False, says_groups=0, - more=(), stage_plan=False, batch_sample=False, stage_sample=False): + more=(), stage_plan=False, batch_sample=False, stage_sample=False, affinity=False): import serve.server as server self.tmp = tempfile.TemporaryDirectory() script = Path(self.tmp.name) / "fake_strata.py" @@ -339,7 +342,8 @@ def start(self, slots, fit=None, slot_cache=False, fail=False, reuse=False, says extra += list(more) with mock.patch.object(server.subprocess, "Popen", lambda cmd, **kw: real([sys.executable, str(script), *cmd[1:]], **kw)): - self.engine = StrataEngine("strata", extra, log=str(Path(self.tmp.name) / "engine.log") if stage_plan else None) + self.engine = StrataEngine("strata", extra, log=str(Path(self.tmp.name) / "engine.log") if stage_plan else None, + env={**os.environ, "STRATA_SLOT_CACHE_AFFINITY": "1"} if affinity else None) tok = ByteTokenizer() self.svc = Service(self.engine, tok, ChatTemplate(Path(__file__).parent / "chat_template.jinja")) self.httpd = serve(self.svc, port=0) @@ -372,6 +376,32 @@ def get(self, path): with urllib.request.urlopen(self.base + path, timeout=10) as r: return json.loads(r.read().decode()) + def _protected_waiter(self): + self.start(6, says_groups=3, affinity=True) + e = self.engine + with e.slot_cv: + e.slot_held = [[9000 + b] * 50000 for b in range(6)] + e.slot_used = [time.time()] * 6 + e.slot_cache_valid = [True] * 6 + # Keep one request marked busy to exercise admission instead of the solo path. + e.slot_busy[0] = True + e.slot_claims.add(0) + cancel, result = threading.Event(), {} + def run(): + try: + result['tokens'] = list(e.generate_batched([9999], 8, {}, cancel)) + except BaseException as ex: + result['error'] = ex + t = threading.Thread(target=run) + t.start() + end = time.monotonic() + 2 + while time.monotonic() < end and not e.admitting: + time.sleep(0.01) + self.assertGreater(e.admitting, 0) + self.assertTrue(e.ctl.acquire(blocking=False)) # waiting never monopolizes the control channel + e.ctl.release() + return cancel, result, t + def _two_at_once(self, text_b): """A long request first, then one that has to go to a batch slot (the engine is busy with the first).""" out = {} @@ -408,6 +438,26 @@ class ParallelService(FakeEngineCase): """The real StrataEngine and Service over HTTP, the fake engine behind them: slots, queues, benchmarks, telemetry. The yielding, restart and failure tests are in serve/test_parallel_yield.py (this file alone ran past 100 s).""" + def test_solo_native_source_is_forwarded_without_reserving_batch_slot(self): + self.start(2, more=["--source-trace"]) + out = {} + worker = threading.Thread(target=lambda: out.update(reply=self.chat("LONGREPLY please"))) + worker.start() + deadline = time.time() + 5 + observed = None + while time.time() < deadline: + live = self.get("/metrics")["live"] + if live.get("solo_cache_source_slot") == 1: + observed = live + break + time.sleep(.01) + worker.join(10) + self.assertIsNotNone(observed) + self.assertEqual(observed["solo_cache_reused_tokens"], 3) + self.assertTrue(observed["solo_request_key"]) + self.assertTrue(all(s["state"] == "idle" for s in observed["slots"])) + self.assertIn("reply", out) + def test_bgen_refused_by_the_engine_is_retried_not_a_400(self): """#1994: the engine refuses a BGEN ("no such free slot") although the server saw the slot free: the request goes to another slot (or waits) and is answered; the same conversation's next turn works too.""" diff --git a/serve/test_slot_alloc.py b/serve/test_slot_alloc.py index b2b4d35bef7..b64d810f063 100644 --- a/serve/test_slot_alloc.py +++ b/serve/test_slot_alloc.py @@ -7,10 +7,14 @@ import random import sys import unittest +import threading +import time +from unittest import mock from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parents[1])) from serve.server import StrataEngine # noqa: E402 +from serve import slot_policy # noqa: E402 def engine(batch, groups, pipelined=True): @@ -60,12 +64,27 @@ def test_never_opens_a_hole_below_the_pick(self): e.slot_busy[b] = True def test_held_tokens_are_ignored(self): - # the engine keeps no pipelined slot as a cache: a held prefix must not pull a request into a hole-making slot + # Affinity is opt-in: the default still prioritizes pipeline placement over held prefixes. e = engine(6, 3) e.slot_held[5] = [1, 2] self.assertEqual(e.pick_slot([1, 2, 3]), 0) +class CacheAffinity(unittest.TestCase): + def engine(self): + e = engine(6, 3) + e.slot_cache_affinity = True + e.slot_used = [time.time()] * 6 + return e + + def test_environment_survives_lazy_restart_setup(self): + e = StrataEngine("missing", [], env={"STRATA_SLOT_CACHE_AFFINITY": "1"}, lazy=True) + self.assertTrue(e.slot_cache_affinity) + restarted = StrataEngine(*e.spawn[:5], lazy=True, cpus=e.spawn[6]) + self.assertTrue(restarted.slot_cache_affinity) + self.assertFalse(StrataEngine("missing", [], env={}, lazy=True).slot_cache_affinity) + + class OneGroup(unittest.TestCase): def test_reuse_then_least_recently_used(self): e = engine(4, 1, pipelined=False) @@ -104,5 +123,130 @@ def test_groups_of_several_slots_still_ignore_held_tokens(self): self.assertEqual(e.pick_slot(hist + [T, 5, 6] + list(range(3000, 3050))), 0) +class RetentionPolicy(unittest.TestCase): + NOW = 1000.0 + + def e(self): + e = engine(6, 3) + e.slot_cache_affinity = True + e.slot_used = [self.NOW] * 6 + return e + + def place(self, e, prompt, **kw): + return slot_policy.place(prompt, e.slot_busy, e.slot_held, e.slot_used, + e.slot_order, e.slot_gs, self.NOW, **kw) + + def test_padding_invalidity_survives_completion_of_higher_row(self): + e = self.e() + e.slot_cv = threading.Condition() + e.slot_cache_valid = [False] * 6 + e._slot_native_active = [False] * 6 + e._slot_steps = [0] * 6 + e._slot_valid_until = [{} for _ in range(6)] + e._observe_cache_line('BADM 2 1') + e._observe_cache_line('BADM 3 1') + e._observe_cache_line('BT 2 99') + e._observe_cache_line('BDONE 2 2 stop 1') + e._observe_cache_line('BT 3 88') # same window: row 2 was not padding + self.assertTrue(e.slot_cache_valid[2]) + e._observe_cache_line('BT 3 87') # next window: row 2 was padding + self.assertFalse(e.slot_cache_valid[2]) + e._observe_cache_line('BDONE 3 3 stop 1') + self.assertFalse(e.slot_cache_valid[2]) + self.assertTrue(e.slot_cache_valid[3]) + # A later real request can restore the lower row's validity. + e._observe_cache_line('BADM 2 1') + e._observe_cache_line('BDONE 2 2 stop 1') + self.assertTrue(e.slot_cache_valid[2]) + + def test_new_higher_row_invalidates_lower_cache_on_its_first_window(self): + e = self.e() + e.slot_cv = threading.Condition() + e.slot_cache_valid = [False] * 6 + e._slot_native_active = [False] * 6 + e._slot_steps = [0] * 6 + e._slot_valid_until = [{} for _ in range(6)] + e._observe_cache_line('BADM 2 1') + e._observe_cache_line('BDONE 2 2 stop 1') + self.assertTrue(e.slot_cache_valid[2]) + e._observe_cache_line('BADM 3 1') # row 3 was absent from row 2's final window + e._observe_cache_line('BT 3 88') + e._observe_cache_line('BDONE 3 2 stop 1') + self.assertFalse(e.slot_cache_valid[2]) + + +class ParkMainSession(unittest.TestCase): + """Engine park=: before a side request replaces the main (solo) session, its conversation is copied into a slot.""" + NOW = 100000.0 + + def e(self): + e = engine(6, 3) + e.slot_cache_affinity = True + e.slot_padding_safe = True + e.info = {"slot_park": 1} + e.slot_used = [self.NOW] * 6 + e.slot_cv = threading.Condition() + e.slot_cache_valid = [True] * 6 + e.slot_cache_peer = [None] * 6 + e._park_claims, e.slot_claims = set(), set() + e.main_held = [7] * 50000 + e.main_peer = "192.0.2.9" + return e + + def dest(self, e, exclude=()): + return slot_policy.park_destination(e.main_held, e.slot_busy, e.slot_held, e.slot_used, e.slot_order, + e.slot_gs, self.NOW, e.slot_cache_valid, True, exclude) + + def test_off_without_engine_capability_or_padding_safety(self): + e = self.e() + e.info = {} + self.assertEqual(e._claim_park([1] * 60), "") + e.info = {"slot_park": 1} + e.slot_padding_safe = False + self.assertEqual(e._claim_park([1] * 60), "") + + +class NativePaddingSafety(unittest.TestCase): + + def test_safe_native_steps_do_not_invalidate_finished_lower_slot(self): + e = engine(6, 3) + e.slot_padding_safe = True + e.slot_cv = threading.Condition() + e.slot_cache_valid = [False] * 6 + e._slot_native_active = [False] * 6 + e._slot_steps = [0] * 6 + e._slot_valid_until = [{} for _ in range(6)] + e._observe_cache_line('BADM 2 1') + e._observe_cache_line('BADM 3 1') + e._observe_cache_line('BDONE 2 2 stop 1') + for _ in range(3): + e._observe_cache_line('BT 3 88') + e._observe_cache_line('BDONE 3 4 stop 1') + self.assertTrue(e.slot_cache_valid[2]) + self.assertTrue(e.slot_cache_valid[3]) + + + +class CacheObservability(unittest.TestCase): + + def test_busy_view_keeps_actual_reuse_separate_from_hint_and_peer(self): + e = engine(2, 1) + e.slot_cache_affinity = True + e.slot_live = [{"state": "decoding", "prompt_tokens": 1000, "generated": 3, + "started": time.time(), "first_token": None, "peer_ip": "2001:db8::7", + "cache_reused_tokens": 0, "cache_match_hint": 900}, None] + row = e.slots_view()[0] + self.assertEqual((row["cache_reused_tokens"], row["cache_match_hint"]), (0, 900)) + self.assertEqual(row["peer_ip"], "2001:db8::7") + + def test_event_ring_is_bounded_and_returns_snapshot(self): + e = engine(2, 1) + for n in range(120): + e.cache_event("allocation", slot=n % 2) + events = e.cache_events_view() + self.assertEqual(len(events), 96) + events.clear() + self.assertEqual(len(e.cache_events_view()), 96) + if __name__ == "__main__": unittest.main() From 82ba0f4bbf53a5ad8affa66a009118ccd09cc703 Mon Sep 17 00:00:00 2001 From: andyhrc <12491949+andyhrc@users.noreply.github.com> Date: Thu, 15 Oct 2026 11:53:27 +0800 Subject: [PATCH 04/21] serve: the allocation event and a slot's live row show this request's own placement plan slot_last_plan is shared by every request; the admission read it after giving slot_cv back, so a request picking a slot meanwhile could put its own plan there. Seen as a match hint larger than the prompt (126,251 for a 125,512-token prompt). The plan is now kept per request, under slot_cv, when the slot is picked (an admission, or a slot reserved for a read that gives way). Display only: placement unchanged. Co-Authored-By: Claude Opus 5.5 --- serve/server.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/serve/server.py b/serve/server.py index b925c55260c..4860429726a 100644 --- a/serve/server.py +++ b/serve/server.py @@ -1948,12 +1948,14 @@ def generate_batched(self, ids, max_new, sampling, cancel, embeddings=None): yields, solo_again, refused = 0, 0, 0 admit_wait, admit_since, admit_said = False, 0.0, False # #1603: waiting for a slot, since when, said protected_since = None + slot_plan = None # this request's own placement plan (slot_last_plan is shared: another request's pick can follow) park_slot = None # a slot claimed for the engine park= copy of the main session solo_leg = None # (prompt, len(out)) of a solo leg in flight - what the main session will hold def choose_slot(): - nonlocal protected_since + nonlocal protected_since, slot_plan chosen, protected_since = self._admission_slot(prompt, protected_since) + slot_plan = getattr(self, "slot_last_plan", None) # (the caller holds slot_cv) return chosen try: while True: # a request in a slot that is left alone goes back to the solo path @@ -1995,11 +1997,12 @@ def others(): return self.waiting > 0 def give_way(): # at a PP line, and between two far apart (the engine takes it at its next boundary) - nonlocal reserved + nonlocal reserved, slot_plan if (reserved is None and not out and not embeddings and yields < self.YIELDS_MAX and self._shorter_waiting(len(prompt))): with self.slot_cv: # the slot the part read will wait in reserved = self.pick_slot(prompt, allow_eviction=not self.slot_cache_affinity) + slot_plan = getattr(self, "slot_last_plan", None) if reserved is not None: self.slot_busy[reserved] = True self.slot_claims.add(reserved) @@ -2126,7 +2129,7 @@ def give_way(): # at a PP line, and between two far apart (the engine takes it "started": time.time(), "first_token": None, "peer_ip": peer_ip, "request_key": request_key} if hasattr(self, "slot_cache_peer"): self.slot_cache_peer[slot] = peer_ip - plan = getattr(self, "slot_last_plan", None) + plan = slot_plan if self.slot_cache_affinity else None if plan is not None: live.update(cache_match_hint=plan.reused_hint, cache_source_hint=plan.source_hint, placement_reason=plan.reason) From 023edc2e43426ce0926bed58361efb3523ea36f0 Mon Sep 17 00:00:00 2001 From: andyhrc <12491949+andyhrc@users.noreply.github.com> Date: Thu, 15 Oct 2026 23:33:19 +0800 Subject: [PATCH 05/21] serve: /metrics live.solo_* clear when the solo request ends live.solo_request_key, solo_cache_source_slot and solo_cache_reused_tokens kept the last solo request's values until the next solo request (only solo_peer_ip was cleared), so a monitor reading them while other requests ran or waited for a slot showed a request that had long ended. They are now cleared with solo_peer_ip, in both places the solo leg ends. Display only: placement unchanged. Co-Authored-By: Claude Opus 5.5 --- serve/server.py | 9 +++++++-- serve/test_parallel.py | 4 ++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/serve/server.py b/serve/server.py index 4860429726a..d1331e855fa 100644 --- a/serve/server.py +++ b/serve/server.py @@ -1810,6 +1810,11 @@ def _reap_orphan_slots(self) -> list[int]: self.slot_cv.notify_all() return freed + def _solo_ended(self): + """/metrics: no request on the solo path now (its key and cache source would otherwise stay until the next one).""" + self.solo_peer_ip = self.solo_request_key = None + self.solo_cache_source_slot = self.solo_cache_reused_tokens = None + YIELDS_MAX = 2 # #656: how often one request's prompt read gives way to a shorter waiting one def _take_control(self, cancel, plen: int, after_epoch: int | None = None, new: bool = False): @@ -2017,7 +2022,7 @@ def give_way(): # at a PP line, and between two far apart (the engine takes it if x is None: # a heartbeat (False: a token, flushed above) give_way() yield None - self.solo_peer_ip = None + self._solo_ended() phase = "none" # its DONE is read while pending: t = pending.pop(0) @@ -2312,7 +2317,7 @@ def note_progress(): self.main_held = [] if self._yielded is not None else solo_leg[0] + out[solo_leg[1]:] self.main_peer = peer_ip if holding: - self.solo_peer_ip = None + self._solo_ended() self.ctl.release() if admit_wait: with self.slot_cv: diff --git a/serve/test_parallel.py b/serve/test_parallel.py index cd63415b0d7..2369084e6a4 100644 --- a/serve/test_parallel.py +++ b/serve/test_parallel.py @@ -457,6 +457,10 @@ def test_solo_native_source_is_forwarded_without_reserving_batch_slot(self): self.assertTrue(observed["solo_request_key"]) self.assertTrue(all(s["state"] == "idle" for s in observed["slots"])) self.assertIn("reply", out) + # once it is answered, /metrics shows no solo request: no key or cache source left from it + live = self.get("/metrics")["live"] + self.assertEqual([live[k] for k in ("solo_peer_ip", "solo_request_key", "solo_cache_source_slot", + "solo_cache_reused_tokens")], [None] * 4) def test_bgen_refused_by_the_engine_is_retried_not_a_400(self): """#1994: the engine refuses a BGEN ("no such free slot") although the server saw the slot free: the request From d926b132975fa51495dc4a3e9f9035cb8395cb1a Mon Sep 17 00:00:00 2001 From: andyhrc <12491949+andyhrc@users.noreply.github.com> Date: Fri, 16 Oct 2026 08:40:02 +0800 Subject: [PATCH 06/21] serve: no protection wait - place() always answers a free slot The admission waited once per request, up to slot_policy.PROTECTION_WAIT_S, when the policy answered "protected" (every idle history still protected), and then asked again allowing an eviction. In production that wait never paid: with more requests than slots, a slot that had just finished held a protected history, and each of 7 waits in 15 minutes (2026-10-15, 3 x RTX A4000, 6 slots, 8 clients) ran the full 5 s and ended in the same eviction, the slot idle meanwhile. The history that the eviction takes is the one with the lowest retention - the least likely to be continued within the wait anyway. Removed: _admission_slot, the per-request protected_since, pick_slot()/place()'s allow_eviction, the protection_wait/protection_end events, PROTECTION_WAIT_S and live.slot_policy.protect_wait_s. A request still waits for a slot only when every slot is busy, re-checked on each release and every 1.0 s as before this change. A read that gives way (#656) reserves its slot with the same placement as an admission. The request keeps its own placement plan (choose_slot()). Co-Authored-By: Claude Opus 5.5 --- docs/BATCHING.md | 7 ++----- serve/server.py | 30 ++++++------------------------ serve/slot_policy.py | 15 ++++++--------- serve/test_parallel.py | 26 -------------------------- 4 files changed, 14 insertions(+), 64 deletions(-) diff --git a/docs/BATCHING.md b/docs/BATCHING.md index e171927faf1..9a18d45b554 100644 --- a/docs/BATCHING.md +++ b/docs/BATCHING.md @@ -444,20 +444,17 @@ policy: end, `BDONE`), whether its state is intact, the spreading order, the group size, the observed prompt rate and whether the engine runs active slots only (`slot_padding_safe=1`). Without that engine capability the server tracks from the `BADM` / `BT` / `BDONE` lines which idle slot a later window of its group has padded over, and marks it. -- An answer "protected" makes the admission wait - the control lines given back, the slots re-checked every 0.1 s - - at most `PROTECTION_WAIT_S` once per request, and then ask again allowing an eviction. A cancelled request leaves - every slot as it was. - With `INFO slot_park=1` the server tracks what the main session holds; when a request is another conversation (the first half differs) and the held one has at least `PARK_MIN_TOKENS`, it asks the policy for a slot and sends `park=B`; `PARKED` updates that slot's state. A claim the engine did not use is released. The policy's interface is `place`, `park_destination`, `cache_state`, `superseded_slots`, `reliable_flags`, -`configure` / `settings` and the constants `PROTECTION_WAIT_S`, `PARK_MIN_TOKENS` (their docstrings say what they +`configure` / `settings` and the constant `PARK_MIN_TOKENS` (their docstrings say what they answer). This part ships them as an interface without a policy: with `STRATA_SLOT_CACHE_AFFINITY=1` (or `STRATA_CACHE_TRACE=1`) and no policy they raise `NotImplementedError`; with both off nothing changes. `/metrics` shows what happened: `live.cache_events` (the last 96: allocation, admission with the tokens reused and -read, resume and cache source, park claims and `parked`, protection waits, expiries), each request's key, client +read, resume and cache source, park claims and `parked`, expiries), each request's key, client address, slot and cache source in `requests[]`, `live.peer_ip` and `live.solo_*` (the request on the solo path and where its cache came from), and for slots the client of the conversation each one holds; with `STRATA_SLOT_CACHE_AFFINITY=1` also each idle slot's protection and scores and `live.slot_policy`; with diff --git a/serve/server.py b/serve/server.py index d1331e855fa..e6cfa552e97 100644 --- a/serve/server.py +++ b/serve/server.py @@ -1952,14 +1952,13 @@ def generate_batched(self, ids, max_new, sampling, cancel, embeddings=None): stop_sent = False yields, solo_again, refused = 0, 0, 0 admit_wait, admit_since, admit_said = False, 0.0, False # #1603: waiting for a slot, since when, said - protected_since = None slot_plan = None # this request's own placement plan (slot_last_plan is shared: another request's pick can follow) park_slot = None # a slot claimed for the engine park= copy of the main session solo_leg = None # (prompt, len(out)) of a solo leg in flight - what the main session will hold def choose_slot(): - nonlocal protected_since, slot_plan - chosen, protected_since = self._admission_slot(prompt, protected_since) + nonlocal slot_plan + chosen = self.pick_slot(prompt) slot_plan = getattr(self, "slot_last_plan", None) # (the caller holds slot_cv) return chosen try: @@ -2006,7 +2005,7 @@ def give_way(): # at a PP line, and between two far apart (the engine takes it if (reserved is None and not out and not embeddings and yields < self.YIELDS_MAX and self._shorter_waiting(len(prompt))): with self.slot_cv: # the slot the part read will wait in - reserved = self.pick_slot(prompt, allow_eviction=not self.slot_cache_affinity) + reserved = self.pick_slot(prompt) slot_plan = getattr(self, "slot_last_plan", None) if reserved is not None: self.slot_busy[reserved] = True @@ -2094,8 +2093,7 @@ def give_way(): # at a PP line, and between two far apart (the engine takes it self._reap_orphan_slots() if choose_slot() is None: wait_remaining = deadline - time.monotonic() if deadline is not None else 1.0 - self.slot_cv.wait(timeout=min(0.1 if self.slot_cache_affinity else 1.0, - max(0, wait_remaining))) + self.slot_cv.wait(timeout=min(1.0, max(0, wait_remaining))) if cancel.is_set(): return if not holding: @@ -2400,22 +2398,7 @@ def _on_parked(self, b: int, n: int) -> None: self.cache_event("parked", slot=b, held_tokens=n) self._release_park(b) - def _admission_slot(self, prompt, protected_since): - """Called under slot_cv. One bounded protection window per request; ctl is released while waiting.""" - if not self.slot_cache_affinity: - return self.pick_slot(prompt), protected_since - now = time.monotonic() - allow = protected_since is not None and now - protected_since >= slot_policy.PROTECTION_WAIT_S - slot = self.pick_slot(prompt, allow_eviction=allow) - if slot is None and getattr(self, "slot_last_plan", None) and self.slot_last_plan.reason == "protected": - if protected_since is None: - protected_since = now - self.cache_event("protection_wait", wait_s=slot_policy.PROTECTION_WAIT_S) - if slot is not None and protected_since is not None: - self.cache_event("protection_end", slot=slot, waited_s=round(now - protected_since, 3)) - return slot, protected_since - - def pick_slot(self, prompt: list[int], allow_eviction: bool = True) -> int | None: + def pick_slot(self, prompt: list[int]) -> int | None: """A free slot for `prompt` (the caller holds slot_cv): the one whose held tokens are the longest start of the prompt (the engine then reads only the rest), else an empty one, else the one used longest ago - so the conversations other slots hold stay for their next turns. None: every slot is busy.""" @@ -2423,8 +2406,7 @@ def pick_slot(self, prompt: list[int], allow_eviction: bool = True) -> int | Non if gs and self.slot_cache_affinity: self.slot_last_plan = slot_policy.place(prompt, self.slot_busy, self.slot_held, self.slot_used, self.slot_order, gs, time.time(), valid=getattr(self, "slot_cache_valid", None), - prefill_rate=getattr(self, "prefill_tok_s_mean", None), allow_eviction=allow_eviction, - padding_safe=self.slot_padding_safe) + prefill_rate=getattr(self, "prefill_tok_s_mean", None), padding_safe=self.slot_padding_safe) return self.slot_last_plan.slot free = [b for b in self.slot_order if not self.slot_busy[b]] diff --git a/serve/slot_policy.py b/serve/slot_policy.py index 86fc7dda284..a2688c09924 100644 --- a/serve/slot_policy.py +++ b/serve/slot_policy.py @@ -1,5 +1,5 @@ -"""The placement policy interface for STRATA_SLOT_CACHE_AFFINITY=1: what serve/server.py (pick_slot, _admission_slot, -_claim_park, slots_view) and serve/cache_trace.py call. No policy is here - which idle slot a request runs in, which +"""The placement policy interface for STRATA_SLOT_CACHE_AFFINITY=1: what serve/server.py (pick_slot, _claim_park, +slots_view) and serve/cache_trace.py call. No policy is here - which idle slot a request runs in, which history is given up, how long one is protected and where the solo path's conversation is parked are a policy's choice. Without one, STRATA_SLOT_CACHE_AFFINITY=1 and STRATA_CACHE_TRACE=1 raise NotImplementedError; with both off nothing here runs (configure() and settings() aside).""" @@ -7,7 +7,6 @@ from dataclasses import dataclass -PROTECTION_WAIT_S = 5.0 # an admission that place() answers "protected" waits this long, once, then may evict PARK_MIN_TOKENS = 8192 # a shorter main-session conversation is not parked @@ -18,7 +17,7 @@ def configure(env) -> dict: def settings() -> dict: """The tunables in force (/metrics live.slot_policy).""" - return {"protect_wait_s": PROTECTION_WAIT_S, "park_min_tokens": PARK_MIN_TOKENS} + return {"park_min_tokens": PARK_MIN_TOKENS} def cache_state(tokens: int, last_used: float, now: float, valid: bool = True) -> dict: @@ -47,7 +46,7 @@ def reliable_flags(busy: list[bool], valid: list[bool] | None, group_size: int, def superseded_slots(held: list[list[int]], busy: list[bool], reliable: list[bool]) -> set[int]: """Idle histories that are the start of another idle slot's history: the conversation moved on in that other - slot, so this copy saves nothing it does not. They count as empty (no protection, no wait).""" + slot, so this copy saves nothing it does not. They count as empty (no protection).""" out = set() for b, h in enumerate(held): if h and not busy[b]: @@ -69,12 +68,10 @@ def park_destination(main: list[int], busy: list[bool], held: list[list[int]], u def place(prompt: list[int], busy: list[bool], held: list[list[int]], used: list[float], order: list[int], group_size: int, now: float, valid: list[bool] | None = None, - prefill_rate: float | None = None, allow_eviction: bool = True, - padding_safe: bool = False) -> Placement: + prefill_rate: float | None = None, padding_safe: bool = False) -> Placement: """The idle slot `prompt` runs in (the caller holds slot_cv). busy/held/used/valid: each slot's state (held: the tokens it holds, used: when it was last used, valid: the server's knowledge that its state is intact); order: the slots in the order that spreads requests over the pipeline groups; group_size: slots per group; prefill_rate: the observed prompt rate (tok/s); padding_safe: the engine runs a group's active slots only (INFO - slot_padding_safe=1). Placement(None, reason="protected") with allow_eviction=False: the request waits (at most - PROTECTION_WAIT_S, once) and asks again with allow_eviction=True. A busy slot is never returned.""" + slot_padding_safe=1). A busy slot is never returned; Placement(None) only when every slot is busy.""" raise NotImplementedError("serve/slot_policy.py has no placement policy") diff --git a/serve/test_parallel.py b/serve/test_parallel.py index 2369084e6a4..cbae1dd42ee 100644 --- a/serve/test_parallel.py +++ b/serve/test_parallel.py @@ -376,32 +376,6 @@ def get(self, path): with urllib.request.urlopen(self.base + path, timeout=10) as r: return json.loads(r.read().decode()) - def _protected_waiter(self): - self.start(6, says_groups=3, affinity=True) - e = self.engine - with e.slot_cv: - e.slot_held = [[9000 + b] * 50000 for b in range(6)] - e.slot_used = [time.time()] * 6 - e.slot_cache_valid = [True] * 6 - # Keep one request marked busy to exercise admission instead of the solo path. - e.slot_busy[0] = True - e.slot_claims.add(0) - cancel, result = threading.Event(), {} - def run(): - try: - result['tokens'] = list(e.generate_batched([9999], 8, {}, cancel)) - except BaseException as ex: - result['error'] = ex - t = threading.Thread(target=run) - t.start() - end = time.monotonic() + 2 - while time.monotonic() < end and not e.admitting: - time.sleep(0.01) - self.assertGreater(e.admitting, 0) - self.assertTrue(e.ctl.acquire(blocking=False)) # waiting never monopolizes the control channel - e.ctl.release() - return cancel, result, t - def _two_at_once(self, text_b): """A long request first, then one that has to go to a batch slot (the engine is busy with the first).""" out = {} From 2d9b68508bae85bfc43fa08fd8cd803e2bea10d9 Mon Sep 17 00:00:00 2001 From: andyhrc <12491949+andyhrc@users.noreply.github.com> Date: Fri, 16 Oct 2026 08:57:15 +0800 Subject: [PATCH 07/21] serve: no PARK_MIN_TOKENS - a park is not cut off by length The server asked the policy for a park destination only when the main session held at least slot_policy.PARK_MIN_TOKENS (8192) tokens. That was a fixed length cut-off beside the policy's own judgement of what a conversation is worth: with the policy's value threshold (STRATA_SLOT_PARK_MIN_VALUE, the next part) it either decided nothing (0.1 already skips conversations under about 18K tokens) or overrode the score. The server now asks whenever the request is another conversation (the first half of what the main session holds differs); whether it is worth a copy is the policy's answer. PARK_MIN_TOKENS and live.slot_policy.park_min_tokens are gone from the interface. Co-Authored-By: Claude Opus 5.5 --- docs/BATCHING.md | 6 +++--- serve/server.py | 4 ++-- serve/slot_policy.py | 3 +-- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/docs/BATCHING.md b/docs/BATCHING.md index 9a18d45b554..aa96bed9e63 100644 --- a/docs/BATCHING.md +++ b/docs/BATCHING.md @@ -445,11 +445,11 @@ policy: whether the engine runs active slots only (`slot_padding_safe=1`). Without that engine capability the server tracks from the `BADM` / `BT` / `BDONE` lines which idle slot a later window of its group has padded over, and marks it. - With `INFO slot_park=1` the server tracks what the main session holds; when a request is another conversation (the - first half differs) and the held one has at least `PARK_MIN_TOKENS`, it asks the policy for a slot and sends - `park=B`; `PARKED` updates that slot's state. A claim the engine did not use is released. + first half differs), it asks the policy for a slot and sends `park=B`; `PARKED` updates that slot's state. A claim + the engine did not use is released. The policy's interface is `place`, `park_destination`, `cache_state`, `superseded_slots`, `reliable_flags`, -`configure` / `settings` and the constant `PARK_MIN_TOKENS` (their docstrings say what they +`configure` / `settings` (their docstrings say what they answer). This part ships them as an interface without a policy: with `STRATA_SLOT_CACHE_AFFINITY=1` (or `STRATA_CACHE_TRACE=1`) and no policy they raise `NotImplementedError`; with both off nothing changes. diff --git a/serve/server.py b/serve/server.py index e6cfa552e97..8681e39d254 100644 --- a/serve/server.py +++ b/serve/server.py @@ -2360,8 +2360,8 @@ def _claim_park(self, prompt, exclude=()) -> str: (self.info or {}).get("slot_park") in (1, "1")): return "" main = getattr(self, "main_held", None) or [] - if len(main) < slot_policy.PARK_MIN_TOKENS or prompt[:len(main) // 2] == main[:len(main) // 2]: - return "" # short, or this prompt continues it (at least half of it): nothing to keep + if prompt[:len(main) // 2] == main[:len(main) // 2]: + return "" # nothing held, or this prompt continues it (at least half of it): nothing to keep with self.slot_cv: b = slot_policy.park_destination(main, self.slot_busy, self.slot_held, self.slot_used, self.slot_order, self.slot_gs, time.time(), getattr(self, "slot_cache_valid", None), diff --git a/serve/slot_policy.py b/serve/slot_policy.py index a2688c09924..e8140147df2 100644 --- a/serve/slot_policy.py +++ b/serve/slot_policy.py @@ -7,7 +7,6 @@ from dataclasses import dataclass -PARK_MIN_TOKENS = 8192 # a shorter main-session conversation is not parked def configure(env) -> dict: @@ -17,7 +16,7 @@ def configure(env) -> dict: def settings() -> dict: """The tunables in force (/metrics live.slot_policy).""" - return {"park_min_tokens": PARK_MIN_TOKENS} + return {} def cache_state(tokens: int, last_used: float, now: float, valid: bool = True) -> dict: From f89de4636cbc5b526bf2442018bcb00a6f99beba Mon Sep 17 00:00:00 2001 From: andyhrc <12491949+andyhrc@users.noreply.github.com> Date: Fri, 16 Oct 2026 11:00:49 +0800 Subject: [PATCH 08/21] batch, serve: park=B tells another conversation by the main session's last turn, not by half of it A park copied the main session's conversation only when the new prompt continued less than half of it (engine: resume * 2 < live; server: the first half differs). Two chats of the same client share the system prompt and the tool definitions - tens of thousands of tokens with agent clients - so a 30K conversation that has 20K of them in common with the next chat counted as the same conversation, and the next chat overwrote its last 10K unparked. The same conversation shares what the main session holds up to its last turn: the last <|im_start|>, where the latest reply begins (the place of the turn checkpoint). A next turn sent back without that reply's thinking, or the reply regenerated, still shares it; a prompt that differs before it is another conversation (or an earlier message edited) and the main session's conversation is parked. The engine checks this itself (--turn-token, default <|im_start|>), says the token in INFO (turn_token=, beside slot_park=1), and the server's claim uses the same test; an engine that does not say it counts any difference. "park=B not used" names the turn and the decision. Checked: the engine compiles (-fsyntax-only, the production build's flags); serve/test_slot_alloc.py covers the shared-system-prompt case, a next turn without the thinking, a regenerated reply and an engine without turn_token. Co-Authored-By: Claude Opus 5.5 --- docs/BATCHING.md | 11 ++++++----- serve/server.py | 11 +++++++++-- serve/test_slot_alloc.py | 19 +++++++++++++++++++ src/program/generate.cpp | 25 ++++++++++++++++++------- 4 files changed, 52 insertions(+), 14 deletions(-) diff --git a/docs/BATCHING.md b/docs/BATCHING.md index aa96bed9e63..0f2a164e9e3 100644 --- a/docs/BATCHING.md +++ b/docs/BATCHING.md @@ -430,8 +430,9 @@ other builds measured 173.9-178.2 tok/s, so this is close to the spread between A request alone runs on the solo path, whose state is the engine's main session, not a slot. When the next request is another conversation - an agent's short side request between two turns of a long one - the main session is overwritten. With `STRATA_PIPELINE_ACTIVE_ROWS=1` the engine takes `park=B` on `GEN` / `BGEN` (`INFO slot_park=1`): -when the prompt continues less than half of what the main session holds, it first copies the main session's -conversation (and its last checkpoint) into idle slot B - never the slot it restores from or is admitted to - and says +when the prompt is another conversation (it does not share what the main session holds up to its last turn - the +last `<|im_start|>`, where its latest reply begins), it first copies the main session's conversation (and its last +checkpoint) into idle slot B - never the slot it restores from or is admitted to - and says `PARKED B `. Copying 36-37K tokens took 414-451 ms (five runs, the machine above). ### Server: placement by what the slots hold (`STRATA_SLOT_CACHE_AFFINITY=1`) @@ -444,8 +445,8 @@ policy: end, `BDONE`), whether its state is intact, the spreading order, the group size, the observed prompt rate and whether the engine runs active slots only (`slot_padding_safe=1`). Without that engine capability the server tracks from the `BADM` / `BT` / `BDONE` lines which idle slot a later window of its group has padded over, and marks it. -- With `INFO slot_park=1` the server tracks what the main session holds; when a request is another conversation (the - first half differs), it asks the policy for a slot and sends `park=B`; `PARKED` updates that slot's state. A claim +- With `INFO slot_park=1` the server tracks what the main session holds; when a request is another conversation (as + the engine decides it, with `INFO turn_token`), it asks the policy for a slot and sends `park=B`; `PARKED` updates that slot's state. A claim the engine did not use is released. The policy's interface is `place`, `park_destination`, `cache_state`, `superseded_slots`, `reliable_flags`, @@ -783,7 +784,7 @@ On top of `GEN` / `GENI`: | `BYIELD ` | in | the prompt being read gives way at its next chunk boundary; its part read waits in `` (the admission's own, or a free slot for a solo request) | | `YIELDED ` | out | before the `DONE cancel` of a read that gave way: the request is sent again later and goes on from there | | `INFO ... batch_slots=N` | out | the slots the engine runs (only with `--batch`) | -| `INFO ... slot_padding_safe=1` | out | `STRATA_PIPELINE_ACTIVE_ROWS=1`: a pipelined group's window runs its active slots only; an idle slot keeps its conversation | +| `INFO ... slot_padding_safe=1` | out | `STRATA_PIPELINE_ACTIVE_ROWS=1`: a pipelined group's window runs its active slots only; an idle slot keeps its conversation; with it `slot_park=1` and `turn_token=` (the token a turn begins with) | | `CACHE_SOURCE ` | out | `STRATA_PIPELINE_ACTIVE_ROWS=1`, after a prompt's restore: the idle slot its reused prefix came from (-1: the main session, a checkpoint or the conversation cache) and how long it is | | `park=B` (a key of `GEN` / `BGEN`) | in | with `INFO slot_park=1`: before a different conversation replaces the main session's, copy that one into idle slot B | | `PARKED ` | out | `park=` copied that many tokens into the slot (0: the copy failed, the slot holds nothing usable) | diff --git a/serve/server.py b/serve/server.py index 8681e39d254..b9764f5018e 100644 --- a/serve/server.py +++ b/serve/server.py @@ -2360,8 +2360,15 @@ def _claim_park(self, prompt, exclude=()) -> str: (self.info or {}).get("slot_park") in (1, "1")): return "" main = getattr(self, "main_held", None) or [] - if prompt[:len(main) // 2] == main[:len(main) // 2]: - return "" # nothing held, or this prompt continues it (at least half of it): nothing to keep + # the same conversation shares the main session's history up to its last turn (the last <|im_start|>, where its + # latest reply begins): a next turn without that reply's thinking, or the reply regenerated (as the engine checks) + try: + turn_token = int((self.info or {}).get("turn_token", -1)) + except (TypeError, ValueError): + turn_token = -1 + turn = next((i for i in range(len(main) - 1, 0, -1) if main[i] == turn_token), len(main)) + if prompt[:turn] == main[:turn]: + return "" # nothing held, or the same conversation: nothing to keep with self.slot_cv: b = slot_policy.park_destination(main, self.slot_busy, self.slot_held, self.slot_used, self.slot_order, self.slot_gs, time.time(), getattr(self, "slot_cache_valid", None), diff --git a/serve/test_slot_alloc.py b/serve/test_slot_alloc.py index b64d810f063..9acea160c2c 100644 --- a/serve/test_slot_alloc.py +++ b/serve/test_slot_alloc.py @@ -197,6 +197,25 @@ def dest(self, e, exclude=()): return slot_policy.park_destination(e.main_held, e.slot_busy, e.slot_held, e.slot_used, e.slot_order, e.slot_gs, self.NOW, e.slot_cache_valid, True, exclude) + def test_another_conversation_is_told_by_the_main_history_up_to_its_last_turn(self): + # main: system (turn 1 at 0), user turn at 20000, reply turn at 29000; a prompt that shares all of it up to + # the reply's turn is the same conversation, one that diverges earlier is another (here: only the 20K system + # prompt and tools in common, which is more than half of the 30K held) + T = 248045 + e = self.e() + e.info = {"slot_park": "1", "turn_token": str(T)} + main = [T] + [5] * 19999 + [T] + [6] * 8999 + [T] + [8] * 999 + e.main_held = list(main) + same = main[:29000] + [T] + [9] * 50 + [T] + [1] * 10 # its reply without the thinking, and a next turn + self.assertEqual(e._claim_park(same), "") + self.assertEqual(e._claim_park(main[:29001]), "") # the reply regenerated + other = main[:20000] + [T] + [3] * 500 # another chat of the same client + with self.assertRaises(NotImplementedError): # another conversation: the policy is asked + e._claim_park(other) + e.info = {"slot_park": "1"} # an engine that does not say: any difference counts + with self.assertRaises(NotImplementedError): + e._claim_park(same) + def test_off_without_engine_capability_or_padding_safety(self): e = self.e() e.info = {} diff --git a/src/program/generate.cpp b/src/program/generate.cpp index de60cdf7928..88ca550e61b 100644 --- a/src/program/generate.cpp +++ b/src/program/generate.cpp @@ -12179,7 +12179,9 @@ int main(int argc, char** argv) { o.batch > 0 ? (" batch_slots=" + std::to_string(o.batch) + " slot_cache=" + std::to_string(o.prompt_cache > 0 ? 1 : 0) + " batch_groups=" + std::to_string(o.batch_groups) + - (pipeline_active_rows ? " slot_padding_safe=1 slot_park=1" : "")).c_str() : ""); + (pipeline_active_rows ? " slot_padding_safe=1 slot_park=1 turn_token=" + + std::to_string(o.turn_token) : std::string())).c_str() + : ""); } // issue #29: a request whose heartbeat (tokens, prompt chunks, verify windows) stops for this long is stuck on // a flag nobody will raise - end the engine with where it was, so the server starts it again instead of the @@ -14210,11 +14212,19 @@ int main(int argc, char** argv) { resume == req_pin && live_ok) for (const ConvCheckpoint& c : checks) if ((int64_t) c.ids.size() == resume && c.pinned) pin_sibling = true; - // park=B (see req_park). Only when this prompt is another conversation (it continues less than - // half of what the main session holds), never into the slot this request restores from or runs in. + // park=B (see req_park). Only when this prompt is another conversation: it does not share what the main + // session holds up to its last turn (the last <|im_start|>, where its latest reply begins - a next turn + // without that reply's thinking, or the reply regenerated, still shares it). Never into the slot this + // request restores from or runs in. + int64_t park_turn = (int64_t) live.size(); + if (o.turn_token >= 0) + for (int64_t i = (int64_t) live.size() - 1; i > 0; --i) + if (live[(size_t) i] == o.turn_token) { park_turn = i; break; } + const bool other_conversation = (int64_t) ids.size() < park_turn || + !std::equal(live.begin(), live.begin() + park_turn, ids.begin()); if (pipeline_active_rows && req_park >= 0 && req_park < (int) bs.size() && !bs[(size_t) req_park].active && req_park != admit_slot && req_park != slot_source && live_ok && !live.empty() && - resume * 2 < (int64_t) live.size() && (!from_live || incoming || slot_source >= 0) && !pin_sibling) { + other_conversation && (!from_live || incoming || slot_source >= 0) && !pin_sibling) { const auto t0 = Clock::now(); BSlot& sl = bs[(size_t) req_park]; if (copy_to_slot(req_park, live, err)) { @@ -14242,9 +14252,10 @@ int main(int argc, char** argv) { } std::fflush(stdout); } else if (req_park >= 0) { - std::fprintf(stderr, "strata batch: park=%d not used (rows=%d live_ok=%d live=%zu resume=%lld " - "from_live=%d source=%d admit=%d active=%d)\n", req_park, - pipeline_active_rows ? 1 : 0, live_ok ? 1 : 0, live.size(), (long long) resume, + std::fprintf(stderr, "strata batch: park=%d not used (rows=%d live_ok=%d live=%zu turn=%lld other=%d " + "resume=%lld from_live=%d source=%d admit=%d active=%d)\n", req_park, + pipeline_active_rows ? 1 : 0, live_ok ? 1 : 0, live.size(), (long long) park_turn, + other_conversation ? 1 : 0, (long long) resume, from_live ? 1 : 0, slot_source, admit_slot, req_park < (int) bs.size() && bs[(size_t) req_park].active ? 1 : 0); } From 17fad4863656f2f66314c096787da70de8cd0201 Mon Sep 17 00:00:00 2001 From: andyhrc <12491949+andyhrc@users.noreply.github.com> Date: Sun, 18 Oct 2026 12:04:47 +0800 Subject: [PATCH 09/21] serve: a continued request's record keeps its first read's cache source and placement hint A request moved between the solo path and a slot is admitted again with its prompt and its reply so far; each admission set the record's cache_match_hint and cache_source_slot, so the record paired the first read's prompt and reuse (#1869) with the last leg's hint - more tokens "resumable" than the prompt had (a 207K prompt showed a 214K hint). The values of that first read are kept, as reused0 is. Co-Authored-By: Claude Opus 5.5 --- serve/server.py | 8 ++++++++ serve/test_parallel_yield.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/serve/server.py b/serve/server.py index b9764f5018e..f400515297f 100644 --- a/serve/server.py +++ b/serve/server.py @@ -1941,6 +1941,7 @@ def generate_batched(self, ids, max_new, sampling, cancel, embeddings=None): self.cache_event("request_start", prompt_tokens=len(ids), cache_debug=tl.cache_debug) peer_ip = getattr(self.__dict__.setdefault("_tl", threading.local()), "peer_ip", None) reused0 = None # the first read's reused tokens: a continued leg's count includes this request's own output + first_read = None # that read's cache source and placement hint, for the record: a continued leg's hint counts it too carry = {} # #1869: the figures of the legs that ended in a hand-back to the solo path (the final DONE adds to them) ok = yield from self._take_control(cancel, len(prompt), new=True) if not ok: @@ -2050,6 +2051,8 @@ def give_way(): # at a PP line, and between two far apart (the engine takes it return if reused0 is None: reused0 = (self.last or {}).get("reused") + if first_read is None: + first_read = (getattr(tl, "cache_source_slot", None), getattr(tl, "cache_match_hint", None)) carry_leg(carry, self.last) # #1869: the solo leg's figures; the slot's admission (a copy of # the held prefix: prompt_read 1) and BDONE replace them prompt = list(ids) + out # promoted: it continues in a batch slot from here @@ -2275,6 +2278,9 @@ def note_progress(): slot, gen0 = None, None if reused0 is None: reused0 = (self.last or {}).get("reused") + if first_read is None: + first_read = (getattr(tl, "cache_source_slot", None), + getattr(tl, "cache_match_hint", None)) prompt, left = list(ids) + out, int(max_new) - len(out) solo_again += 1 btrace("back to the solo path") @@ -2305,6 +2311,8 @@ def note_progress(): bdone_in = True if reused0 is not None and isinstance(self.last, dict): self.last = {**self.last, "reused": reused0} + if first_read is not None: + tl.cache_source_slot, tl.cache_match_hint = first_read if carry and isinstance(self.last, dict): self.last = add_carry(carry, self.last) if park_slot is not None: diff --git a/serve/test_parallel_yield.py b/serve/test_parallel_yield.py index eeabe383ae9..9a3248d80e0 100644 --- a/serve/test_parallel_yield.py +++ b/serve/test_parallel_yield.py @@ -286,6 +286,34 @@ def test_a_request_moved_between_solo_and_slot_keeps_its_totals(self): self.assertEqual(t["cache_n"], 3, t) # the first read's reuse only self.assertGreater(t["predicted_ms"], 60.0, t) + def test_a_continued_leg_keeps_the_records_cache_source_and_hint(self): + """A request that read its prompt on the solo path and went on in a slot: its record keeps that first read's + cache source and placement hint, as it keeps its reuse (#1869). The slot admission's hint counts the + request's own output (its prompt there is the prompt and the reply so far), so the record said more could + be resumed than its prompt had.""" + from serve.slot_policy import Placement + self.start(2, slot_cache=True, reuse=True, affinity=True, more=("--source-trace",)) + e, pick = self.engine, self.engine.pick_slot + + def resume_all(prompt): # a plan that resumes the whole prompt it is given + b = pick(prompt) + e.slot_last_plan = Placement(slot=b, source_hint=b, reused_hint=len(prompt), reason="reuse") + return b + e.pick_slot = resume_all + res = {} + long = threading.Thread(target=lambda: res.setdefault("long", self.chat("LONGREPLY please", max_tokens=200))) + long.start() + time.sleep(0.25) # a few solo tokens first + self.chat("short", max_tokens=200) + long.join(30) + kinds = [x.split()[0] for x in self.log.read_text().splitlines()] + self.assertEqual(kinds[:2], ["GEN", "BGEN"], kinds) # solo, then continued in a slot + rows = sorted(self.svc.history, key=lambda h: h["output_tokens"]) + short, longest = rows[0], rows[-1] + self.assertEqual(short["cache_match_hint"], short["prompt_tokens"], short) # its one read, in a slot + self.assertIsNone(longest["cache_match_hint"], longest) # its first read was on the solo path + self.assertEqual(longest["cache_source_slot"], 1, longest) # that read's CACHE_SOURCE + def test_a_leaked_busy_slot_does_not_hang_the_next_request(self): """#1603: a slot whose busy flag has no owner (no request holds it, no BDONE wait is running) kept every later request waiting for a slot nothing would free: running 0, queued 0, and the frozen-engine check never looked. From c7edf14a2263a4a787c53392df6d4e1420a25d06 Mon Sep 17 00:00:00 2001 From: andyhrc <12491949+andyhrc@users.noreply.github.com> Date: Wed, 21 Oct 2026 00:16:23 +0800 Subject: [PATCH 10/21] serve: test_cache_trace imports serve.* by name, so it also loads when serve/ is discovered on its own The "Minimal serving runtime" workflow runs `unittest discover -s serve -p test_*.py` (no top-level directory), where the module has no parent package and its relative imports failed: "attempted relative import with no known parent package". Absolute imports, as the other serve tests use, load either way. Co-Authored-By: Claude Opus 5.5 --- serve/test_cache_trace.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/serve/test_cache_trace.py b/serve/test_cache_trace.py index d2cf27370ab..a7b5ee94e0e 100644 --- a/serve/test_cache_trace.py +++ b/serve/test_cache_trace.py @@ -1,11 +1,11 @@ import unittest -from .cache_trace import snapshot +from serve.cache_trace import snapshot class CacheTrace(unittest.TestCase): def test_resume_after_main_source_is_actual_reuse(self): import queue, threading - from .server import StrataEngine + from serve.server import StrataEngine engine=object.__new__(StrataEngine) engine.lines=queue.Queue() engine._ctl_mode='solo' @@ -21,7 +21,7 @@ def test_resume_after_main_source_is_actual_reuse(self): self.assertEqual(engine.cache_events_view()[0]['reused_tokens'],700) def test_global_expiry_does_not_inherit_request(self): import threading - from .server import StrataEngine + from serve.server import StrataEngine engine=object.__new__(StrataEngine) engine._tl=threading.local() engine._tl.request_key='request-a' From c3e5c8380ccfc09749d1ef6278625a353f8aa111 Mon Sep 17 00:00:00 2001 From: andyhrc <12491949+andyhrc@users.noreply.github.com> Date: Thu, 15 Oct 2026 10:04:20 +0800 Subject: [PATCH 11/21] serve: the slot placement policy - protection by history length, retention scores, one bounded wait, park destinations The policy behind the interface of the previous commit (serve/slot_policy.py). Every threshold is a parameter in the server config's "env": - protection by length: 0.1 s at 500 tokens, 2 s at 2K, 10 s at 8K, 45 s at 32K, 65 s at 50K, 90 s at 100K, 120 s at 262144 (STRATA_SLOT_TTL_MAX_S scales the curve); - retention = (n/50000)^0.7 x (n/(n+2000))^2 x (0.25 + 0.75 x freshness), 0 once expired; overwrite score = 1/(1 + retention); - blank slots (empty, invalid, or a superseded copy) count as expired and worthless; an expired history still matches until it is overwritten; - a continued conversation runs in its slot or a blank slot of a less loaded group - never in a slot that holds another conversation (an earlier version moved two idle Claude Code conversations into each other's slot on every turn; each turn read 60-100K tokens from token 0); near matches (2% of the best, one second of the prompt rate, 4096 tokens) count as sources; - a new conversation takes what is not worth keeping first (blank, or an expired history scoring below STRATA_SLOT_EXPIRED_KEEP_SCORE, default 0.01 - about 2.5K tokens), then by group load, blank before history, the lowest score; - every history protected: one wait of STRATA_SLOT_PROTECT_WAIT_S (default 5 s), then the least loss (losses within 10% by group load); the same wait when only histories worth keeping are left and a cheaper one leaves its protection within it; - park: the conversation's own older copy, else a blank slot of the least loaded group, never another conversation's slot; not under STRATA_SLOT_PARK_MIN_TOKENS (default 8192) or when a slot holds 90% of it already; - on an engine without slot_padding_safe=1, starting a row above a protected idle history costs its retention too. Measured on 3 x RTX A4000 (0.1.44.1, --batch 6 in 3 groups, STRATA_PIPELINE_ACTIVE_ROWS=1): conversation A (37K) parked, two clients generating 1,500-token answers in a loop, conversations B (9K) and C (7.9K) taking turns about every 30 s, past their protection (about 11 and 10 s). Tokens reused on turns 1-3: - expired histories by group load first (an earlier ordering): B 0, C 0 on every turn; - + an expired history worth keeping goes last: B all but the ~20 new tokens, C 0; - + wait for a cheaper slot about to leave its protection (this policy): B and C all but the ~20 new tokens. Park: a 37,359-token conversation's next turn after a side request reused 37,352 tokens and took about 1 s (0 reused, 14 s without), with the same greedy output. Tests: the policy's own in serve/test_slot_alloc.py, serve/test_parallel.py (the wait is bounded, wakes when a protection ends, a cancel leaves every cache in place) and serve/test_cache_trace.py. Co-Authored-By: Claude Opus 5.5 --- docs/BATCHING.md | 57 +++++- serve/slot_policy.py | 180 +++++++++++++++--- serve/test_cache_trace.py | 15 ++ serve/test_parallel.py | 94 ++++++++++ serve/test_slot_alloc.py | 377 +++++++++++++++++++++++++++++++++++++- 5 files changed, 694 insertions(+), 29 deletions(-) diff --git a/docs/BATCHING.md b/docs/BATCHING.md index 0f2a164e9e3..dc6a909afec 100644 --- a/docs/BATCHING.md +++ b/docs/BATCHING.md @@ -450,9 +450,7 @@ policy: the engine did not use is released. The policy's interface is `place`, `park_destination`, `cache_state`, `superseded_slots`, `reliable_flags`, -`configure` / `settings` (their docstrings say what they -answer). This part ships them as an interface without a policy: with `STRATA_SLOT_CACHE_AFFINITY=1` (or -`STRATA_CACHE_TRACE=1`) and no policy they raise `NotImplementedError`; with both off nothing changes. +`configure` / `settings` (their docstrings say what they answer). The policy is described below. `/metrics` shows what happened: `live.cache_events` (the last 96: allocation, admission with the tokens reused and read, resume and cache source, park claims and `parked`, expiries), each request's key, client @@ -463,6 +461,59 @@ where its cache came from), and for slots the client of the conversation each on `serve/cache_trace.py`). The client address is the socket's (behind a reverse proxy, the proxy's); `X-Forwarded-For` and request fields are not used. No prompt text or token ids are recorded. +### The placement policy (`serve/slot_policy.py`) + +Rules alone cannot tell which history to give up when a new conversation needs a slot and none is free of value: the +one used longest ago may be a 90K-token conversation whose user is reading the answer, the newest a side request's 300 +tokens. The policy weighs length and age, and every threshold is a parameter: + +- **Protection grows with length**: 0.1 s at 500 tokens, 2 s at 2K, 10 s at 8K, 45 s at 32K, 65 s at 50K, 90 s at + 100K, 120 s at 262144 (piecewise linear; `STRATA_SLOT_TTL_MAX_S`, default 120, scales the whole curve). A side + request's history is free again almost at once; a long conversation outlasts the time its user reads and types. +- **Retention score** = (n / 50000)^0.7 x (n / (n + 2000))^2 x (0.25 + 0.75 x freshness), freshness = 1 - age / TTL, + and 0 once expired; overwrite score = 1 / (1 + retention). A fresh 50K history is worth about 1, a 500-token one + 0.002; a fresh 50K history outweighs a 150K one about to expire. +- **Blank slots** - empty, known invalid, or a superseded copy (its history is the start of another idle slot's: the + conversation went on there) - count as expired and worth nothing. +- **A continued conversation** (an idle slot's whole history is the start of the prompt; matches within 2% of the + longest, one second of the prompt rate and 4096 tokens count too) runs in its slot or in a blank slot of a less + loaded group - the engine copies it there, so moving costs nothing - and **never in a slot that holds another + conversation**, expired or not. On the machine above an earlier version that put group balance first moved two Claude + Code conversations, idle for minutes between turns, into each other's slot on every turn, and each turn read 60-100K + tokens from token 0. +- **An expired history is not deleted**: it still matches until it is overwritten. A new conversation takes a blank + expired or blank slot: first one not worth keeping (blank, or its score at freshness 0 below + `STRATA_SLOT_EXPIRED_KEEP_SCORE`, default 0.01 - about 2.5K tokens), then by group load, a blank slot before a + history, and the lowest score. +- **One bounded wait**: when every idle history is still protected, the request waits up to + `STRATA_SLOT_PROTECT_WAIT_S` (default 5 s, once per request), then overwrites the history that loses least (losses + within 10% go by group load). It also waits, within the same single wait, when only expired histories worth keeping + are left and a cheaper protected one ends its protection within it. +- **Park** (with `slot_padding_safe=1`): the conversation's own older copy, else a blank slot of the least loaded + group - never a slot that holds another conversation; not when a slot holds 90% of it already, or when it is + shorter than `STRATA_SLOT_PARK_MIN_TOKENS` (default 8192). +- A busy slot is never matched or overwritten. On an engine without `slot_padding_safe=1`, a slot that would pad + over a protected idle history below it costs that history's retention too. + +Measured on the machine above (0.1.44.1, `--batch 6` in 3 groups, `STRATA_PIPELINE_ACTIVE_ROWS=1`): + +- Park: a 37,359-token conversation, a short side request, then its next turn. Without `park=`: 0 tokens reused, the + turn took 14 s; with it: 37,352 reused, about 1 s, and the same greedy output (temperature 0) as before the side + request. +- Retention under load: conversation A (37K tokens) parked, two clients generating 1,500-token answers in a loop (so + batch mode stays on), conversations B (9K) and C (7.9K) taking turns about every 30 s - past their protection of + about 11 and 10 s. Tokens reused on turns 1-3: + +| Ordering of expired histories | B | C | +| --- | --- | --- | +| by group load first (an earlier version) | 0 every turn | 0 every turn | +| + one still worth keeping goes last | all but the ~20 new | 0 every turn | +| + wait for a cheaper slot about to leave its protection (this policy) | all but the ~20 new | all but the ~20 new | + +In the first row a new turn took the other conversation's expired slot for group balance, so B and C overwrote each +other; in the second, C went to a side request that arrived a second after another one finished, while that one's +1.5K-token history was about to leave its protection. + ## The VRAM expert tier keeps adapting in batch windows Batch windows count which experts they route to, and every `--adapt-every` windows (default 4) the engine swaps the diff --git a/serve/slot_policy.py b/serve/slot_policy.py index e8140147df2..c38384f6b0b 100644 --- a/serve/slot_policy.py +++ b/serve/slot_policy.py @@ -1,28 +1,70 @@ -"""The placement policy interface for STRATA_SLOT_CACHE_AFFINITY=1: what serve/server.py (pick_slot, _claim_park, -slots_view) and serve/cache_trace.py call. No policy is here - which idle slot a request runs in, which -history is given up, how long one is protected and where the solo path's conversation is parked are a policy's -choice. Without one, STRATA_SLOT_CACHE_AFFINITY=1 and STRATA_CACHE_TRACE=1 raise NotImplementedError; with both -off nothing here runs (configure() and settings() aside).""" +"""Cache retention and pipelined placement policy for STRATA_SLOT_CACHE_AFFINITY=1; no engine or I/O.""" from __future__ import annotations +import math from dataclasses import dataclass +MAX_CACHE_TOKENS = 262144 +# Tunables (environment, read by configure() when the engine starts; the menu sets them per profile under +# config_overrides.env). Defaults are the values the policy was specified with. +PROTECTION_WAIT_S = 5.0 # STRATA_SLOT_PROTECT_WAIT_S: wait when every idle history is still protected +PARK_MIN_TOKENS = 8192 # STRATA_SLOT_PARK_MIN_TOKENS: a shorter main-session conversation is not parked +EXPIRED_KEEP_SCORE = 0.01 # STRATA_SLOT_EXPIRED_KEEP_SCORE: an expired history scoring at least this (retention at + # freshness 0; 0.01 is about 2.5K tokens) is overwritten only when nothing else is free +TTL_MAX_S = 120.0 # STRATA_SLOT_TTL_MAX_S: protection at 262144 tokens; the whole curve scales with it +_TTL_SHAPE = ((0, 0.0), (500, 0.1), (2000, 2.0), (8000, 10.0), + (32000, 45.0), (50000, 65.0), (100000, 90.0), (MAX_CACHE_TOKENS, 120.0)) +TTL_POINTS = _TTL_SHAPE def configure(env) -> dict: - """Called by StrataEngine with the engine's environment: read the policy's tunables, return settings().""" + """Read the tunables from `env` (the engine's environment); bad or missing values keep the defaults.""" + global PROTECTION_WAIT_S, PARK_MIN_TOKENS, TTL_MAX_S, TTL_POINTS, EXPIRED_KEEP_SCORE + + def value(name, default): + try: + v = float((env or {}).get(name, default)) + return v if math.isfinite(v) and v >= 0 else default + except (TypeError, ValueError): + return default + TTL_MAX_S = value("STRATA_SLOT_TTL_MAX_S", 120.0) + EXPIRED_KEEP_SCORE = value("STRATA_SLOT_EXPIRED_KEEP_SCORE", 0.01) + TTL_POINTS = tuple((n, s * TTL_MAX_S / 120.0) for n, s in _TTL_SHAPE) + PROTECTION_WAIT_S = value("STRATA_SLOT_PROTECT_WAIT_S", 5.0) + PARK_MIN_TOKENS = int(value("STRATA_SLOT_PARK_MIN_TOKENS", 8192)) return settings() def settings() -> dict: - """The tunables in force (/metrics live.slot_policy).""" - return {} + return {"ttl_max_s": TTL_MAX_S, "protect_wait_s": PROTECTION_WAIT_S, + "park_min_tokens": PARK_MIN_TOKENS, "expired_keep_score": EXPIRED_KEEP_SCORE} + + +def cache_ttl(tokens: int) -> float: + n = min(MAX_CACHE_TOKENS, max(0, tokens)) + for (lo, a), (hi, b) in zip(TTL_POINTS, TTL_POINTS[1:]): + if n <= hi: + return a + (b - a) * (n - lo) / (hi - lo) + return TTL_POINTS[-1][1] + + +def stale_retention(tokens: int) -> float: + """The retention score at freshness 0 (the same formula): orders expired histories - lowest is reclaimed first.""" + n = min(MAX_CACHE_TOKENS, max(0, tokens)) + return (n / 50000.0) ** 0.7 * (n / (n + 2000.0)) ** 2 * 0.25 def cache_state(tokens: int, last_used: float, now: float, valid: bool = True) -> dict: - """An idle history of `tokens`, last used at `last_used`: a dict with tokens, ttl_s, age_s, expired, valid, - retention, overwrite_score (/metrics live.slots and serve/cache_trace.py show it).""" - raise NotImplementedError("serve/slot_policy.py has no placement policy") + age = max(0.0, now - last_used) + ttl = cache_ttl(tokens) + expired = tokens == 0 or age >= ttl or not valid + # A 50K history has substantial value. The additional small-history factor makes + # tiny probes much cheaper to replace without flattening all medium histories. + n = min(MAX_CACHE_TOKENS, max(0, tokens)) + freshness = max(0.0, 1.0 - age / ttl) if ttl else 0.0 + retention = 0.0 if expired else (n / 50000.0) ** 0.7 * (n / (n + 2000.0)) ** 2 * (0.25 + 0.75 * freshness) + return {"tokens": tokens, "ttl_s": ttl, "age_s": age, "expired": expired, + "valid": valid, "retention": retention, "overwrite_score": 1.0 / (1.0 + retention)} @dataclass(frozen=True) @@ -45,7 +87,7 @@ def reliable_flags(busy: list[bool], valid: list[bool] | None, group_size: int, def superseded_slots(held: list[list[int]], busy: list[bool], reliable: list[bool]) -> set[int]: """Idle histories that are the start of another idle slot's history: the conversation moved on in that other - slot, so this copy saves nothing it does not. They count as empty (no protection).""" + slot, so this copy saves nothing it does not. They count as empty (no protection, no wait).""" out = set() for b, h in enumerate(held): if h and not busy[b]: @@ -57,20 +99,114 @@ def superseded_slots(held: list[list[int]], busy: list[bool], reliable: list[boo return out +def _blank(held, busy, used, now, reliable): + superseded = superseded_slots(held, busy, reliable) + states = [cache_state(len(h), used[b], now, reliable[b] and b not in superseded) for b, h in enumerate(held)] + # Nothing worth keeping: no tokens, known invalid, or a superseded copy. An expired history counts as empty for + # the protection rules, but it is still a cache (it can match until overwritten), so it is overwritten only after + # these - the one with the lowest score first. + blank = {b for b in range(len(held)) if not held[b] or not reliable[b] or b in superseded} + return states, blank + + def park_destination(main: list[int], busy: list[bool], held: list[list[int]], used: list[float], order: list[int], group_size: int, now: float, valid: list[bool] | None = None, padding_safe: bool = False, exclude=()) -> int | None: - """The idle slot to keep the main session's conversation `main` in before another request replaces it (the - engine's park=B), never one in `exclude`; None: do not park.""" - raise NotImplementedError("serve/slot_policy.py has no placement policy") + """An idle slot to keep the main session's conversation in before another request replaces it (engine park=): + its own older copy, else a blank slot of the least loaded group. Never another conversation's history - parking + must not evict anything. None: nowhere to keep it (or it is already kept).""" + if not padding_safe or not main: + return None + gs = max(1, group_size) + reliable = reliable_flags(busy, valid, gs, padding_safe) + for b, h in enumerate(held): # already kept (all but a short tail) - in an idle slot or one decoding it + if h and len(h) * 10 >= len(main) * 9 and (main[:len(h)] == h or h[:len(main)] == main): + return None + _, blank = _blank(held, busy, used, now, reliable) + positions = {b: i for i, b in enumerate(order)} + loads = {g: sum(busy[g:g + gs]) for g in range(0, len(busy), gs)} + own = [b for b in order if not busy[b] and b not in exclude and held[b] and reliable[b] and + len(held[b]) < len(main) and main[:len(held[b])] == held[b]] + if own: + return max(own, key=lambda b: (len(held[b]), -positions[b])) + free = [b for b in order if not busy[b] and b not in exclude and b in blank] + return min(free, key=lambda b: (loads[b - b % gs], positions[b])) if free else None def place(prompt: list[int], busy: list[bool], held: list[list[int]], used: list[float], order: list[int], group_size: int, now: float, valid: list[bool] | None = None, - prefill_rate: float | None = None, padding_safe: bool = False) -> Placement: - """The idle slot `prompt` runs in (the caller holds slot_cv). busy/held/used/valid: each slot's state (held: the - tokens it holds, used: when it was last used, valid: the server's knowledge that its state is intact); order: the - slots in the order that spreads requests over the pipeline groups; group_size: slots per group; prefill_rate: the - observed prompt rate (tok/s); padding_safe: the engine runs a group's active slots only (INFO - slot_padding_safe=1). A busy slot is never returned; Placement(None) only when every slot is busy.""" - raise NotImplementedError("serve/slot_policy.py has no placement policy") + prefill_rate: float | None = None, allow_eviction: bool = True, + padding_safe: bool = False) -> Placement: + free = [b for b in order if not busy[b]] + if not free: + return Placement(None) + gs = max(1, group_size) + validity = valid if valid is not None else [True] * len(busy) + # Conservative current-window check plus remembered native-line invalidations. + reliable = reliable_flags(busy, validity, gs, padding_safe) + states, blank = _blank(held, busy, used, now, reliable) + matches = {b: len(held[b]) for b in free if reliable[b] and held[b] and + len(held[b]) < len(prompt) and prompt[:len(held[b])] == held[b]} + best = max(matches.values(), default=0) + # Near = at most 2% of the best prefix, 1 second of observed prefill, and 4096 tokens. + rate = prefill_rate if prefill_rate and math.isfinite(prefill_rate) and prefill_rate > 0 else 1000.0 + near = int(min(best * 0.02, rate, 4096)) + sources = [b for b, n in matches.items() if best - n <= near] + loads = {g: sum(busy[g:g + gs]) for g in range(0, len(busy), gs)} + positions = {b: i for i, b in enumerate(order)} + + def collateral(b): + if padding_safe: + return 0.0 + # Starting an upper row can pad over a lower idle protected history. A matching + # ancestor is the same conversation, so its displacement does not lose another chat. + return sum(states[x]["retention"] for x in range(b - b % gs, b) + if not busy[x] and x not in matches) + + def loss(b): + return (0.0 if b in matches else states[b]["retention"]) + collateral(b) + + def pipeline(b): + g0 = b - b % gs + high = max([x - g0 for x in range(g0, g0 + gs) if busy[x]] + [-1]) + extra_rows = 1 if padding_safe else max(0, b - g0 - high) + return loads[g0], extra_rows, positions[b] + + if sources: + # Cache source and execution destination are separate: the native engine restores the longest valid + # prefix from any idle slot, then copies the conversation into the destination (it copies even when they + # are the same slot, so moving costs nothing extra). Move only into a blank slot, and only for a less + # loaded group; never over another conversation's history, expired or not (2026-10-14: two chats moved + # into each other's expired slot on every turn and re-read 60-100K tokens each time). + candidates = sources + [b for b in free if b not in sources and b in blank and collateral(b) == 0] + slot = min(candidates, key=lambda b: (loss(b), *pipeline(b)[:2], b not in sources, positions[b])) + source = max(sources, key=lambda b: (matches[b], -positions[b])) + return Placement(slot, source, best, near, "reuse") + + safe = [b for b in free if states[b]["expired"] and collateral(b) == 0] + + def stale(b): + return 0.0 if b in blank else stale_retention(len(held[b])) + + def worthy(b): + return stale(b) >= EXPIRED_KEEP_SCORE and stale(b) > 0 + + if safe: + # Expired = empty for protection, but an expired history can still match until it is overwritten: one that + # still scores EXPIRED_KEEP_SCORE goes only when no blank or worthless slot is free (else a new chat in the + # idle group overwrote the other chat every turn - GPU run 2026-10-15). Then group load; then the lowest score. + slot = min(safe, key=lambda b: (worthy(b), *pipeline(b)[:2], b not in blank, stale(b), positions[b])) + # Only worthy histories are left, but a protected slot worth less ends its protection within the wait window + # (a side request that has just finished): wait for it under the same one-per-request 5 s protection wait + # rather than destroy a conversation that can still match. After the wait the overwrite goes ahead. + if worthy(slot) and not allow_eviction and any( + not states[b]["expired"] and collateral(b) == 0 and stale(b) < stale(slot) and + states[b]["ttl_s"] - states[b]["age_s"] <= PROTECTION_WAIT_S for b in free): + return Placement(None, reason="protected") + return Placement(slot, reason="reclaim") + if not allow_eviction: + return Placement(None, reason="protected") + cheapest = min(loss(b) for b in free) + # Loss scores within 10% are close enough to let group load/padding break the tie. + candidates = [b for b in free if loss(b) <= cheapest * 1.10 + 1e-12] + return Placement(min(candidates, key=lambda b: (*pipeline(b), loss(b), used[b])), reason="evict") diff --git a/serve/test_cache_trace.py b/serve/test_cache_trace.py index a7b5ee94e0e..30222523e1c 100644 --- a/serve/test_cache_trace.py +++ b/serve/test_cache_trace.py @@ -2,6 +2,21 @@ from serve.cache_trace import snapshot class CacheTrace(unittest.TestCase): + def test_prefix_boundary_busy_and_expired(self): + result=snapshot([1,2,3,4], [[1,2],[1,7],[1,2,3,4,5],[]], + [False,True,False,False],[True,True,False,True],[0,99,99,0],100) + self.assertEqual([r['common_prefix_tokens'] for r in result['slots']], [2,1,4,0]) + self.assertEqual([r['full_prefix_match'] for r in result['slots']], [True,False,False,False]) + self.assertTrue(result['slots'][0]['expired']) + self.assertTrue(result['slots'][1]['busy']) + self.assertFalse(result['slots'][2]['valid_hint']) + self.assertNotIn('prompt', result) + def test_matches_equal_prefixes_at_every_boundary(self): + held=list(range(300)) + for i in range(301): + prompt=held[:i]+[-1] + result=snapshot(prompt,[held],[False],[True],[0],100) + self.assertEqual(result['slots'][0]['common_prefix_tokens'],i) def test_resume_after_main_source_is_actual_reuse(self): import queue, threading diff --git a/serve/test_parallel.py b/serve/test_parallel.py index cbae1dd42ee..9ed4348ffa9 100644 --- a/serve/test_parallel.py +++ b/serve/test_parallel.py @@ -376,6 +376,71 @@ def get(self, path): with urllib.request.urlopen(self.base + path, timeout=10) as r: return json.loads(r.read().decode()) + def _protected_waiter(self): + self.start(6, says_groups=3, affinity=True) + e = self.engine + with e.slot_cv: + e.slot_held = [[9000 + b] * 50000 for b in range(6)] + e.slot_used = [time.time()] * 6 + e.slot_cache_valid = [True] * 6 + # Keep one request marked busy to exercise admission instead of the solo path. + e.slot_busy[0] = True + e.slot_claims.add(0) + cancel, result = threading.Event(), {} + def run(): + try: + result['tokens'] = list(e.generate_batched([9999], 8, {}, cancel)) + except BaseException as ex: + result['error'] = ex + t = threading.Thread(target=run) + t.start() + end = time.monotonic() + 2 + while time.monotonic() < end and not e.admitting: + time.sleep(0.01) + self.assertGreater(e.admitting, 0) + self.assertTrue(e.ctl.acquire(blocking=False)) # waiting never monopolizes the control channel + e.ctl.release() + return cancel, result, t + + def test_affinity_protection_wait_is_bounded_and_then_admits(self): + started = time.monotonic() + cancel, result, t = self._protected_waiter() + try: + t.join(8) + self.assertFalse(t.is_alive()) + self.assertNotIn('error', result) + self.assertGreaterEqual(time.monotonic() - started, 4.8) + self.assertTrue(result['tokens']) + finally: + cancel.set() + t.join(3) + + def test_affinity_wait_wakes_when_a_short_history_expires(self): + cancel, result, t = self._protected_waiter() + try: + with self.engine.slot_cv: + self.engine.slot_held[4] = [42] * 500 + self.engine.slot_used[4] = time.time() - 0.2 + self.engine.slot_cv.notify_all() + t.join(2) + self.assertFalse(t.is_alive()) + self.assertNotIn('error', result) + self.assertTrue(result['tokens']) + finally: + cancel.set() + t.join(3) + + def test_affinity_wait_can_be_cancelled_without_overwriting_cache(self): + cancel, result, t = self._protected_waiter() + before = list(self.engine.slot_held) + cancel.set() + t.join(2) + self.assertFalse(t.is_alive()) + self.assertNotIn('error', result) + self.assertEqual(result.get('tokens'), []) + self.assertEqual(self.engine.slot_held, before) + self.assertEqual(self.engine.admitting, 0) + def _two_at_once(self, text_b): """A long request first, then one that has to go to a batch slot (the engine is busy with the first).""" out = {} @@ -412,6 +477,16 @@ class ParallelService(FakeEngineCase): """The real StrataEngine and Service over HTTP, the fake engine behind them: slots, queues, benchmarks, telemetry. The yielding, restart and failure tests are in serve/test_parallel_yield.py (this file alone ran past 100 s).""" + def test_expiry_state_and_event_are_visible_in_same_metrics_response(self): + self.start(4, says_groups=2, affinity=True) + self.engine.slot_held[0] = [1] * 500 + self.engine.slot_used[0] = time.time() - 1 + self.engine.slot_cache_valid[0] = True + metrics = self.get("/metrics")["live"] + self.assertTrue(metrics["slots"][0]["cache_expired"]) + self.assertTrue(any(event["kind"] == "expired" and event["slot"] == 0 + for event in metrics["cache_events"])) + def test_solo_native_source_is_forwarded_without_reserving_batch_slot(self): self.start(2, more=["--source-trace"]) out = {} @@ -436,6 +511,25 @@ def test_solo_native_source_is_forwarded_without_reserving_batch_slot(self): self.assertEqual([live[k] for k in ("solo_peer_ip", "solo_request_key", "solo_cache_source_slot", "solo_cache_reused_tokens")], [None] * 4) + def test_http_peer_is_recorded_without_trusting_forwarded_ip(self): + self.start(2, says_groups=2, affinity=True) + body = {"messages": [{"role": "user", "content": "hello"}], + "max_tokens": 16, "reasoning_effort": "none", "peer_ip": "192.0.2.99"} + req = urllib.request.Request(self.base + "/v1/chat/completions", data=json.dumps(body).encode(), + headers={"Content-Type": "application/json", "X-Forwarded-For": "192.0.2.99"}) + with urllib.request.urlopen(req, timeout=30) as response: + self.assertEqual(response.status, 200) + response.read() + history = self.get("/metrics")["requests"] + self.assertEqual(history[0]["peer_ip"], "127.0.0.1") + self.assertIsNone(self.get("/metrics")["live"]["peer_ip"]) + out = self._two_at_once("another") + self.assertEqual(out["b"][0], "ok", out) + events = self.get("/metrics")["live"]["cache_events"] + allocations = [x for x in events if x["kind"] == "allocation"] + self.assertTrue(allocations) + self.assertTrue(all(x["peer_ip"] == "127.0.0.1" for x in allocations)) + def test_bgen_refused_by_the_engine_is_retried_not_a_400(self): """#1994: the engine refuses a BGEN ("no such free slot") although the server saw the slot free: the request goes to another slot (or waits) and is answered; the same conversation's next turn works too.""" diff --git a/serve/test_slot_alloc.py b/serve/test_slot_alloc.py index 9acea160c2c..6292321fd09 100644 --- a/serve/test_slot_alloc.py +++ b/serve/test_slot_alloc.py @@ -84,6 +84,87 @@ def test_environment_survives_lazy_restart_setup(self): self.assertTrue(restarted.slot_cache_affinity) self.assertFalse(StrataEngine("missing", [], env={}, lazy=True).slot_cache_affinity) + def test_two_long_histories_stay_in_their_slots_over_alternating_turns(self): + e = self.engine() + a, b = list(range(1000)), list(range(2000, 3000)) + e.slot_held[0], e.slot_held[2] = list(a), list(b) + for turn in range(8): + for slot, history in ((2, b), (0, a)): + history.append(4000 + turn) + self.assertEqual(e.pick_slot(history), slot) + e.slot_held[slot] = list(history) + + def test_longest_available_prefix_beats_group_spreading(self): + e = self.engine() + e.slot_held[0] = list(range(600)) + e.slot_held[5] = list(range(900)) + e.pick_slot(list(range(1000))) + self.assertEqual(e.slot_last_plan.source_hint, 5) + e.slot_busy[5] = True + self.assertEqual(e.pick_slot(list(range(1000))), 0) + + def test_new_helper_uses_empty_group_without_padding_old_histories(self): + e = self.engine() + e.slot_held[0], e.slot_held[2] = list(range(600)), list(range(2000, 2600)) + self.assertEqual(e.pick_slot([9999]), 4) + e.slot_busy[4] = True + self.assertEqual(e.pick_slot([9999]), 5) + + def test_all_expired_histories_are_logically_empty(self): + e = self.engine() + e.slot_held = [[100 + b] for b in range(6)] + e.slot_used = [8, 7, 6, 5, 1, 3] + self.assertEqual(e.pick_slot([9999]), 0) + + def test_short_and_equal_histories_do_not_force_affinity(self): + e = self.engine() + e.slot_held[5] = [1, 2] + self.assertEqual(e.pick_slot([1, 2, 3]), 0) + e.slot_held[5] = list(range(900)) + self.assertEqual(e.pick_slot(list(range(900))), 0) + + def test_full_load_waits_and_empty_slots_still_spread(self): + e = self.engine() + got = [] + for _ in range(6): + slot = e.pick_slot([9999]) + got.append(slot) + e.slot_busy[slot] = True + self.assertEqual(got, [0, 2, 4, 1, 3, 5]) + self.assertIsNone(e.pick_slot([9999])) + + def test_cold_request_uses_idle_group_before_filling_busy_group(self): + e = self.engine() + e.slot_busy[3] = e.slot_busy[5] = True + # Every free slot has historical tokens, as in the reported live 2/3/5 placement. + e.slot_held = [[100 + b] for b in range(6)] + e.slot_used = [90, 90, 1, 90, 2, 90] + self.assertEqual(e.pick_slot([9999]), 0) + + def test_padding_threatened_prefix_does_not_override_idle_group(self): + e = self.engine() + e.slot_busy[3] = e.slot_busy[5] = True + e.slot_held = [[100 + b] for b in range(6)] + e.slot_held[2] = list(range(1000)) + e.slot_used = [0] * 6 + self.assertEqual(e.pick_slot(list(range(1001))), 0) + + def test_valid_match_still_beats_group_load(self): + e = self.engine() + e.slot_busy[2] = True + e.slot_held[3] = list(range(1000)) + # An upper slot's history is not padded by a lower busy slot. + e.pick_slot(list(range(1001))) + self.assertEqual(e.slot_last_plan.source_hint, 3) + + def test_cold_requests_spread_when_a_group_has_a_leading_hole(self): + e = self.engine() + e.slot_busy[3] = e.slot_busy[5] = True + self.assertEqual(e.pick_slot([9999]), 0) + e.slot_busy[0] = True + # Now all groups have one request: filling a leading hole beats another row. + self.assertEqual(e.pick_slot([9999]), 2) + class OneGroup(unittest.TestCase): def test_reuse_then_least_recently_used(self): @@ -136,6 +217,205 @@ def place(self, e, prompt, **kw): return slot_policy.place(prompt, e.slot_busy, e.slot_held, e.slot_used, e.slot_order, e.slot_gs, self.NOW, **kw) + def test_ttl_curve_anchors_cap_and_monotonicity(self): + for n, seconds in ((500, 0.1), (50000, 65), (100000, 90), (262144, 120), (900000, 120)): + self.assertAlmostEqual(slot_policy.cache_ttl(n), seconds) + curve = [slot_policy.cache_ttl(n) for n in range(0, 300000, 137)] + self.assertEqual(curve, sorted(curve)) + + def test_expiry_is_virtual_and_expired_history_still_matches(self): + e = self.e() + h = list(range(500)) + e.slot_held[2] = h + e.slot_used[2] -= 10 + plan = self.place(e, h + [9999], allow_eviction=False) + self.assertEqual(plan.source_hint, 2) + self.assertEqual(plan.reused_hint, 500) + self.assertEqual(e.slot_held[2], h) # no token deletion on expiry + + def test_short_prompt_is_far_cheaper_than_fifty_k(self): + tiny = slot_policy.cache_state(500, self.NOW, self.NOW) + medium = slot_policy.cache_state(50000, self.NOW, self.NOW) + long = slot_policy.cache_state(150000, self.NOW, self.NOW) + self.assertLess(tiny['retention'], medium['retention'] / 100) + self.assertGreater(medium['retention'], 0.5) + self.assertLess(medium['retention'], long['retention']) + + def test_fresh_medium_outweighs_nearly_expired_long_history(self): + a = slot_policy.cache_state(50000, self.NOW, self.NOW) + b = slot_policy.cache_state(150000, self.NOW - 97, self.NOW) + self.assertGreater(a['retention'], b['retention']) + + def test_unexpired_fifty_k_is_selected_before_similarly_fresh_150k(self): + e = self.e() + e.slot_held = [[b] * (50000 if b == 2 else 150000) for b in range(6)] + self.assertEqual(self.place(e, [9999]).slot, 2) + self.assertEqual(self.place(e, [9999], allow_eviction=False).reason, 'protected') + + def test_expired_short_helper_is_reclaimed_without_touching_long_histories(self): + e = self.e() + e.slot_held = [[b] * 100000 for b in range(6)] + e.slot_held[4] = [4] * 500 + e.slot_used[4] -= 0.11 + self.assertEqual(self.place(e, [9999], allow_eviction=False).slot, 4) + + def test_cache_source_can_be_restored_into_another_pipeline_group(self): + e = self.e() + e.slot_busy[0] = True + e.slot_held[1] = list(range(10000)) + plan = self.place(e, list(range(10001))) + self.assertEqual(plan.source_hint, 1) + self.assertEqual(plan.slot, 2) + + def test_close_matches_use_group_load_without_forcing_shorter_native_reuse(self): + e = self.e() + e.slot_busy[0] = True + e.slot_held = [[b] * 50000 for b in range(6)] + e.slot_held[1] = list(range(10000)) + e.slot_held[2] = list(range(9850)) + plan = self.place(e, list(range(10001)), prefill_rate=3000) + self.assertEqual(plan.near_tokens, 200) + self.assertEqual(plan.slot, 2) + self.assertEqual(plan.reused_hint, 10000) + e.slot_held[2] = list(range(9000)) # a superseded copy of slot 1: blank, in a less loaded group + plan = self.place(e, list(range(10001)), prefill_rate=3000) + self.assertEqual((plan.slot, plan.source_hint), (2, 1)) + e.slot_held[2] = [2] * 50000 # another chat: never overwritten just to balance groups + self.assertEqual(self.place(e, list(range(10001)), prefill_rate=3000).slot, 1) + + def test_two_idle_conversations_do_not_overwrite_each_other(self): + # Production 2026-10-14: A (slot 2) and B (slot 0), idle past their TTL between turns. A was moved into + # B's expired slot 0 and B back into slot 0 over A, so every turn re-read 60-100K tokens from 0. + for safe in (False, True): + e = self.e() + a, b = [1] * 92000, [2] * 61000 + e.slot_held[2], e.slot_held[0] = list(a), list(b) + e.slot_used[2] = e.slot_used[0] = self.NOW - 300 + for turn in range(6): + for slot, h in ((2, a), (0, b)): + h += [3] * 500 + plan = self.place(e, h + [4], padding_safe=safe) + self.assertEqual((plan.slot, plan.source_hint), (slot, slot), (safe, turn)) + e.slot_held[slot] = list(h) + e.slot_used[slot] = self.NOW - 300 + + def test_match_moves_only_into_a_blank_slot_of_a_less_loaded_group(self): + e = self.e() + e.slot_busy[0] = True + e.slot_held[1] = list(range(10000)) + e.slot_held[2] = [2] * 50000 + e.slot_used[2] = self.NOW - 999 # expired, but still another chat's cache + plan = self.place(e, list(range(10001)), padding_safe=True) + self.assertEqual((plan.slot, plan.source_hint), (4, 1)) # blank slot 4, not expired slot 2 + e.slot_held[4], e.slot_held[5] = [4] * 50000, [5] * 50000 + e.slot_held[3] = [3] * 50000 + plan = self.place(e, list(range(10001)), padding_safe=True) + self.assertEqual(plan.slot, 1) # no blank slot elsewhere: stay in the busy group + + def test_superseded_copy_is_blank_and_needs_no_wait(self): + e = self.e() + e.slot_held = [[b] * 50000 for b in range(6)] # all fresh: a new chat would have to wait + e.slot_held[0] = list(range(9000)) # where the conversation was before it moved + e.slot_held[4] = list(range(10000)) # where it is now + self.assertEqual(slot_policy.superseded_slots(e.slot_held, e.slot_busy, [True] * 6), {0}) + plan = self.place(e, [9999], allow_eviction=False) + self.assertEqual((plan.slot, plan.reason), (0, "reclaim")) + + def test_reclaim_overwrites_expired_history_only_after_blank_slots(self): + e = self.e() + e.slot_held = [[b] * 50000 for b in range(6)] + e.slot_held[2] = [2] * 80000 + e.slot_held[3] = [3] * 40000 + e.slot_used[2] = e.slot_used[3] = self.NOW - 999 # both expired + e.slot_held[5] = [] + self.assertEqual(self.place(e, [9999], padding_safe=True).slot, 5) + e.slot_held[5] = [5] * 50000 + self.assertEqual(self.place(e, [9999], padding_safe=True).slot, 3) # shorter expired history first + + def test_protected_history_is_kept_and_expired_ones_go_lowest_score_first(self): + # Under the TTL rules an expired history is reclaimable and a protected one is not, whatever their lengths; + # among expired ones the retention formula (freshness 0) decides. + e = self.e() + e.slot_busy[1] = e.slot_busy[4] = e.slot_busy[5] = True + e.slot_held[0], e.slot_used[0] = [0] * 37365, self.NOW - 300 # expired + e.slot_held[2], e.slot_used[2] = [2] * 1526, self.NOW - 1 # still inside its TTL: protected + e.slot_held[3], e.slot_used[3] = [3] * 7848, self.NOW - 13 # expired + self.assertEqual(self.place(e, [9999], padding_safe=True).slot, 3) + self.assertLess(slot_policy.stale_retention(7848), slot_policy.stale_retention(37365)) + + def test_new_chat_takes_a_worthless_slot_in_a_busier_group_before_an_expired_conversation(self): + # GPU run 2026-10-15 on 0.1.44.1: B (slot 1, 9K, past its 11 s TTL) sat in the idle group; new chat C took + # slot 1 on group load and the two overwrote each other every turn. Old 1.5K filler histories were free. + e = self.e() + e.slot_busy[2] = e.slot_busy[4] = True + e.slot_held[0], e.slot_used[0] = [0] * 37365, self.NOW - 300 # parked A, expired + e.slot_held[1], e.slot_used[1] = [1] * 8970, self.NOW - 20 # B, expired + e.slot_held[3], e.slot_used[3] = [3] * 1526, self.NOW - 30 # filler leftovers, expired + e.slot_held[5], e.slot_used[5] = [5] * 1526, self.NOW - 30 + self.assertEqual(self.place(e, [9999], padding_safe=True).slot, 3) + slot_policy.configure({"STRATA_SLOT_EXPIRED_KEEP_SCORE": "1"}) # keep nothing expired: load decides + try: + self.assertEqual(self.place(e, [9999], padding_safe=True).slot, 1) + finally: + slot_policy.configure({}) + + def test_waits_for_a_side_request_slot_about_to_expire_instead_of_a_conversation(self): + # GPU run 2026-10-15: a filler arriving 1 s after another filler finished overwrote expired 7.8K chat C, + # while the finished filler's 1.5K history left its 1.4 s protection a moment later. + e = self.e() + e.slot_busy[1] = e.slot_busy[5] = True + e.slot_held[0], e.slot_used[0] = [0] * 37365, self.NOW - 300 # expired, worthy + e.slot_held[3], e.slot_used[3] = [3] * 7848, self.NOW - 15 # C: expired, worthy + e.slot_held[2], e.slot_used[2] = [2] * 1526, self.NOW - 0.5 # filler: protected for under 1 s more + e.slot_held[4], e.slot_used[4] = [4] * 1526, self.NOW - 0.2 + self.assertEqual(self.place(e, [9999], padding_safe=True, allow_eviction=False).reason, "protected") + self.assertEqual(self.place(e, [9999], padding_safe=True, allow_eviction=True).slot, 3) # after the wait + e.slot_used[2] = self.NOW - 2 # the filler slot expired: no wait + self.assertEqual(self.place(e, [9999], padding_safe=True, allow_eviction=False).slot, 2) + + def test_near_match_budget_respects_one_second_and_4096_cap(self): + e = self.e() + e.slot_held[0] = list(range(260000)) + p = list(range(260001)) + self.assertEqual(self.place(e, p, prefill_rate=500).near_tokens, 500) + self.assertEqual(self.place(e, p, prefill_rate=10000).near_tokens, 4096) + + def test_busy_slots_never_match_or_get_overwritten_after_wait(self): + e = self.e() + e.slot_busy = [True] * 6 + e.slot_held[2] = list(range(500)) + self.assertIsNone(self.place(e, list(range(501)), allow_eviction=True).slot) + with mock.patch('serve.server.time.monotonic', return_value=20): + self.assertEqual(e._admission_slot(list(range(501)), None), (None, None)) + + def test_padding_loss_protects_another_chat_in_a_lower_row(self): + e = self.e() + e.slot_held[0] = [77] * 100000 + e.slot_busy[2:] = [True] * 4 + # Slot 1 is empty, but using it would pad over the protected history at 0. + self.assertEqual(self.place(e, [9999], allow_eviction=False).reason, 'protected') + self.assertEqual(self.place(e, [9999], allow_eviction=True).slot, 0) + + def test_protection_window_once_then_eviction_and_early_reclaim(self): + e = self.e() + e.slot_held = [[b] * 50000 for b in range(6)] + with mock.patch('serve.server.time.time', return_value=self.NOW), \ + mock.patch('serve.server.time.monotonic', return_value=10): + slot, since = e._admission_slot([9999], None) + self.assertIsNone(slot) + self.assertEqual(since, 10) + with mock.patch('serve.server.time.time', return_value=self.NOW), \ + mock.patch('serve.server.time.monotonic', return_value=14.9): + self.assertEqual(e._admission_slot([9999], since), (None, 10)) + with mock.patch('serve.server.time.time', return_value=self.NOW), \ + mock.patch('serve.server.time.monotonic', return_value=15): + self.assertIsNotNone(e._admission_slot([9999], since)[0]) + e.slot_held[4] = [4] * 500 + e.slot_used[4] = self.NOW - 0.2 + with mock.patch('serve.server.time.time', return_value=self.NOW), \ + mock.patch('serve.server.time.monotonic', return_value=10.2): + self.assertEqual(e._admission_slot([9999], since)[0], 4) + def test_padding_invalidity_survives_completion_of_higher_row(self): e = self.e() e.slot_cv = threading.Condition() @@ -210,11 +490,53 @@ def test_another_conversation_is_told_by_the_main_history_up_to_its_last_turn(se self.assertEqual(e._claim_park(same), "") self.assertEqual(e._claim_park(main[:29001]), "") # the reply regenerated other = main[:20000] + [T] + [3] * 500 # another chat of the same client - with self.assertRaises(NotImplementedError): # another conversation: the policy is asked - e._claim_park(other) + self.assertTrue(e._claim_park(other).startswith(" park=")) # another conversation: parked e.info = {"slot_park": "1"} # an engine that does not say: any difference counts - with self.assertRaises(NotImplementedError): - e._claim_park(same) + self.assertTrue(e._claim_park(same).startswith(" park=")) + + def test_side_request_claims_a_blank_slot_and_parked_line_records_it(self): + e = self.e() + e.slot_busy[0] = True + e.slot_held[2] = [2] * 30000 # another chat: never a park target + key = e._claim_park([1] * 60) + self.assertEqual(key, " park=4") # blank slot of the least loaded group + self.assertTrue(e.slot_busy[4]) + e._on_parked(4, 49999) + self.assertEqual((len(e.slot_held[4]), e.slot_busy[4], e.slot_cache_peer[4]), (49999, False, "192.0.2.9")) + self.assertEqual(e.cache_events_view()[0]["kind"], "parked") + e._claim_park([1] * 60) # the engine held two tail tokens more than we know + e._on_parked(e._park_claims and next(iter(e._park_claims)) or 0, 50002) + self.assertIn(50000, [len(h) for h in e.slot_held]) + + def test_continuing_short_or_already_kept_conversations_are_not_parked(self): + e = self.e() + self.assertEqual(e._claim_park(e.main_held + [1, 2, 3]), "") # the same conversation continues + e.main_held = [7] * 5000 + self.assertEqual(e._claim_park([1] * 60), "") # short: cheap to read again + e.main_held = [7] * 50000 + e.slot_held[3] = [7] * 48000 # a slot holds 96% of it already + self.assertEqual(e._claim_park([1] * 60), "") + + def test_own_older_copy_is_refreshed_before_using_a_blank_slot(self): + e = self.e() + e.slot_held[5] = [7] * 30000 # this conversation, a few turns back + self.assertEqual(self.dest(e), 5) + + def test_nowhere_to_park_without_evicting(self): + e = self.e() + e.slot_held = [[b] * 40000 for b in range(6)] # every slot holds another fresh conversation + self.assertIsNone(self.dest(e)) + self.assertEqual(e._claim_park([1] * 60), "") + self.assertFalse(any(e.slot_busy)) + + def test_admission_slot_is_never_the_park_slot_and_claim_is_released(self): + e = self.e() + self.assertNotEqual(self.dest(e, exclude=(0,)), 0) + key = e._claim_park([1] * 60, exclude=(0,)) + b = int(key.split("=")[1]) + e._release_park(b) + self.assertFalse(e.slot_busy[b]) + self.assertEqual(e.slot_held[b], []) def test_off_without_engine_capability_or_padding_safety(self): e = self.e() @@ -225,7 +547,37 @@ def test_off_without_engine_capability_or_padding_safety(self): self.assertEqual(e._claim_park([1] * 60), "") +class PolicyTunables(unittest.TestCase): + def tearDown(self): + slot_policy.configure({}) + + def test_environment_overrides_and_bad_values_keep_defaults(self): + got = slot_policy.configure({"STRATA_SLOT_TTL_MAX_S": "240", "STRATA_SLOT_PROTECT_WAIT_S": "2", + "STRATA_SLOT_PARK_MIN_TOKENS": "x"}) + self.assertEqual(got, {"ttl_max_s": 240.0, "protect_wait_s": 2.0, "park_min_tokens": 8192, + "expired_keep_score": 0.01}) + self.assertAlmostEqual(slot_policy.cache_ttl(100000), 180.0) # the whole curve doubles + self.assertAlmostEqual(slot_policy.cache_ttl(10 ** 6), 240.0) + self.assertEqual(slot_policy.configure({})["ttl_max_s"], 120.0) + self.assertAlmostEqual(slot_policy.cache_ttl(100000), 90.0) + + def test_engine_reads_them_from_its_environment(self): + e = StrataEngine("missing", [], env={"STRATA_SLOT_PARK_MIN_TOKENS": "20000"}, lazy=True) + self.assertEqual(e.slot_policy_settings["park_min_tokens"], 20000) + + class NativePaddingSafety(unittest.TestCase): + def test_padding_safe_engine_keeps_lower_match_available(self): + e = engine(6, 3) + e.slot_busy[3] = True + e.slot_held[2] = list(range(1000)) + p = list(range(1001)) + legacy = slot_policy.place(p, e.slot_busy, e.slot_held, e.slot_used, e.slot_order, 2, 1000) + safe = slot_policy.place(p, e.slot_busy, e.slot_held, e.slot_used, e.slot_order, 2, 1000, + padding_safe=True) + self.assertEqual(legacy.reused_hint, 0) + self.assertEqual(safe.source_hint, 2) + self.assertEqual(safe.reused_hint, 1000) def test_safe_native_steps_do_not_invalidate_finished_lower_slot(self): e = engine(6, 3) @@ -247,6 +599,23 @@ def test_safe_native_steps_do_not_invalidate_finished_lower_slot(self): class CacheObservability(unittest.TestCase): + def test_expiry_is_reported_once_and_does_not_erase_match(self): + e = engine(6, 3) + e.slot_cache_affinity = True + e.slot_live = [None] * 6 + e.slot_cv = threading.Condition() + e.slot_cache_valid = [True] * 6 + e.slot_cache_peer = ["192.0.2.8"] * 6 + e.slot_held[0] = [1] * 500 + e.slot_used[0] = time.time() - 1 + for _ in range(3): + row = e.slots_view()[0] + self.assertTrue(row["cache_expired"]) + self.assertEqual(row["cache_peer_ip"], "192.0.2.8") + self.assertEqual(row["held_tokens"], 500) + self.assertEqual([x["kind"] for x in e.cache_events_view()], ["expired"]) + self.assertIsNotNone(e.pick_slot([1] * 500 + [2])) + self.assertEqual(e.slot_last_plan.reused_hint, 500) def test_busy_view_keeps_actual_reuse_separate_from_hint_and_peer(self): e = engine(2, 1) From f175c58c6b4a76bce23490cad6b39b69466238bb Mon Sep 17 00:00:00 2001 From: andyhrc <12491949+andyhrc@users.noreply.github.com> Date: Thu, 15 Oct 2026 15:10:16 +0800 Subject: [PATCH 12/21] serve: a park may take an expired history worth less than STRATA_SLOT_EXPIRED_KEEP_SCORE Parking kept the main session's conversation only in its own older copy or a blank slot, so with every slot holding another conversation it was dropped even when one of them was a short expired side chat. The parked conversation is at least STRATA_SLOT_PARK_MIN_TOKENS (8192) long; an expired history below STRATA_SLOT_EXPIRED_KEEP_SCORE (about 2,600 tokens at the default 0.01) is worth less, and place() already overwrites those first. Such a slot is now the last choice for a park (least loaded group, then the lowest score); a history worth keeping is still never taken. Co-Authored-By: Claude Opus 5.5 --- serve/slot_policy.py | 15 +++++++++++---- serve/test_slot_alloc.py | 14 ++++++++++++++ 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/serve/slot_policy.py b/serve/slot_policy.py index c38384f6b0b..60bd194d403 100644 --- a/serve/slot_policy.py +++ b/serve/slot_policy.py @@ -113,8 +113,9 @@ def park_destination(main: list[int], busy: list[bool], held: list[list[int]], u order: list[int], group_size: int, now: float, valid: list[bool] | None = None, padding_safe: bool = False, exclude=()) -> int | None: """An idle slot to keep the main session's conversation in before another request replaces it (engine park=): - its own older copy, else a blank slot of the least loaded group. Never another conversation's history - parking - must not evict anything. None: nowhere to keep it (or it is already kept).""" + its own older copy, else a blank slot of the least loaded group, else an expired history worth less than + EXPIRED_KEEP_SCORE (the ones place() overwrites first). Never a history worth keeping. None: nowhere to keep it + (or it is already kept).""" if not padding_safe or not main: return None gs = max(1, group_size) @@ -122,7 +123,7 @@ def park_destination(main: list[int], busy: list[bool], held: list[list[int]], u for b, h in enumerate(held): # already kept (all but a short tail) - in an idle slot or one decoding it if h and len(h) * 10 >= len(main) * 9 and (main[:len(h)] == h or h[:len(main)] == main): return None - _, blank = _blank(held, busy, used, now, reliable) + states, blank = _blank(held, busy, used, now, reliable) positions = {b: i for i, b in enumerate(order)} loads = {g: sum(busy[g:g + gs]) for g in range(0, len(busy), gs)} own = [b for b in order if not busy[b] and b not in exclude and held[b] and reliable[b] and @@ -130,7 +131,13 @@ def park_destination(main: list[int], busy: list[bool], held: list[list[int]], u if own: return max(own, key=lambda b: (len(held[b]), -positions[b])) free = [b for b in order if not busy[b] and b not in exclude and b in blank] - return min(free, key=lambda b: (loads[b - b % gs], positions[b])) if free else None + if free: + return min(free, key=lambda b: (loads[b - b % gs], positions[b])) + # The main session's conversation is at least PARK_MIN_TOKENS long; an expired one below EXPIRED_KEEP_SCORE is + # a short side chat: worth less than what parking keeps. The least loaded group, then the lowest score. + worthless = [b for b in order if not busy[b] and b not in exclude and states[b]["expired"] and + stale_retention(len(held[b])) < EXPIRED_KEEP_SCORE] + return min(worthless, key=lambda b: (loads[b - b % gs], stale_retention(len(held[b])), positions[b])) if worthless else None def place(prompt: list[int], busy: list[bool], held: list[list[int]], used: list[float], diff --git a/serve/test_slot_alloc.py b/serve/test_slot_alloc.py index 6292321fd09..9ee4d9e2243 100644 --- a/serve/test_slot_alloc.py +++ b/serve/test_slot_alloc.py @@ -529,6 +529,20 @@ def test_nowhere_to_park_without_evicting(self): self.assertEqual(e._claim_park([1] * 60), "") self.assertFalse(any(e.slot_busy)) + def test_an_expired_history_worth_less_than_the_keep_score_takes_the_park(self): + e = self.e() + e.slot_held = [[b] * 40000 for b in range(6)] # every slot holds another conversation + e.slot_used = [self.NOW - 3600] * 6 # all expired + self.assertIsNone(self.dest(e)) # each still worth keeping: nowhere to park + e.slot_held[1] = [1] * 1000 # a short side chat, below EXPIRED_KEEP_SCORE + e.slot_held[4] = [4] * 2000 + e.slot_busy[0] = True + self.assertEqual(self.dest(e), 4) # the less loaded group (slot 1 shares one with busy 0) + e.slot_busy[0] = False + self.assertEqual(self.dest(e), 1) # same load: the lowest score + e.slot_used[1] = self.NOW # protected: not a park target + self.assertEqual(self.dest(e), 4) + def test_admission_slot_is_never_the_park_slot_and_claim_is_released(self): e = self.e() self.assertNotEqual(self.dest(e, exclude=(0,)), 0) From 868b6e9456136939885321b3dcdc01f3333119f5 Mon Sep 17 00:00:00 2001 From: andyhrc <12491949+andyhrc@users.noreply.github.com> Date: Thu, 15 Oct 2026 15:37:14 +0800 Subject: [PATCH 13/21] serve: decay expired slot value with time since protection ended Keep the value at expiry, then halve it every STRATA_SLOT_EXPIRED_HALF_LIFE_S seconds (default 120; 0 disables decay). Allocation and main-session parking use the same age-aware value, so a long unused history can eventually yield below the keep threshold. Protection and prefix matching are unchanged. Verify expiry continuity and half-life, long-history reclamation and refresh, matching after decay, bounded waits, busy/excluded park targets and configuration. 97 policy/cache/parallel CPU and mock-engine tests pass; no GPU performance claim. --- serve/slot_policy.py | 32 +++++++++++------ serve/test_slot_alloc.py | 78 +++++++++++++++++++++++++++++++++++++--- 2 files changed, 95 insertions(+), 15 deletions(-) diff --git a/serve/slot_policy.py b/serve/slot_policy.py index 60bd194d403..463a9007bfb 100644 --- a/serve/slot_policy.py +++ b/serve/slot_policy.py @@ -10,7 +10,8 @@ PROTECTION_WAIT_S = 5.0 # STRATA_SLOT_PROTECT_WAIT_S: wait when every idle history is still protected PARK_MIN_TOKENS = 8192 # STRATA_SLOT_PARK_MIN_TOKENS: a shorter main-session conversation is not parked EXPIRED_KEEP_SCORE = 0.01 # STRATA_SLOT_EXPIRED_KEEP_SCORE: an expired history scoring at least this (retention at - # freshness 0; 0.01 is about 2.5K tokens) is overwritten only when nothing else is free + # expiry, then decayed; 0.01 is about 2.5K tokens at expiry) goes last +EXPIRED_HALF_LIFE_S = 120.0 # STRATA_SLOT_EXPIRED_HALF_LIFE_S: seconds after expiry to halve its value; 0 disables decay TTL_MAX_S = 120.0 # STRATA_SLOT_TTL_MAX_S: protection at 262144 tokens; the whole curve scales with it _TTL_SHAPE = ((0, 0.0), (500, 0.1), (2000, 2.0), (8000, 10.0), (32000, 45.0), (50000, 65.0), (100000, 90.0), (MAX_CACHE_TOKENS, 120.0)) @@ -19,7 +20,7 @@ def configure(env) -> dict: """Read the tunables from `env` (the engine's environment); bad or missing values keep the defaults.""" - global PROTECTION_WAIT_S, PARK_MIN_TOKENS, TTL_MAX_S, TTL_POINTS, EXPIRED_KEEP_SCORE + global PROTECTION_WAIT_S, PARK_MIN_TOKENS, TTL_MAX_S, TTL_POINTS, EXPIRED_KEEP_SCORE, EXPIRED_HALF_LIFE_S def value(name, default): try: @@ -29,6 +30,7 @@ def value(name, default): return default TTL_MAX_S = value("STRATA_SLOT_TTL_MAX_S", 120.0) EXPIRED_KEEP_SCORE = value("STRATA_SLOT_EXPIRED_KEEP_SCORE", 0.01) + EXPIRED_HALF_LIFE_S = value("STRATA_SLOT_EXPIRED_HALF_LIFE_S", 120.0) TTL_POINTS = tuple((n, s * TTL_MAX_S / 120.0) for n, s in _TTL_SHAPE) PROTECTION_WAIT_S = value("STRATA_SLOT_PROTECT_WAIT_S", 5.0) PARK_MIN_TOKENS = int(value("STRATA_SLOT_PARK_MIN_TOKENS", 8192)) @@ -37,7 +39,8 @@ def value(name, default): def settings() -> dict: return {"ttl_max_s": TTL_MAX_S, "protect_wait_s": PROTECTION_WAIT_S, - "park_min_tokens": PARK_MIN_TOKENS, "expired_keep_score": EXPIRED_KEEP_SCORE} + "park_min_tokens": PARK_MIN_TOKENS, "expired_keep_score": EXPIRED_KEEP_SCORE, + "expired_half_life_s": EXPIRED_HALF_LIFE_S} def cache_ttl(tokens: int) -> float: @@ -48,10 +51,16 @@ def cache_ttl(tokens: int) -> float: return TTL_POINTS[-1][1] -def stale_retention(tokens: int) -> float: - """The retention score at freshness 0 (the same formula): orders expired histories - lowest is reclaimed first.""" +def stale_retention(tokens: int, age_s: float = 0.0) -> float: + """Value at expiry, halved for each EXPIRED_HALF_LIFE_S spent past the TTL. Age is time since last use. + + Before expiry this is the projected value at expiry; a zero half-life keeps the old length-only value. + Expiry changes replacement priority, never whether a history can match. + """ n = min(MAX_CACHE_TOKENS, max(0, tokens)) - return (n / 50000.0) ** 0.7 * (n / (n + 2000.0)) ** 2 * 0.25 + base = (n / 50000.0) ** 0.7 * (n / (n + 2000.0)) ** 2 * 0.25 + overdue = max(0.0, age_s - cache_ttl(tokens)) + return base * 2.0 ** (-overdue / EXPIRED_HALF_LIFE_S) if EXPIRED_HALF_LIFE_S > 0 else base def cache_state(tokens: int, last_used: float, now: float, valid: bool = True) -> dict: @@ -133,11 +142,12 @@ def park_destination(main: list[int], busy: list[bool], held: list[list[int]], u free = [b for b in order if not busy[b] and b not in exclude and b in blank] if free: return min(free, key=lambda b: (loads[b - b % gs], positions[b])) - # The main session's conversation is at least PARK_MIN_TOKENS long; an expired one below EXPIRED_KEEP_SCORE is - # a short side chat: worth less than what parking keeps. The least loaded group, then the lowest score. + # The main session is at least PARK_MIN_TOKENS long; give up a short side chat or a long history left unused + # until its value decays below EXPIRED_KEEP_SCORE. The least loaded group, then the lowest current score. + scores = {b: stale_retention(len(held[b]), states[b]["age_s"]) for b in order} worthless = [b for b in order if not busy[b] and b not in exclude and states[b]["expired"] and - stale_retention(len(held[b])) < EXPIRED_KEEP_SCORE] - return min(worthless, key=lambda b: (loads[b - b % gs], stale_retention(len(held[b])), positions[b])) if worthless else None + scores[b] < EXPIRED_KEEP_SCORE] + return min(worthless, key=lambda b: (loads[b - b % gs], scores[b], positions[b])) if worthless else None def place(prompt: list[int], busy: list[bool], held: list[list[int]], used: list[float], @@ -193,7 +203,7 @@ def pipeline(b): safe = [b for b in free if states[b]["expired"] and collateral(b) == 0] def stale(b): - return 0.0 if b in blank else stale_retention(len(held[b])) + return 0.0 if b in blank else stale_retention(len(held[b]), states[b]["age_s"]) def worthy(b): return stale(b) >= EXPIRED_KEEP_SCORE and stale(b) > 0 diff --git a/serve/test_slot_alloc.py b/serve/test_slot_alloc.py index 9ee4d9e2243..ddd221bd725 100644 --- a/serve/test_slot_alloc.py +++ b/serve/test_slot_alloc.py @@ -246,6 +246,46 @@ def test_fresh_medium_outweighs_nearly_expired_long_history(self): b = slot_policy.cache_state(150000, self.NOW - 97, self.NOW) self.assertGreater(a['retention'], b['retention']) + def test_expired_value_is_continuous_then_halves_without_changing_protection(self): + ttl = slot_policy.cache_ttl(50000) + base = slot_policy.stale_retention(50000) + self.assertAlmostEqual(base, slot_policy.cache_state(50000, 0, ttl - 1e-8)['retention']) + for age in (0, ttl / 2, ttl): + self.assertEqual(slot_policy.stale_retention(50000, age), base) + self.assertAlmostEqual(slot_policy.stale_retention(50000, ttl + 120), base / 2) + self.assertAlmostEqual(slot_policy.stale_retention(50000, ttl + 240), base / 4) + self.assertEqual(slot_policy.cache_ttl(50000), ttl) + self.assertEqual(slot_policy.stale_retention(50000, 1e12), 0) + self.assertEqual(slot_policy.stale_retention(0, 1e12), 0) + + def test_old_long_history_yields_before_a_recent_shorter_one_even_in_busier_group(self): + e = self.e() + e.slot_held = [[b] * 50000 for b in range(6)] + e.slot_busy[1] = True + e.slot_held[0], e.slot_used[0] = [0] * 150000, self.NOW - 1800 + e.slot_held[4], e.slot_used[4] = [4] * 8000, self.NOW - 15 + self.assertEqual(self.place(e, [9999], padding_safe=True, allow_eviction=False).slot, 0) + # A use refreshes the age: the long history is protected again and the expired shorter one yields. + e.slot_used[0] = self.NOW + self.assertEqual(self.place(e, [9999], padding_safe=True, allow_eviction=False).slot, 4) + + def test_long_unused_history_still_matches_after_its_value_falls_below_keep_score(self): + e = self.e() + e.slot_held = [[b] * 50000 for b in range(6)] + e.slot_held[2] = list(range(100000)) + e.slot_used[2] = self.NOW - 3600 + plan = self.place(e, list(range(100001)), padding_safe=True, allow_eviction=False) + self.assertEqual((plan.slot, plan.source_hint, plan.reused_hint), (2, 2, 100000)) + self.assertEqual(len(e.slot_held[2]), 100000) + + def test_decayed_history_does_not_wait_for_a_cheaper_protected_side_request(self): + e = self.e() + e.slot_held = [[b] * 50000 for b in range(6)] + e.slot_used[0] = self.NOW - 1800 + e.slot_held[2], e.slot_used[2] = [2] * 1500, self.NOW - 0.5 + plan = self.place(e, [9999], padding_safe=True, allow_eviction=False) + self.assertEqual((plan.slot, plan.reason), (0, 'reclaim')) + def test_unexpired_fifty_k_is_selected_before_similarly_fresh_150k(self): e = self.e() e.slot_held = [[b] * (50000 if b == 2 else 150000) for b in range(6)] @@ -334,7 +374,7 @@ def test_reclaim_overwrites_expired_history_only_after_blank_slots(self): def test_protected_history_is_kept_and_expired_ones_go_lowest_score_first(self): # Under the TTL rules an expired history is reclaimable and a protected one is not, whatever their lengths; - # among expired ones the retention formula (freshness 0) decides. + # among expired ones the retention at expiry, decayed by time past expiry, decides. e = self.e() e.slot_busy[1] = e.slot_busy[4] = e.slot_busy[5] = True e.slot_held[0], e.slot_used[0] = [0] * 37365, self.NOW - 300 # expired @@ -348,7 +388,7 @@ def test_new_chat_takes_a_worthless_slot_in_a_busier_group_before_an_expired_con # slot 1 on group load and the two overwrote each other every turn. Old 1.5K filler histories were free. e = self.e() e.slot_busy[2] = e.slot_busy[4] = True - e.slot_held[0], e.slot_used[0] = [0] * 37365, self.NOW - 300 # parked A, expired + e.slot_held[0], e.slot_used[0] = [0] * 37365, self.NOW - 100 # parked A, expired e.slot_held[1], e.slot_used[1] = [1] * 8970, self.NOW - 20 # B, expired e.slot_held[3], e.slot_used[3] = [3] * 1526, self.NOW - 30 # filler leftovers, expired e.slot_held[5], e.slot_used[5] = [5] * 1526, self.NOW - 30 @@ -459,6 +499,11 @@ class ParkMainSession(unittest.TestCase): """Engine park=: before a side request replaces the main (solo) session, its conversation is copied into a slot.""" NOW = 100000.0 + def setUp(self): + clock = mock.patch('serve.server.time.time', return_value=self.NOW) + clock.start() + self.addCleanup(clock.stop) + def e(self): e = engine(6, 3) e.slot_cache_affinity = True @@ -532,7 +577,7 @@ def test_nowhere_to_park_without_evicting(self): def test_an_expired_history_worth_less_than_the_keep_score_takes_the_park(self): e = self.e() e.slot_held = [[b] * 40000 for b in range(6)] # every slot holds another conversation - e.slot_used = [self.NOW - 3600] * 6 # all expired + e.slot_used = [self.NOW - 100] * 6 # all expired, still worth keeping self.assertIsNone(self.dest(e)) # each still worth keeping: nowhere to park e.slot_held[1] = [1] * 1000 # a short side chat, below EXPIRED_KEEP_SCORE e.slot_held[4] = [4] * 2000 @@ -543,6 +588,20 @@ def test_an_expired_history_worth_less_than_the_keep_score_takes_the_park(self): e.slot_used[1] = self.NOW # protected: not a park target self.assertEqual(self.dest(e), 4) + def test_a_long_unused_history_can_take_a_park_but_busy_and_excluded_slots_cannot(self): + e = self.e() + e.slot_held = [[b] * 40000 for b in range(6)] + e.slot_used = [self.NOW - 100] * 6 + self.assertIsNone(self.dest(e)) + e.slot_used[4] = self.NOW - 1800 + self.assertEqual(self.dest(e), 4) + self.assertIsNone(self.dest(e, exclude=(4,))) + e.slot_busy[4] = True + self.assertIsNone(self.dest(e)) + e.slot_busy[4] = False + e.slot_used[4] = self.NOW + self.assertIsNone(self.dest(e)) + def test_admission_slot_is_never_the_park_slot_and_claim_is_released(self): e = self.e() self.assertNotEqual(self.dest(e, exclude=(0,)), 0) @@ -569,7 +628,7 @@ def test_environment_overrides_and_bad_values_keep_defaults(self): got = slot_policy.configure({"STRATA_SLOT_TTL_MAX_S": "240", "STRATA_SLOT_PROTECT_WAIT_S": "2", "STRATA_SLOT_PARK_MIN_TOKENS": "x"}) self.assertEqual(got, {"ttl_max_s": 240.0, "protect_wait_s": 2.0, "park_min_tokens": 8192, - "expired_keep_score": 0.01}) + "expired_keep_score": 0.01, "expired_half_life_s": 120.0}) self.assertAlmostEqual(slot_policy.cache_ttl(100000), 180.0) # the whole curve doubles self.assertAlmostEqual(slot_policy.cache_ttl(10 ** 6), 240.0) self.assertEqual(slot_policy.configure({})["ttl_max_s"], 120.0) @@ -579,6 +638,17 @@ def test_engine_reads_them_from_its_environment(self): e = StrataEngine("missing", [], env={"STRATA_SLOT_PARK_MIN_TOKENS": "20000"}, lazy=True) self.assertEqual(e.slot_policy_settings["park_min_tokens"], 20000) + def test_expired_half_life_override_disable_and_invalid_values(self): + ttl = slot_policy.cache_ttl(50000) + base = slot_policy.stale_retention(50000) + slot_policy.configure({"STRATA_SLOT_EXPIRED_HALF_LIFE_S": "30"}) + self.assertAlmostEqual(slot_policy.stale_retention(50000, ttl + 30), base / 2) + slot_policy.configure({"STRATA_SLOT_EXPIRED_HALF_LIFE_S": "0"}) + self.assertEqual(slot_policy.stale_retention(50000, ttl + 3600), base) + for bad in ("nan", "inf", "-1", "bad"): + got = slot_policy.configure({"STRATA_SLOT_EXPIRED_HALF_LIFE_S": bad}) + self.assertEqual(got["expired_half_life_s"], 120) + class NativePaddingSafety(unittest.TestCase): def test_padding_safe_engine_keeps_lower_match_available(self): From f4158db54949a408545b9c4ede57b99231e8c4cc Mon Sep 17 00:00:00 2001 From: andyhrc <12491949+andyhrc@users.noreply.github.com> Date: Thu, 15 Oct 2026 16:23:54 +0800 Subject: [PATCH 14/21] serve: accelerate idle value decay and skip low-value main parks Use value_at_expiry * (1 - x*x), where x runs from protection expiry to a total idle age of STRATA_SLOT_IDLE_VALUE_ZERO_S (default 1800 seconds, including protection; 0 disables decay). Loss accelerates until value reaches zero. Protection takes precedence if a configured cutoff is shorter than TTL. Record the main session last actual use, check its current value before claiming a park, and copy its timestamp into the parked slot. Score zero does not invalidate history or exclude prefix matches. Existing cache affinity remains opt-in; no native code changed. Verified 103 CPU/mock policy/cache/parallel tests plus a subsequently added main-continuation regression (104 distinct tests). No GPU benchmark, model restart or live activation. Local review only; not pushed. --- serve/server.py | 14 ++++++- serve/slot_policy.py | 24 +++++++---- serve/test_parallel.py | 9 ++++ serve/test_slot_alloc.py | 88 +++++++++++++++++++++++++++++++++++----- 4 files changed, 114 insertions(+), 21 deletions(-) diff --git a/serve/server.py b/serve/server.py index f400515297f..2bd49657531 100644 --- a/serve/server.py +++ b/serve/server.py @@ -1182,6 +1182,7 @@ def __init__(self, exe: str, args: list[str], cwd: str | None = None, log: str | # park= copy of it until the engine says PARKED self.main_held: list[int] = [] self.main_peer = None + self.main_used = 0.0 # last actual use; a park copies this age instead of refreshing it self._park_claims: set[int] = set() self.slot_cache_peer = [None] * self.batch self._cache_expiry_seen = [None] * self.batch @@ -2033,6 +2034,7 @@ def give_way(): # at a PP line, and between two far apart (the engine takes it park_slot = None # the main session now holds this leg's conversation (a read that gave way did not finish) self.main_held = [] if self._yielded is not None else solo_leg[0] + out[solo_leg[1]:] + self.main_used = time.time() self.main_peer, solo_leg = peer_ip, None if self._done_finish() == "nonfinite": # #879 (never promoted to a slot) raise EngineNonFinite("the engine's logits became non-finite (#879)") @@ -2321,6 +2323,7 @@ def note_progress(): # the consumer stopped mid-leg (usually right after the last token): the main session holds # the leg all the same; a read that gave way (YIELDED) went into its slot instead self.main_held = [] if self._yielded is not None else solo_leg[0] + out[solo_leg[1]:] + self.main_used = time.time() self.main_peer = peer_ip if holding: self._solo_ended() @@ -2377,15 +2380,22 @@ def _claim_park(self, prompt, exclude=()) -> str: turn = next((i for i in range(len(main) - 1, 0, -1) if main[i] == turn_token), len(main)) if prompt[:turn] == main[:turn]: return "" # nothing held, or the same conversation: nothing to keep + now = time.time() + main_used = getattr(self, "main_used", 0.0) + value = slot_policy.stale_retention(len(main), max(0.0, now - main_used)) + if value <= 0 or value < slot_policy.EXPIRED_KEEP_SCORE: + self.cache_event("park_skip", reason="main_low_value", held_tokens=len(main), value=value) + return "" # an unused main history need not be copied into a slot just to be reclaimed with self.slot_cv: b = slot_policy.park_destination(main, self.slot_busy, self.slot_held, self.slot_used, self.slot_order, - self.slot_gs, time.time(), getattr(self, "slot_cache_valid", None), + self.slot_gs, now, getattr(self, "slot_cache_valid", None), self.slot_padding_safe, exclude) if b is not None: self.slot_busy[b] = True self.slot_claims.add(b) self._park_claims.add(b) self._park_main, self._park_peer = list(main), getattr(self, "main_peer", None) + self._park_used = main_used # outside slot_cv: cache_event takes its own lock self.cache_event("park_claim" if b is not None else "park_skip", slot=b, held_tokens=len(main)) return f" park={b}" if b is not None else "" @@ -2405,7 +2415,7 @@ def _on_parked(self, b: int, n: int) -> None: with self.slot_cv: if 0 <= b < self.batch: self.slot_held[b] = list(main[:n]) if n > 0 else [] # the engine may hold a few more tail tokens - self.slot_used[b] = time.time() + self.slot_used[b] = getattr(self, "_park_used", 0.0) if hasattr(self, "slot_cache_valid"): self.slot_cache_valid[b] = n > 0 if hasattr(self, "slot_cache_peer"): diff --git a/serve/slot_policy.py b/serve/slot_policy.py index 463a9007bfb..919569de50a 100644 --- a/serve/slot_policy.py +++ b/serve/slot_policy.py @@ -11,7 +11,7 @@ PARK_MIN_TOKENS = 8192 # STRATA_SLOT_PARK_MIN_TOKENS: a shorter main-session conversation is not parked EXPIRED_KEEP_SCORE = 0.01 # STRATA_SLOT_EXPIRED_KEEP_SCORE: an expired history scoring at least this (retention at # expiry, then decayed; 0.01 is about 2.5K tokens at expiry) goes last -EXPIRED_HALF_LIFE_S = 120.0 # STRATA_SLOT_EXPIRED_HALF_LIFE_S: seconds after expiry to halve its value; 0 disables decay +IDLE_VALUE_ZERO_S = 1800.0 # STRATA_SLOT_IDLE_VALUE_ZERO_S: idle age when expired value reaches zero; 0 disables decay TTL_MAX_S = 120.0 # STRATA_SLOT_TTL_MAX_S: protection at 262144 tokens; the whole curve scales with it _TTL_SHAPE = ((0, 0.0), (500, 0.1), (2000, 2.0), (8000, 10.0), (32000, 45.0), (50000, 65.0), (100000, 90.0), (MAX_CACHE_TOKENS, 120.0)) @@ -20,7 +20,7 @@ def configure(env) -> dict: """Read the tunables from `env` (the engine's environment); bad or missing values keep the defaults.""" - global PROTECTION_WAIT_S, PARK_MIN_TOKENS, TTL_MAX_S, TTL_POINTS, EXPIRED_KEEP_SCORE, EXPIRED_HALF_LIFE_S + global PROTECTION_WAIT_S, PARK_MIN_TOKENS, TTL_MAX_S, TTL_POINTS, EXPIRED_KEEP_SCORE, IDLE_VALUE_ZERO_S def value(name, default): try: @@ -30,7 +30,7 @@ def value(name, default): return default TTL_MAX_S = value("STRATA_SLOT_TTL_MAX_S", 120.0) EXPIRED_KEEP_SCORE = value("STRATA_SLOT_EXPIRED_KEEP_SCORE", 0.01) - EXPIRED_HALF_LIFE_S = value("STRATA_SLOT_EXPIRED_HALF_LIFE_S", 120.0) + IDLE_VALUE_ZERO_S = value("STRATA_SLOT_IDLE_VALUE_ZERO_S", 1800.0) TTL_POINTS = tuple((n, s * TTL_MAX_S / 120.0) for n, s in _TTL_SHAPE) PROTECTION_WAIT_S = value("STRATA_SLOT_PROTECT_WAIT_S", 5.0) PARK_MIN_TOKENS = int(value("STRATA_SLOT_PARK_MIN_TOKENS", 8192)) @@ -40,7 +40,7 @@ def value(name, default): def settings() -> dict: return {"ttl_max_s": TTL_MAX_S, "protect_wait_s": PROTECTION_WAIT_S, "park_min_tokens": PARK_MIN_TOKENS, "expired_keep_score": EXPIRED_KEEP_SCORE, - "expired_half_life_s": EXPIRED_HALF_LIFE_S} + "idle_value_zero_s": IDLE_VALUE_ZERO_S} def cache_ttl(tokens: int) -> float: @@ -52,15 +52,21 @@ def cache_ttl(tokens: int) -> float: def stale_retention(tokens: int, age_s: float = 0.0) -> float: - """Value at expiry, halved for each EXPIRED_HALF_LIFE_S spent past the TTL. Age is time since last use. + """Value at expiry times (1 - x*x), x = (age - TTL) / (IDLE_VALUE_ZERO_S - TTL), clamped to [0, 1]. - Before expiry this is the projected value at expiry; a zero half-life keeps the old length-only value. - Expiry changes replacement priority, never whether a history can match. + Age is time since last actual use, including protection. Loss per second increases until value reaches zero. + Before expiry this is the projected value at expiry; zero disables decay. Protection takes precedence if a + configured zero age is shorter than the TTL. Replacement priority never changes whether a history can match. """ n = min(MAX_CACHE_TOKENS, max(0, tokens)) base = (n / 50000.0) ** 0.7 * (n / (n + 2000.0)) ** 2 * 0.25 - overdue = max(0.0, age_s - cache_ttl(tokens)) - return base * 2.0 ** (-overdue / EXPIRED_HALF_LIFE_S) if EXPIRED_HALF_LIFE_S > 0 else base + ttl = cache_ttl(tokens) + if IDLE_VALUE_ZERO_S == 0 or age_s < ttl: + return base + if age_s >= IDLE_VALUE_ZERO_S: + return 0.0 + x = max(0.0, (age_s - ttl) / (IDLE_VALUE_ZERO_S - ttl)) + return base * (1.0 - x * x) def cache_state(tokens: int, last_used: float, now: float, valid: bool = True) -> dict: diff --git a/serve/test_parallel.py b/serve/test_parallel.py index 9ed4348ffa9..bcc711ffece 100644 --- a/serve/test_parallel.py +++ b/serve/test_parallel.py @@ -511,6 +511,15 @@ def test_solo_native_source_is_forwarded_without_reserving_batch_slot(self): self.assertEqual([live[k] for k in ("solo_peer_ip", "solo_request_key", "solo_cache_source_slot", "solo_cache_reused_tokens")], [None] * 4) + def test_solo_completion_records_actual_last_use_of_main_cache(self): + self.start(2, says_groups=2, affinity=True) + self.engine.main_used = time.time() - 3600 + before = time.time() + self.chat('hello', max_tokens=8) + self.assertTrue(self.engine.main_held) + self.assertGreaterEqual(self.engine.main_used, before) + self.assertLessEqual(self.engine.main_used, time.time()) + def test_http_peer_is_recorded_without_trusting_forwarded_ip(self): self.start(2, says_groups=2, affinity=True) body = {"messages": [{"role": "user", "content": "hello"}], diff --git a/serve/test_slot_alloc.py b/serve/test_slot_alloc.py index ddd221bd725..4da88eea39f 100644 --- a/serve/test_slot_alloc.py +++ b/serve/test_slot_alloc.py @@ -246,18 +246,32 @@ def test_fresh_medium_outweighs_nearly_expired_long_history(self): b = slot_policy.cache_state(150000, self.NOW - 97, self.NOW) self.assertGreater(a['retention'], b['retention']) - def test_expired_value_is_continuous_then_halves_without_changing_protection(self): + def test_expired_value_loses_more_each_interval_and_reaches_zero_at_total_idle_limit(self): ttl = slot_policy.cache_ttl(50000) base = slot_policy.stale_retention(50000) self.assertAlmostEqual(base, slot_policy.cache_state(50000, 0, ttl - 1e-8)['retention']) for age in (0, ttl / 2, ttl): self.assertEqual(slot_policy.stale_retention(50000, age), base) - self.assertAlmostEqual(slot_policy.stale_retention(50000, ttl + 120), base / 2) - self.assertAlmostEqual(slot_policy.stale_retention(50000, ttl + 240), base / 4) + values = [slot_policy.stale_retention(50000, ttl + (1800 - ttl) * i / 4) for i in range(5)] + self.assertAlmostEqual(values[2], base * 0.75) + losses = [values[i] - values[i + 1] for i in range(4)] + self.assertTrue(all(losses[i] < losses[i + 1] for i in range(3))) + self.assertEqual(values[-1], 0) self.assertEqual(slot_policy.cache_ttl(50000), ttl) self.assertEqual(slot_policy.stale_retention(50000, 1e12), 0) self.assertEqual(slot_policy.stale_retention(0, 1e12), 0) + def test_approved_lengths_cross_the_keep_threshold_before_thirty_minutes(self): + for n, crossing in ((8000, 1585), (16000, 1697), (32000, 1745), (50000, 1762), + (100000, 1778), (200000, 1787), (262144, 1789)): + with self.subTest(tokens=n): + self.assertGreater(slot_policy.stale_retention(n, crossing - 1), 0.01) + self.assertLess(slot_policy.stale_retention(n, crossing + 1), 0.01) + self.assertEqual(slot_policy.stale_retention(n, 1800), 0) + self.assertEqual(slot_policy.stale_retention(n, 1801), 0) + for n in (500, 2000): + self.assertLess(slot_policy.stale_retention(n), 0.01) + def test_old_long_history_yields_before_a_recent_shorter_one_even_in_busier_group(self): e = self.e() e.slot_held = [[b] * 50000 for b in range(6)] @@ -274,6 +288,7 @@ def test_long_unused_history_still_matches_after_its_value_falls_below_keep_scor e.slot_held = [[b] * 50000 for b in range(6)] e.slot_held[2] = list(range(100000)) e.slot_used[2] = self.NOW - 3600 + self.assertEqual(slot_policy.stale_retention(100000, self.NOW - e.slot_used[2]), 0) plan = self.place(e, list(range(100001)), padding_safe=True, allow_eviction=False) self.assertEqual((plan.slot, plan.source_hint, plan.reused_hint), (2, 2, 100000)) self.assertEqual(len(e.slot_held[2]), 100000) @@ -516,6 +531,7 @@ def e(self): e._park_claims, e.slot_claims = set(), set() e.main_held = [7] * 50000 e.main_peer = "192.0.2.9" + e.main_used = self.NOW return e def dest(self, e, exclude=()): @@ -562,6 +578,51 @@ def test_continuing_short_or_already_kept_conversations_are_not_parked(self): e.slot_held[3] = [7] * 48000 # a slot holds 96% of it already self.assertEqual(e._claim_park([1] * 60), "") + def test_parking_inherits_the_main_history_last_use_and_does_not_renew_its_value(self): + e = self.e() + e.main_used = self.NOW - 1700 + value = slot_policy.stale_retention(50000, 1700) + key = e._claim_park([1] * 60) + b = int(key.split('=')[1]) + # The main session may begin a new request before PARKED is processed; the snapshot owns this age. + e.main_used = self.NOW + e._on_parked(b, 49999) + self.assertEqual(e.slot_used[b], self.NOW - 1700) + self.assertAlmostEqual(slot_policy.stale_retention(len(e.slot_held[b]), self.NOW - e.slot_used[b]), value, + delta=0.00001) + e.slot_live = [None] * 6 + with mock.patch('serve.server.time.time', return_value=self.NOW + 30): + e.slots_view() + self.assertEqual(e.slot_used[b], self.NOW - 1700) # observation cannot renew the cache either + + def test_main_below_keep_threshold_is_not_parked_even_when_slots_are_blank(self): + e = self.e() + for age in (1770, 1800, 3600): + e.main_used = self.NOW - age + self.assertEqual(e._claim_park([1] * 60), '') + self.assertFalse(any(e.slot_busy)) + self.assertFalse(e._park_claims) + self.assertEqual(e.main_held, [7] * 50000) # a skipped park does not itself erase the main cache + self.assertEqual(e.cache_events_view()[0]['reason'], 'main_low_value') + + def test_main_above_keep_threshold_is_parked_with_its_old_age(self): + e = self.e() + e.main_used = self.NOW - 1760 + key = e._claim_park([1] * 60) + self.assertTrue(key.startswith(' park=')) + b = int(key.split('=')[1]) + e._on_parked(b, 49999) + self.assertEqual(e.slot_used[b], self.NOW - 1760) + + def test_zero_value_main_can_still_continue_without_a_park_or_deletion(self): + e = self.e() + e.main_used = self.NOW - 3600 + self.assertEqual(slot_policy.stale_retention(len(e.main_held), 3600), 0) + self.assertEqual(e._claim_park(e.main_held + [1, 2, 3]), '') + self.assertEqual(e.main_held, [7] * 50000) + self.assertFalse(e._park_claims) + self.assertEqual(e.cache_events_view(), []) # no low-value rejection of a continued conversation + def test_own_older_copy_is_refreshed_before_using_a_blank_slot(self): e = self.e() e.slot_held[5] = [7] * 30000 # this conversation, a few turns back @@ -628,7 +689,7 @@ def test_environment_overrides_and_bad_values_keep_defaults(self): got = slot_policy.configure({"STRATA_SLOT_TTL_MAX_S": "240", "STRATA_SLOT_PROTECT_WAIT_S": "2", "STRATA_SLOT_PARK_MIN_TOKENS": "x"}) self.assertEqual(got, {"ttl_max_s": 240.0, "protect_wait_s": 2.0, "park_min_tokens": 8192, - "expired_keep_score": 0.01, "expired_half_life_s": 120.0}) + "expired_keep_score": 0.01, "idle_value_zero_s": 1800.0}) self.assertAlmostEqual(slot_policy.cache_ttl(100000), 180.0) # the whole curve doubles self.assertAlmostEqual(slot_policy.cache_ttl(10 ** 6), 240.0) self.assertEqual(slot_policy.configure({})["ttl_max_s"], 120.0) @@ -638,16 +699,23 @@ def test_engine_reads_them_from_its_environment(self): e = StrataEngine("missing", [], env={"STRATA_SLOT_PARK_MIN_TOKENS": "20000"}, lazy=True) self.assertEqual(e.slot_policy_settings["park_min_tokens"], 20000) - def test_expired_half_life_override_disable_and_invalid_values(self): + def test_idle_limit_override_disable_and_invalid_values(self): ttl = slot_policy.cache_ttl(50000) base = slot_policy.stale_retention(50000) - slot_policy.configure({"STRATA_SLOT_EXPIRED_HALF_LIFE_S": "30"}) - self.assertAlmostEqual(slot_policy.stale_retention(50000, ttl + 30), base / 2) - slot_policy.configure({"STRATA_SLOT_EXPIRED_HALF_LIFE_S": "0"}) + slot_policy.configure({"STRATA_SLOT_IDLE_VALUE_ZERO_S": "3600"}) + self.assertEqual(slot_policy.stale_retention(50000, 3600), 0) + self.assertAlmostEqual(slot_policy.stale_retention(50000, (ttl + 3600) / 2), base * 0.75) + slot_policy.configure({"STRATA_SLOT_IDLE_VALUE_ZERO_S": "0"}) self.assertEqual(slot_policy.stale_retention(50000, ttl + 3600), base) for bad in ("nan", "inf", "-1", "bad"): - got = slot_policy.configure({"STRATA_SLOT_EXPIRED_HALF_LIFE_S": bad}) - self.assertEqual(got["expired_half_life_s"], 120) + got = slot_policy.configure({"STRATA_SLOT_IDLE_VALUE_ZERO_S": bad}) + self.assertEqual(got["idle_value_zero_s"], 1800) + + def test_a_configured_idle_limit_cannot_shorten_protection(self): + slot_policy.configure({"STRATA_SLOT_IDLE_VALUE_ZERO_S": "30"}) + base = slot_policy.stale_retention(50000) + self.assertEqual(slot_policy.stale_retention(50000, 64), base) + self.assertEqual(slot_policy.stale_retention(50000, 65), 0) class NativePaddingSafety(unittest.TestCase): From e6329f930d4c8542e57c34c455f27421676053a5 Mon Sep 17 00:00:00 2001 From: andyhrc <12491949+andyhrc@users.noreply.github.com> Date: Thu, 15 Oct 2026 16:50:01 +0800 Subject: [PATCH 15/21] serve: count only a valid copy of the main session's conversation as already kept park_destination skips the park when a slot holds 90% of the main session's conversation; it now also requires that copy to be reliable, as the own-older-copy choice below it already does. A defensive check: on the current code paths a slot that is marked invalid holds nothing (an admission, a non-finite result and a failed park clear it), and a park needs slot_padding_safe, so it is not triggered today. Co-Authored-By: Claude Opus 5.5 --- serve/slot_policy.py | 2 +- serve/test_slot_alloc.py | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/serve/slot_policy.py b/serve/slot_policy.py index 919569de50a..9662aa8c388 100644 --- a/serve/slot_policy.py +++ b/serve/slot_policy.py @@ -136,7 +136,7 @@ def park_destination(main: list[int], busy: list[bool], held: list[list[int]], u gs = max(1, group_size) reliable = reliable_flags(busy, valid, gs, padding_safe) for b, h in enumerate(held): # already kept (all but a short tail) - in an idle slot or one decoding it - if h and len(h) * 10 >= len(main) * 9 and (main[:len(h)] == h or h[:len(main)] == main): + if h and reliable[b] and len(h) * 10 >= len(main) * 9 and (main[:len(h)] == h or h[:len(main)] == main): return None states, blank = _blank(held, busy, used, now, reliable) positions = {b: i for i, b in enumerate(order)} diff --git a/serve/test_slot_alloc.py b/serve/test_slot_alloc.py index 4da88eea39f..7e2433a808d 100644 --- a/serve/test_slot_alloc.py +++ b/serve/test_slot_alloc.py @@ -623,6 +623,14 @@ def test_zero_value_main_can_still_continue_without_a_park_or_deletion(self): self.assertFalse(e._park_claims) self.assertEqual(e.cache_events_view(), []) # no low-value rejection of a continued conversation + def test_an_invalid_copy_does_not_count_as_already_kept(self): + e = self.e() + e.slot_held[3] = [7] * 48000 # 96% of the main conversation, but no longer usable + e.slot_cache_valid[3] = False + self.assertIsNotNone(self.dest(e)) + e.slot_cache_valid[3] = True + self.assertIsNone(self.dest(e)) # a usable copy: nothing to park + def test_own_older_copy_is_refreshed_before_using_a_blank_slot(self): e = self.e() e.slot_held[5] = [7] * 30000 # this conversation, a few turns back From e101cc75f7b55787f9a20c6e28d6779a4319a073 Mon Sep 17 00:00:00 2001 From: andyhrc <12491949+andyhrc@users.noreply.github.com> Date: Thu, 15 Oct 2026 18:29:33 +0800 Subject: [PATCH 16/21] serve: a park uses place()'s reclaim order, and takes a history worth keeping at STRATA_SLOT_PARK_VALUE_RATIO place() and park_destination() now share one reclaim order (reclaim_slot) over idle expired slots: the slots not worth keeping (blank, or below STRATA_SLOT_EXPIRED_KEEP_SCORE) before histories worth keeping; then the group's load; then the lowest current value, a blank slot first at an equal value; then the slot order. place() is unchanged: its key had the blank-slot test before the value, and a blank slot's value is 0, the lowest (200,000 random placements compared, none differ). A park changes in two ways: - a blank slot no longer comes first whatever its group's load: a slot not worth keeping in a less loaded group is taken first, as place() does; - an expired history worth keeping may take the park when the main session's conversation is worth at least STRATA_SLOT_PARK_VALUE_RATIO (default 2) times its current value; 0 keeps every such history. A park still gives up rather than take a protected slot or one below the ratio. The server passes the main session's current value (the one its pre-park check already computes). Co-Authored-By: Claude Opus 5.5 --- serve/server.py | 2 +- serve/slot_policy.py | 57 +++++++++++++++++++++++++++------------- serve/test_slot_alloc.py | 36 ++++++++++++++++++++++--- 3 files changed, 73 insertions(+), 22 deletions(-) diff --git a/serve/server.py b/serve/server.py index 2bd49657531..463e7f78751 100644 --- a/serve/server.py +++ b/serve/server.py @@ -2389,7 +2389,7 @@ def _claim_park(self, prompt, exclude=()) -> str: with self.slot_cv: b = slot_policy.park_destination(main, self.slot_busy, self.slot_held, self.slot_used, self.slot_order, self.slot_gs, now, getattr(self, "slot_cache_valid", None), - self.slot_padding_safe, exclude) + self.slot_padding_safe, exclude, main_value=value) if b is not None: self.slot_busy[b] = True self.slot_claims.add(b) diff --git a/serve/slot_policy.py b/serve/slot_policy.py index 9662aa8c388..f06c31a3a73 100644 --- a/serve/slot_policy.py +++ b/serve/slot_policy.py @@ -11,6 +11,8 @@ PARK_MIN_TOKENS = 8192 # STRATA_SLOT_PARK_MIN_TOKENS: a shorter main-session conversation is not parked EXPIRED_KEEP_SCORE = 0.01 # STRATA_SLOT_EXPIRED_KEEP_SCORE: an expired history scoring at least this (retention at # expiry, then decayed; 0.01 is about 2.5K tokens at expiry) goes last +PARK_VALUE_RATIO = 2.0 # STRATA_SLOT_PARK_VALUE_RATIO: a park takes an expired history worth keeping only when the + # main session's conversation is worth at least this many times more; 0: never IDLE_VALUE_ZERO_S = 1800.0 # STRATA_SLOT_IDLE_VALUE_ZERO_S: idle age when expired value reaches zero; 0 disables decay TTL_MAX_S = 120.0 # STRATA_SLOT_TTL_MAX_S: protection at 262144 tokens; the whole curve scales with it _TTL_SHAPE = ((0, 0.0), (500, 0.1), (2000, 2.0), (8000, 10.0), @@ -20,7 +22,8 @@ def configure(env) -> dict: """Read the tunables from `env` (the engine's environment); bad or missing values keep the defaults.""" - global PROTECTION_WAIT_S, PARK_MIN_TOKENS, TTL_MAX_S, TTL_POINTS, EXPIRED_KEEP_SCORE, IDLE_VALUE_ZERO_S + global PROTECTION_WAIT_S, PARK_MIN_TOKENS, TTL_MAX_S, TTL_POINTS, EXPIRED_KEEP_SCORE, IDLE_VALUE_ZERO_S, \ + PARK_VALUE_RATIO def value(name, default): try: @@ -34,13 +37,14 @@ def value(name, default): TTL_POINTS = tuple((n, s * TTL_MAX_S / 120.0) for n, s in _TTL_SHAPE) PROTECTION_WAIT_S = value("STRATA_SLOT_PROTECT_WAIT_S", 5.0) PARK_MIN_TOKENS = int(value("STRATA_SLOT_PARK_MIN_TOKENS", 8192)) + PARK_VALUE_RATIO = value("STRATA_SLOT_PARK_VALUE_RATIO", 2.0) return settings() def settings() -> dict: return {"ttl_max_s": TTL_MAX_S, "protect_wait_s": PROTECTION_WAIT_S, "park_min_tokens": PARK_MIN_TOKENS, "expired_keep_score": EXPIRED_KEEP_SCORE, - "idle_value_zero_s": IDLE_VALUE_ZERO_S} + "idle_value_zero_s": IDLE_VALUE_ZERO_S, "park_value_ratio": PARK_VALUE_RATIO} def cache_ttl(tokens: int) -> float: @@ -124,13 +128,32 @@ def _blank(held, busy, used, now, reliable): return states, blank +def _slot_values(held, states, blank) -> dict[int, float]: + """Each slot's current value: 0 for a blank slot, else its value decayed with its idle age.""" + return {b: 0.0 if b in blank else stale_retention(len(h), states[b]["age_s"]) for b, h in enumerate(held)} + + +def _worth_keeping(value: float) -> bool: + return value >= EXPIRED_KEEP_SCORE and value > 0 + + +def reclaim_slot(candidates, values, blank, rank, positions) -> int | None: + """The reclaim order place() and park_destination() share, over idle expired slots (blank ones included): the + slots not worth keeping (blank, or a value below EXPIRED_KEEP_SCORE) before histories worth keeping; then `rank` + (the group's load); then the lowest value, a blank slot first at an equal value; then the slot order.""" + if not candidates: + return None + return min(candidates, key=lambda b: (_worth_keeping(values[b]), *rank(b), values[b], b not in blank, + positions[b])) + + def park_destination(main: list[int], busy: list[bool], held: list[list[int]], used: list[float], order: list[int], group_size: int, now: float, valid: list[bool] | None = None, - padding_safe: bool = False, exclude=()) -> int | None: + padding_safe: bool = False, exclude=(), main_value: float = 0.0) -> int | None: """An idle slot to keep the main session's conversation in before another request replaces it (engine park=): - its own older copy, else a blank slot of the least loaded group, else an expired history worth less than - EXPIRED_KEEP_SCORE (the ones place() overwrites first). Never a history worth keeping. None: nowhere to keep it - (or it is already kept).""" + its own older copy, else an expired slot in place()'s reclaim order (reclaim_slot), where a history worth keeping + counts only when `main_value` (the main session's current value) is at least PARK_VALUE_RATIO times its own. + Never a protected slot. None: nowhere to keep it (or it is already kept).""" if not padding_safe or not main: return None gs = max(1, group_size) @@ -145,15 +168,11 @@ def park_destination(main: list[int], busy: list[bool], held: list[list[int]], u len(held[b]) < len(main) and main[:len(held[b])] == held[b]] if own: return max(own, key=lambda b: (len(held[b]), -positions[b])) - free = [b for b in order if not busy[b] and b not in exclude and b in blank] - if free: - return min(free, key=lambda b: (loads[b - b % gs], positions[b])) - # The main session is at least PARK_MIN_TOKENS long; give up a short side chat or a long history left unused - # until its value decays below EXPIRED_KEEP_SCORE. The least loaded group, then the lowest current score. - scores = {b: stale_retention(len(held[b]), states[b]["age_s"]) for b in order} - worthless = [b for b in order if not busy[b] and b not in exclude and states[b]["expired"] and - scores[b] < EXPIRED_KEEP_SCORE] - return min(worthless, key=lambda b: (loads[b - b % gs], scores[b], positions[b])) if worthless else None + values = _slot_values(held, states, blank) + candidates = [b for b in order if not busy[b] and b not in exclude and states[b]["expired"] and + (not _worth_keeping(values[b]) or + (PARK_VALUE_RATIO > 0 and main_value >= PARK_VALUE_RATIO * values[b]))] + return reclaim_slot(candidates, values, blank, lambda b: (loads[b - b % gs],), positions) def place(prompt: list[int], busy: list[bool], held: list[list[int]], used: list[float], @@ -208,17 +227,19 @@ def pipeline(b): safe = [b for b in free if states[b]["expired"] and collateral(b) == 0] + values = _slot_values(held, states, blank) + def stale(b): - return 0.0 if b in blank else stale_retention(len(held[b]), states[b]["age_s"]) + return values[b] def worthy(b): - return stale(b) >= EXPIRED_KEEP_SCORE and stale(b) > 0 + return _worth_keeping(values[b]) if safe: # Expired = empty for protection, but an expired history can still match until it is overwritten: one that # still scores EXPIRED_KEEP_SCORE goes only when no blank or worthless slot is free (else a new chat in the # idle group overwrote the other chat every turn - GPU run 2026-10-15). Then group load; then the lowest score. - slot = min(safe, key=lambda b: (worthy(b), *pipeline(b)[:2], b not in blank, stale(b), positions[b])) + slot = reclaim_slot(safe, values, blank, lambda b: pipeline(b)[:2], positions) # Only worthy histories are left, but a protected slot worth less ends its protection within the wait window # (a side request that has just finished): wait for it under the same one-per-request 5 s protection wait # rather than destroy a conversation that can still match. After the wait the overwrite goes ahead. diff --git a/serve/test_slot_alloc.py b/serve/test_slot_alloc.py index 7e2433a808d..fc4e7a11ee8 100644 --- a/serve/test_slot_alloc.py +++ b/serve/test_slot_alloc.py @@ -534,9 +534,9 @@ def e(self): e.main_used = self.NOW return e - def dest(self, e, exclude=()): + def dest(self, e, exclude=(), main_value=0.0): return slot_policy.park_destination(e.main_held, e.slot_busy, e.slot_held, e.slot_used, e.slot_order, - e.slot_gs, self.NOW, e.slot_cache_valid, True, exclude) + e.slot_gs, self.NOW, e.slot_cache_valid, True, exclude, main_value) def test_another_conversation_is_told_by_the_main_history_up_to_its_last_turn(self): # main: system (turn 1 at 0), user turn at 20000, reply turn at 29000; a prompt that shares all of it up to @@ -657,6 +657,36 @@ def test_an_expired_history_worth_less_than_the_keep_score_takes_the_park(self): e.slot_used[1] = self.NOW # protected: not a park target self.assertEqual(self.dest(e), 4) + def test_park_uses_the_reclaim_order_a_lighter_group_before_a_blank_slot(self): + e = self.e() + e.slot_held = [[b] * 40000 for b in range(6)] + e.slot_used = [self.NOW - 100] * 6 # expired, worth keeping + e.slot_held[1] = [] # blank, in the group of busy slot 0 + e.slot_busy[0] = True + e.slot_held[4] = [4] * 1000 # not worth keeping, in an idle group + self.assertEqual(self.dest(e), 4) # group load first, as place() does + e.slot_busy[0] = False + self.assertEqual(self.dest(e), 1) # the same load: the lower value (a blank slot is 0) + + def test_a_history_worth_keeping_takes_a_park_only_at_the_value_ratio(self): + e = self.e() + e.slot_held = [[b] * 40000 for b in range(6)] + e.slot_used = [self.NOW - 100] * 6 # all expired, worth keeping + worth = slot_policy.stale_retention(40000, 100) + self.assertIsNone(self.dest(e, main_value=worth * 1.9)) # less than twice as much: every one is kept + self.assertEqual(self.dest(e, main_value=worth * 2), 0) # twice as much: the slot order breaks the tie + e.slot_busy[0] = True + e.slot_used[3] = self.NOW - 1000 # worth less than the others + self.assertEqual(self.dest(e, main_value=worth * 2), 3) # least loaded group, then the lowest value + cheap = slot_policy.stale_retention(40000, 1000) + self.assertEqual(self.dest(e, main_value=cheap * 2), 3) # only slot 3 is cheap enough + self.assertIsNone(self.dest(e, main_value=cheap * 1.5)) + slot_policy.configure({"STRATA_SLOT_PARK_VALUE_RATIO": "0"}) + try: + self.assertIsNone(self.dest(e, main_value=100.0)) # 0: a park never takes one + finally: + slot_policy.configure({}) + def test_a_long_unused_history_can_take_a_park_but_busy_and_excluded_slots_cannot(self): e = self.e() e.slot_held = [[b] * 40000 for b in range(6)] @@ -697,7 +727,7 @@ def test_environment_overrides_and_bad_values_keep_defaults(self): got = slot_policy.configure({"STRATA_SLOT_TTL_MAX_S": "240", "STRATA_SLOT_PROTECT_WAIT_S": "2", "STRATA_SLOT_PARK_MIN_TOKENS": "x"}) self.assertEqual(got, {"ttl_max_s": 240.0, "protect_wait_s": 2.0, "park_min_tokens": 8192, - "expired_keep_score": 0.01, "idle_value_zero_s": 1800.0}) + "expired_keep_score": 0.01, "idle_value_zero_s": 1800.0, "park_value_ratio": 2.0}) self.assertAlmostEqual(slot_policy.cache_ttl(100000), 180.0) # the whole curve doubles self.assertAlmostEqual(slot_policy.cache_ttl(10 ** 6), 240.0) self.assertEqual(slot_policy.configure({})["ttl_max_s"], 120.0) From e10df8cb952f4cb4957e38526ca39be7c1def5ee Mon Sep 17 00:00:00 2001 From: andyhrc <12491949+andyhrc@users.noreply.github.com> Date: Fri, 16 Oct 2026 03:02:33 +0800 Subject: [PATCH 17/21] serve: STRATA_SLOT_PARK_MIN_VALUE - the main session's own threshold for a park A park copied the main session's conversation into a slot whenever its value at its idle age was at least STRATA_SLOT_EXPIRED_KEEP_SCORE, the threshold that also decides which expired slot histories are worth keeping. With the default 0.01 every conversation of 8K tokens or more was parked until about 26-30 minutes idle, and the copy is not free: 24 production parks of 10K-170K tokens took 0.17-2.56 s each, on the control lines, while no batch window runs. STRATA_SLOT_PARK_MIN_VALUE (default 0.01, so nothing changes unless it is set) is now the park's own threshold; STRATA_SLOT_EXPIRED_KEEP_SCORE keeps deciding slot reclaim only. live.slot_policy shows it. Tests: a 50K conversation idle 25 minutes (value ~0.07) is not parked at 0.1 while slot reclaim keeps 0.01, and is parked idle 20 minutes (~0.13); STRATA_SLOT_PROTECT_WAIT_S=0 admits at once over protected histories. docs/BATCHING.md: the park bullet described an earlier version; it now lists the park's thresholds and destination order, and the value decay. Full serve suite: one failure, test_responses NoLeakedSampler, also without this change. Co-Authored-By: Claude Opus 5.5 --- docs/BATCHING.md | 13 ++++++++++--- serve/server.py | 2 +- serve/slot_policy.py | 8 ++++++-- serve/test_parallel.py | 23 +++++++++++++++++++++++ serve/test_slot_alloc.py | 22 +++++++++++++++++++++- 5 files changed, 61 insertions(+), 7 deletions(-) diff --git a/docs/BATCHING.md b/docs/BATCHING.md index dc6a909afec..136d55722f5 100644 --- a/docs/BATCHING.md +++ b/docs/BATCHING.md @@ -489,9 +489,16 @@ tokens. The policy weighs length and age, and every threshold is a parameter: `STRATA_SLOT_PROTECT_WAIT_S` (default 5 s, once per request), then overwrites the history that loses least (losses within 10% go by group load). It also waits, within the same single wait, when only expired histories worth keeping are left and a cheaper protected one ends its protection within it. -- **Park** (with `slot_padding_safe=1`): the conversation's own older copy, else a blank slot of the least loaded - group - never a slot that holds another conversation; not when a slot holds 90% of it already, or when it is - shorter than `STRATA_SLOT_PARK_MIN_TOKENS` (default 8192). +- **Expired value decays**: after its protection an expired history is worth its score at freshness 0 times + `1 - x^2`, where x runs from 0 at the end of its protection to 1 at a total idle age of + `STRATA_SLOT_IDLE_VALUE_ZERO_S` (default 1800 s; 0 disables decay). Reclaiming and parking use this value; it never + changes whether a history can match. +- **Park** (with `slot_padding_safe=1`): not when the main session's conversation is shorter than + `STRATA_SLOT_PARK_MIN_TOKENS` (default 8192), when its value at its idle age is below `STRATA_SLOT_PARK_MIN_VALUE` + (default 0.01), or when a slot holds 90% of it already. Then its own older copy, else an idle expired slot in the + reclaim order above (not worth keeping first, then group load, then the lowest value); a history worth keeping is + taken only when the main session's conversation is worth at least `STRATA_SLOT_PARK_VALUE_RATIO` (default 2) times + more. Never another conversation's protected slot. The parked copy keeps the main session's last-use time. - A busy slot is never matched or overwritten. On an engine without `slot_padding_safe=1`, a slot that would pad over a protected idle history below it costs that history's retention too. diff --git a/serve/server.py b/serve/server.py index 463e7f78751..cf1904206f9 100644 --- a/serve/server.py +++ b/serve/server.py @@ -2383,7 +2383,7 @@ def _claim_park(self, prompt, exclude=()) -> str: now = time.time() main_used = getattr(self, "main_used", 0.0) value = slot_policy.stale_retention(len(main), max(0.0, now - main_used)) - if value <= 0 or value < slot_policy.EXPIRED_KEEP_SCORE: + if value <= 0 or value < slot_policy.PARK_MIN_VALUE: self.cache_event("park_skip", reason="main_low_value", held_tokens=len(main), value=value) return "" # an unused main history need not be copied into a slot just to be reclaimed with self.slot_cv: diff --git a/serve/slot_policy.py b/serve/slot_policy.py index f06c31a3a73..5c792ef5fc8 100644 --- a/serve/slot_policy.py +++ b/serve/slot_policy.py @@ -14,6 +14,8 @@ PARK_VALUE_RATIO = 2.0 # STRATA_SLOT_PARK_VALUE_RATIO: a park takes an expired history worth keeping only when the # main session's conversation is worth at least this many times more; 0: never IDLE_VALUE_ZERO_S = 1800.0 # STRATA_SLOT_IDLE_VALUE_ZERO_S: idle age when expired value reaches zero; 0 disables decay +PARK_MIN_VALUE = 0.01 # STRATA_SLOT_PARK_MIN_VALUE: the main session's conversation is parked only while its value + # (stale_retention at its idle age) is at least this; 0.01 is EXPIRED_KEEP_SCORE's default TTL_MAX_S = 120.0 # STRATA_SLOT_TTL_MAX_S: protection at 262144 tokens; the whole curve scales with it _TTL_SHAPE = ((0, 0.0), (500, 0.1), (2000, 2.0), (8000, 10.0), (32000, 45.0), (50000, 65.0), (100000, 90.0), (MAX_CACHE_TOKENS, 120.0)) @@ -23,7 +25,7 @@ def configure(env) -> dict: """Read the tunables from `env` (the engine's environment); bad or missing values keep the defaults.""" global PROTECTION_WAIT_S, PARK_MIN_TOKENS, TTL_MAX_S, TTL_POINTS, EXPIRED_KEEP_SCORE, IDLE_VALUE_ZERO_S, \ - PARK_VALUE_RATIO + PARK_VALUE_RATIO, PARK_MIN_VALUE def value(name, default): try: @@ -38,13 +40,15 @@ def value(name, default): PROTECTION_WAIT_S = value("STRATA_SLOT_PROTECT_WAIT_S", 5.0) PARK_MIN_TOKENS = int(value("STRATA_SLOT_PARK_MIN_TOKENS", 8192)) PARK_VALUE_RATIO = value("STRATA_SLOT_PARK_VALUE_RATIO", 2.0) + PARK_MIN_VALUE = value("STRATA_SLOT_PARK_MIN_VALUE", 0.01) return settings() def settings() -> dict: return {"ttl_max_s": TTL_MAX_S, "protect_wait_s": PROTECTION_WAIT_S, "park_min_tokens": PARK_MIN_TOKENS, "expired_keep_score": EXPIRED_KEEP_SCORE, - "idle_value_zero_s": IDLE_VALUE_ZERO_S, "park_value_ratio": PARK_VALUE_RATIO} + "idle_value_zero_s": IDLE_VALUE_ZERO_S, "park_value_ratio": PARK_VALUE_RATIO, + "park_min_value": PARK_MIN_VALUE} def cache_ttl(tokens: int) -> float: diff --git a/serve/test_parallel.py b/serve/test_parallel.py index bcc711ffece..69e0efad4ef 100644 --- a/serve/test_parallel.py +++ b/serve/test_parallel.py @@ -20,6 +20,7 @@ from serve.server import ByteTokenizer, Service, StrataEngine, engine_args, parallel_args, serve from strata_cli import bench from strata_cli.client import Endpoint +from serve import slot_policy # The fake engine: one token every STEP seconds per active slot (a "window" serves every active slot at once); # GEN (solo) streams T lines; BGEN reads the prompt (one T line, DONE), answers BADM and continues in the slot. @@ -415,6 +416,28 @@ def test_affinity_protection_wait_is_bounded_and_then_admits(self): cancel.set() t.join(3) + def test_affinity_protection_wait_zero_admits_at_once(self): + # STRATA_SLOT_PROTECT_WAIT_S=0: every idle history protected, the request overwrites one without waiting + self.start(6, says_groups=3, affinity=True) + slot_policy.configure({"STRATA_SLOT_PROTECT_WAIT_S": "0"}) + e = self.engine + try: + with e.slot_cv: + e.slot_held = [[9000 + b] * 50000 for b in range(6)] + e.slot_used = [time.time()] * 6 + e.slot_cache_valid = [True] * 6 + e.slot_busy[0] = True + e.slot_claims.add(0) + started = time.monotonic() + tokens = list(e.generate_batched([9999], 8, {}, threading.Event())) + self.assertTrue(tokens) + self.assertLess(time.monotonic() - started, 1.0) + waits = [x for x in e.cache_events_view() if x["kind"] == "protection_end"] + self.assertTrue(waits) + self.assertLess(waits[-1]["waited_s"], 0.5) + finally: + slot_policy.configure({}) + def test_affinity_wait_wakes_when_a_short_history_expires(self): cancel, result, t = self._protected_waiter() try: diff --git a/serve/test_slot_alloc.py b/serve/test_slot_alloc.py index fc4e7a11ee8..90fffc02cb0 100644 --- a/serve/test_slot_alloc.py +++ b/serve/test_slot_alloc.py @@ -614,6 +614,25 @@ def test_main_above_keep_threshold_is_parked_with_its_old_age(self): e._on_parked(b, 49999) self.assertEqual(e.slot_used[b], self.NOW - 1760) + def test_park_min_value_is_its_own_threshold(self): + # a 50K conversation idle for 25 minutes is worth about 0.07, for 20 minutes about 0.13 + slot_policy.configure({"STRATA_SLOT_PARK_MIN_VALUE": "0.1"}) + try: + self.assertEqual(slot_policy.EXPIRED_KEEP_SCORE, 0.01) # slot reclaim keeps its own threshold + e = self.e() + e.main_used = self.NOW - 1500 + self.assertLess(slot_policy.stale_retention(50000, 1500), 0.1) + self.assertGreater(slot_policy.stale_retention(50000, 1500), slot_policy.EXPIRED_KEEP_SCORE) + self.assertEqual(e._claim_park([1] * 60), '') + self.assertEqual(e.cache_events_view()[0]['reason'], 'main_low_value') + self.assertFalse(any(e.slot_busy)) + e = self.e() + e.main_used = self.NOW - 1200 + self.assertGreater(slot_policy.stale_retention(50000, 1200), 0.1) + self.assertTrue(e._claim_park([1] * 60).startswith(' park=')) + finally: + slot_policy.configure({}) + def test_zero_value_main_can_still_continue_without_a_park_or_deletion(self): e = self.e() e.main_used = self.NOW - 3600 @@ -727,7 +746,8 @@ def test_environment_overrides_and_bad_values_keep_defaults(self): got = slot_policy.configure({"STRATA_SLOT_TTL_MAX_S": "240", "STRATA_SLOT_PROTECT_WAIT_S": "2", "STRATA_SLOT_PARK_MIN_TOKENS": "x"}) self.assertEqual(got, {"ttl_max_s": 240.0, "protect_wait_s": 2.0, "park_min_tokens": 8192, - "expired_keep_score": 0.01, "idle_value_zero_s": 1800.0, "park_value_ratio": 2.0}) + "expired_keep_score": 0.01, "idle_value_zero_s": 1800.0, "park_value_ratio": 2.0, + "park_min_value": 0.01}) self.assertAlmostEqual(slot_policy.cache_ttl(100000), 180.0) # the whole curve doubles self.assertAlmostEqual(slot_policy.cache_ttl(10 ** 6), 240.0) self.assertEqual(slot_policy.configure({})["ttl_max_s"], 120.0) From be3467473461b3680fb581b5873189d385840b52 Mon Sep 17 00:00:00 2001 From: andyhrc <12491949+andyhrc@users.noreply.github.com> Date: Fri, 16 Oct 2026 08:43:09 +0800 Subject: [PATCH 18/21] serve: the policy does not wait - STRATA_SLOT_PROTECT_WAIT_S is gone place() answered "protected" in two cases, and the admission waited once per request, up to STRATA_SLOT_PROTECT_WAIT_S (5 s): - every idle history still protected. The wait ended in the overwrite it delayed: the one that loses least is the one with the lowest retention, the least likely to be continued within 5 s. In production (2026-10-15, 6 slots, 8 clients) all 7 such waits ran the full 5 s while requests were queued and a slot stood idle; - only expired histories worth keeping left, and a cheaper protected one ending its protection within the wait (a side request that had just finished). This one kept conversation C in the ABC run; without it C is overwritten while the side request's 1.5K history is still protected - a case of the protection rules, accepted. Both are removed with the mechanism's wait (previous commit): a free slot is always placed - a match, else a reclaim of an expired or blank slot, else the overwrite that loses least. STRATA_SLOT_PROTECT_WAIT_S and live.slot_policy.protect_wait_s are gone. Tests of the waits are removed; the ABC case is kept as a test of the behaviour now. docs/BATCHING.md: the "one bounded wait" rule becomes "no wait", and the ABC table marks the waiting row as removed. Co-Authored-By: Claude Opus 5.5 --- docs/BATCHING.md | 16 ++++---- serve/slot_policy.py | 28 +++---------- serve/test_parallel.py | 87 ---------------------------------------- serve/test_slot_alloc.py | 71 +++++++++++--------------------- 4 files changed, 38 insertions(+), 164 deletions(-) diff --git a/docs/BATCHING.md b/docs/BATCHING.md index 136d55722f5..5bf1ac850e8 100644 --- a/docs/BATCHING.md +++ b/docs/BATCHING.md @@ -485,10 +485,10 @@ tokens. The policy weighs length and age, and every threshold is a parameter: expired or blank slot: first one not worth keeping (blank, or its score at freshness 0 below `STRATA_SLOT_EXPIRED_KEEP_SCORE`, default 0.01 - about 2.5K tokens), then by group load, a blank slot before a history, and the lowest score. -- **One bounded wait**: when every idle history is still protected, the request waits up to - `STRATA_SLOT_PROTECT_WAIT_S` (default 5 s, once per request), then overwrites the history that loses least (losses - within 10% go by group load). It also waits, within the same single wait, when only expired histories worth keeping - are left and a cheaper protected one ends its protection within it. +- **No wait**: when every idle history is still protected, the request overwrites the one that loses least at once + (losses within 10% go by group load). An earlier version waited up to 5 s first, once per request; with more + requests than slots each wait ran its full 5 s, the slot idle meanwhile, and ended in the same overwrite (7 of 7 in + a 15-minute test on the machine above), so it was removed. - **Expired value decays**: after its protection an expired history is worth its score at freshness 0 times `1 - x^2`, where x runs from 0 at the end of its protection to 1 at a total idle age of `STRATA_SLOT_IDLE_VALUE_ZERO_S` (default 1800 s; 0 disables decay). Reclaiming and parking use this value; it never @@ -514,12 +514,14 @@ Measured on the machine above (0.1.44.1, `--batch 6` in 3 groups, `STRATA_PIPELI | Ordering of expired histories | B | C | | --- | --- | --- | | by group load first (an earlier version) | 0 every turn | 0 every turn | -| + one still worth keeping goes last | all but the ~20 new | 0 every turn | -| + wait for a cheaper slot about to leave its protection (this policy) | all but the ~20 new | all but the ~20 new | +| + one still worth keeping goes last (this policy) | all but the ~20 new | 0 every turn | +| + wait for a cheaper slot about to leave its protection (removed) | all but the ~20 new | all but the ~20 new | In the first row a new turn took the other conversation's expired slot for group balance, so B and C overwrote each other; in the second, C went to a side request that arrived a second after another one finished, while that one's -1.5K-token history was about to leave its protection. +1.5K-token history was about to leave its protection. The third row waited for that protection to end, under the +same one-per-request wait as above; it went with that wait. The policy accepts this case: a protected history is +not overwritten while an expired one can be. ## The VRAM expert tier keeps adapting in batch windows diff --git a/serve/slot_policy.py b/serve/slot_policy.py index 5c792ef5fc8..11f110bd521 100644 --- a/serve/slot_policy.py +++ b/serve/slot_policy.py @@ -7,7 +7,6 @@ MAX_CACHE_TOKENS = 262144 # Tunables (environment, read by configure() when the engine starts; the menu sets them per profile under # config_overrides.env). Defaults are the values the policy was specified with. -PROTECTION_WAIT_S = 5.0 # STRATA_SLOT_PROTECT_WAIT_S: wait when every idle history is still protected PARK_MIN_TOKENS = 8192 # STRATA_SLOT_PARK_MIN_TOKENS: a shorter main-session conversation is not parked EXPIRED_KEEP_SCORE = 0.01 # STRATA_SLOT_EXPIRED_KEEP_SCORE: an expired history scoring at least this (retention at # expiry, then decayed; 0.01 is about 2.5K tokens at expiry) goes last @@ -24,7 +23,7 @@ def configure(env) -> dict: """Read the tunables from `env` (the engine's environment); bad or missing values keep the defaults.""" - global PROTECTION_WAIT_S, PARK_MIN_TOKENS, TTL_MAX_S, TTL_POINTS, EXPIRED_KEEP_SCORE, IDLE_VALUE_ZERO_S, \ + global PARK_MIN_TOKENS, TTL_MAX_S, TTL_POINTS, EXPIRED_KEEP_SCORE, IDLE_VALUE_ZERO_S, \ PARK_VALUE_RATIO, PARK_MIN_VALUE def value(name, default): @@ -37,7 +36,6 @@ def value(name, default): EXPIRED_KEEP_SCORE = value("STRATA_SLOT_EXPIRED_KEEP_SCORE", 0.01) IDLE_VALUE_ZERO_S = value("STRATA_SLOT_IDLE_VALUE_ZERO_S", 1800.0) TTL_POINTS = tuple((n, s * TTL_MAX_S / 120.0) for n, s in _TTL_SHAPE) - PROTECTION_WAIT_S = value("STRATA_SLOT_PROTECT_WAIT_S", 5.0) PARK_MIN_TOKENS = int(value("STRATA_SLOT_PARK_MIN_TOKENS", 8192)) PARK_VALUE_RATIO = value("STRATA_SLOT_PARK_VALUE_RATIO", 2.0) PARK_MIN_VALUE = value("STRATA_SLOT_PARK_MIN_VALUE", 0.01) @@ -45,8 +43,8 @@ def value(name, default): def settings() -> dict: - return {"ttl_max_s": TTL_MAX_S, "protect_wait_s": PROTECTION_WAIT_S, - "park_min_tokens": PARK_MIN_TOKENS, "expired_keep_score": EXPIRED_KEEP_SCORE, + return {"ttl_max_s": TTL_MAX_S, "park_min_tokens": PARK_MIN_TOKENS, + "expired_keep_score": EXPIRED_KEEP_SCORE, "idle_value_zero_s": IDLE_VALUE_ZERO_S, "park_value_ratio": PARK_VALUE_RATIO, "park_min_value": PARK_MIN_VALUE} @@ -181,8 +179,7 @@ def park_destination(main: list[int], busy: list[bool], held: list[list[int]], u def place(prompt: list[int], busy: list[bool], held: list[list[int]], used: list[float], order: list[int], group_size: int, now: float, valid: list[bool] | None = None, - prefill_rate: float | None = None, allow_eviction: bool = True, - padding_safe: bool = False) -> Placement: + prefill_rate: float | None = None, padding_safe: bool = False) -> Placement: free = [b for b in order if not busy[b]] if not free: return Placement(None) @@ -233,27 +230,14 @@ def pipeline(b): values = _slot_values(held, states, blank) - def stale(b): - return values[b] - - def worthy(b): - return _worth_keeping(values[b]) - if safe: # Expired = empty for protection, but an expired history can still match until it is overwritten: one that # still scores EXPIRED_KEEP_SCORE goes only when no blank or worthless slot is free (else a new chat in the # idle group overwrote the other chat every turn - GPU run 2026-10-15). Then group load; then the lowest score. slot = reclaim_slot(safe, values, blank, lambda b: pipeline(b)[:2], positions) - # Only worthy histories are left, but a protected slot worth less ends its protection within the wait window - # (a side request that has just finished): wait for it under the same one-per-request 5 s protection wait - # rather than destroy a conversation that can still match. After the wait the overwrite goes ahead. - if worthy(slot) and not allow_eviction and any( - not states[b]["expired"] and collateral(b) == 0 and stale(b) < stale(slot) and - states[b]["ttl_s"] - states[b]["age_s"] <= PROTECTION_WAIT_S for b in free): - return Placement(None, reason="protected") return Placement(slot, reason="reclaim") - if not allow_eviction: - return Placement(None, reason="protected") + # Every idle history is protected: no wait (it cost a free slot 5 s per request when requests were queued, and + # ended in the same overwrite) - the history that loses least goes now. cheapest = min(loss(b) for b in free) # Loss scores within 10% are close enough to let group load/padding break the tie. candidates = [b for b in free if loss(b) <= cheapest * 1.10 + 1e-12] diff --git a/serve/test_parallel.py b/serve/test_parallel.py index 69e0efad4ef..accb7c297fd 100644 --- a/serve/test_parallel.py +++ b/serve/test_parallel.py @@ -377,93 +377,6 @@ def get(self, path): with urllib.request.urlopen(self.base + path, timeout=10) as r: return json.loads(r.read().decode()) - def _protected_waiter(self): - self.start(6, says_groups=3, affinity=True) - e = self.engine - with e.slot_cv: - e.slot_held = [[9000 + b] * 50000 for b in range(6)] - e.slot_used = [time.time()] * 6 - e.slot_cache_valid = [True] * 6 - # Keep one request marked busy to exercise admission instead of the solo path. - e.slot_busy[0] = True - e.slot_claims.add(0) - cancel, result = threading.Event(), {} - def run(): - try: - result['tokens'] = list(e.generate_batched([9999], 8, {}, cancel)) - except BaseException as ex: - result['error'] = ex - t = threading.Thread(target=run) - t.start() - end = time.monotonic() + 2 - while time.monotonic() < end and not e.admitting: - time.sleep(0.01) - self.assertGreater(e.admitting, 0) - self.assertTrue(e.ctl.acquire(blocking=False)) # waiting never monopolizes the control channel - e.ctl.release() - return cancel, result, t - - def test_affinity_protection_wait_is_bounded_and_then_admits(self): - started = time.monotonic() - cancel, result, t = self._protected_waiter() - try: - t.join(8) - self.assertFalse(t.is_alive()) - self.assertNotIn('error', result) - self.assertGreaterEqual(time.monotonic() - started, 4.8) - self.assertTrue(result['tokens']) - finally: - cancel.set() - t.join(3) - - def test_affinity_protection_wait_zero_admits_at_once(self): - # STRATA_SLOT_PROTECT_WAIT_S=0: every idle history protected, the request overwrites one without waiting - self.start(6, says_groups=3, affinity=True) - slot_policy.configure({"STRATA_SLOT_PROTECT_WAIT_S": "0"}) - e = self.engine - try: - with e.slot_cv: - e.slot_held = [[9000 + b] * 50000 for b in range(6)] - e.slot_used = [time.time()] * 6 - e.slot_cache_valid = [True] * 6 - e.slot_busy[0] = True - e.slot_claims.add(0) - started = time.monotonic() - tokens = list(e.generate_batched([9999], 8, {}, threading.Event())) - self.assertTrue(tokens) - self.assertLess(time.monotonic() - started, 1.0) - waits = [x for x in e.cache_events_view() if x["kind"] == "protection_end"] - self.assertTrue(waits) - self.assertLess(waits[-1]["waited_s"], 0.5) - finally: - slot_policy.configure({}) - - def test_affinity_wait_wakes_when_a_short_history_expires(self): - cancel, result, t = self._protected_waiter() - try: - with self.engine.slot_cv: - self.engine.slot_held[4] = [42] * 500 - self.engine.slot_used[4] = time.time() - 0.2 - self.engine.slot_cv.notify_all() - t.join(2) - self.assertFalse(t.is_alive()) - self.assertNotIn('error', result) - self.assertTrue(result['tokens']) - finally: - cancel.set() - t.join(3) - - def test_affinity_wait_can_be_cancelled_without_overwriting_cache(self): - cancel, result, t = self._protected_waiter() - before = list(self.engine.slot_held) - cancel.set() - t.join(2) - self.assertFalse(t.is_alive()) - self.assertNotIn('error', result) - self.assertEqual(result.get('tokens'), []) - self.assertEqual(self.engine.slot_held, before) - self.assertEqual(self.engine.admitting, 0) - def _two_at_once(self, text_b): """A long request first, then one that has to go to a batch slot (the engine is busy with the first).""" out = {} diff --git a/serve/test_slot_alloc.py b/serve/test_slot_alloc.py index 90fffc02cb0..a1153c2217b 100644 --- a/serve/test_slot_alloc.py +++ b/serve/test_slot_alloc.py @@ -228,7 +228,7 @@ def test_expiry_is_virtual_and_expired_history_still_matches(self): h = list(range(500)) e.slot_held[2] = h e.slot_used[2] -= 10 - plan = self.place(e, h + [9999], allow_eviction=False) + plan = self.place(e, h + [9999]) self.assertEqual(plan.source_hint, 2) self.assertEqual(plan.reused_hint, 500) self.assertEqual(e.slot_held[2], h) # no token deletion on expiry @@ -278,10 +278,10 @@ def test_old_long_history_yields_before_a_recent_shorter_one_even_in_busier_grou e.slot_busy[1] = True e.slot_held[0], e.slot_used[0] = [0] * 150000, self.NOW - 1800 e.slot_held[4], e.slot_used[4] = [4] * 8000, self.NOW - 15 - self.assertEqual(self.place(e, [9999], padding_safe=True, allow_eviction=False).slot, 0) + self.assertEqual(self.place(e, [9999], padding_safe=True).slot, 0) # A use refreshes the age: the long history is protected again and the expired shorter one yields. e.slot_used[0] = self.NOW - self.assertEqual(self.place(e, [9999], padding_safe=True, allow_eviction=False).slot, 4) + self.assertEqual(self.place(e, [9999], padding_safe=True).slot, 4) def test_long_unused_history_still_matches_after_its_value_falls_below_keep_score(self): e = self.e() @@ -289,30 +289,29 @@ def test_long_unused_history_still_matches_after_its_value_falls_below_keep_scor e.slot_held[2] = list(range(100000)) e.slot_used[2] = self.NOW - 3600 self.assertEqual(slot_policy.stale_retention(100000, self.NOW - e.slot_used[2]), 0) - plan = self.place(e, list(range(100001)), padding_safe=True, allow_eviction=False) + plan = self.place(e, list(range(100001)), padding_safe=True) self.assertEqual((plan.slot, plan.source_hint, plan.reused_hint), (2, 2, 100000)) self.assertEqual(len(e.slot_held[2]), 100000) - def test_decayed_history_does_not_wait_for_a_cheaper_protected_side_request(self): + def test_decayed_history_is_reclaimed_while_a_side_request_is_protected(self): e = self.e() e.slot_held = [[b] * 50000 for b in range(6)] e.slot_used[0] = self.NOW - 1800 e.slot_held[2], e.slot_used[2] = [2] * 1500, self.NOW - 0.5 - plan = self.place(e, [9999], padding_safe=True, allow_eviction=False) + plan = self.place(e, [9999], padding_safe=True) self.assertEqual((plan.slot, plan.reason), (0, 'reclaim')) def test_unexpired_fifty_k_is_selected_before_similarly_fresh_150k(self): e = self.e() e.slot_held = [[b] * (50000 if b == 2 else 150000) for b in range(6)] self.assertEqual(self.place(e, [9999]).slot, 2) - self.assertEqual(self.place(e, [9999], allow_eviction=False).reason, 'protected') def test_expired_short_helper_is_reclaimed_without_touching_long_histories(self): e = self.e() e.slot_held = [[b] * 100000 for b in range(6)] e.slot_held[4] = [4] * 500 e.slot_used[4] -= 0.11 - self.assertEqual(self.place(e, [9999], allow_eviction=False).slot, 4) + self.assertEqual(self.place(e, [9999]).slot, 4) def test_cache_source_can_be_restored_into_another_pipeline_group(self): e = self.e() @@ -367,13 +366,13 @@ def test_match_moves_only_into_a_blank_slot_of_a_less_loaded_group(self): plan = self.place(e, list(range(10001)), padding_safe=True) self.assertEqual(plan.slot, 1) # no blank slot elsewhere: stay in the busy group - def test_superseded_copy_is_blank_and_needs_no_wait(self): + def test_superseded_copy_is_blank_and_reclaimed_before_protected_histories(self): e = self.e() - e.slot_held = [[b] * 50000 for b in range(6)] # all fresh: a new chat would have to wait + e.slot_held = [[b] * 50000 for b in range(6)] # all fresh: every other one is protected e.slot_held[0] = list(range(9000)) # where the conversation was before it moved e.slot_held[4] = list(range(10000)) # where it is now self.assertEqual(slot_policy.superseded_slots(e.slot_held, e.slot_busy, [True] * 6), {0}) - plan = self.place(e, [9999], allow_eviction=False) + plan = self.place(e, [9999]) self.assertEqual((plan.slot, plan.reason), (0, "reclaim")) def test_reclaim_overwrites_expired_history_only_after_blank_slots(self): @@ -414,19 +413,19 @@ def test_new_chat_takes_a_worthless_slot_in_a_busier_group_before_an_expired_con finally: slot_policy.configure({}) - def test_waits_for_a_side_request_slot_about_to_expire_instead_of_a_conversation(self): + def test_no_wait_for_a_side_request_slot_about_to_expire(self): # GPU run 2026-10-15: a filler arriving 1 s after another filler finished overwrote expired 7.8K chat C, - # while the finished filler's 1.5K history left its 1.4 s protection a moment later. + # while the finished filler's 1.5K history left its 1.4 s protection a moment later. A wait for it was + # removed (2026-10-16): the protection rules decide, so C goes while the filler slot is protected. e = self.e() e.slot_busy[1] = e.slot_busy[5] = True e.slot_held[0], e.slot_used[0] = [0] * 37365, self.NOW - 300 # expired, worthy e.slot_held[3], e.slot_used[3] = [3] * 7848, self.NOW - 15 # C: expired, worthy e.slot_held[2], e.slot_used[2] = [2] * 1526, self.NOW - 0.5 # filler: protected for under 1 s more e.slot_held[4], e.slot_used[4] = [4] * 1526, self.NOW - 0.2 - self.assertEqual(self.place(e, [9999], padding_safe=True, allow_eviction=False).reason, "protected") - self.assertEqual(self.place(e, [9999], padding_safe=True, allow_eviction=True).slot, 3) # after the wait - e.slot_used[2] = self.NOW - 2 # the filler slot expired: no wait - self.assertEqual(self.place(e, [9999], padding_safe=True, allow_eviction=False).slot, 2) + self.assertEqual(self.place(e, [9999], padding_safe=True).slot, 3) + e.slot_used[2] = self.NOW - 2 # the filler slot expired + self.assertEqual(self.place(e, [9999], padding_safe=True).slot, 2) def test_near_match_budget_respects_one_second_and_4096_cap(self): e = self.e() @@ -435,41 +434,18 @@ def test_near_match_budget_respects_one_second_and_4096_cap(self): self.assertEqual(self.place(e, p, prefill_rate=500).near_tokens, 500) self.assertEqual(self.place(e, p, prefill_rate=10000).near_tokens, 4096) - def test_busy_slots_never_match_or_get_overwritten_after_wait(self): + def test_busy_slots_never_match_or_get_overwritten(self): e = self.e() e.slot_busy = [True] * 6 e.slot_held[2] = list(range(500)) - self.assertIsNone(self.place(e, list(range(501)), allow_eviction=True).slot) - with mock.patch('serve.server.time.monotonic', return_value=20): - self.assertEqual(e._admission_slot(list(range(501)), None), (None, None)) + self.assertIsNone(self.place(e, list(range(501))).slot) - def test_padding_loss_protects_another_chat_in_a_lower_row(self): + def test_padding_loss_counts_a_protected_history_in_a_lower_row(self): e = self.e() e.slot_held[0] = [77] * 100000 e.slot_busy[2:] = [True] * 4 - # Slot 1 is empty, but using it would pad over the protected history at 0. - self.assertEqual(self.place(e, [9999], allow_eviction=False).reason, 'protected') - self.assertEqual(self.place(e, [9999], allow_eviction=True).slot, 0) - - def test_protection_window_once_then_eviction_and_early_reclaim(self): - e = self.e() - e.slot_held = [[b] * 50000 for b in range(6)] - with mock.patch('serve.server.time.time', return_value=self.NOW), \ - mock.patch('serve.server.time.monotonic', return_value=10): - slot, since = e._admission_slot([9999], None) - self.assertIsNone(slot) - self.assertEqual(since, 10) - with mock.patch('serve.server.time.time', return_value=self.NOW), \ - mock.patch('serve.server.time.monotonic', return_value=14.9): - self.assertEqual(e._admission_slot([9999], since), (None, 10)) - with mock.patch('serve.server.time.time', return_value=self.NOW), \ - mock.patch('serve.server.time.monotonic', return_value=15): - self.assertIsNotNone(e._admission_slot([9999], since)[0]) - e.slot_held[4] = [4] * 500 - e.slot_used[4] = self.NOW - 0.2 - with mock.patch('serve.server.time.time', return_value=self.NOW), \ - mock.patch('serve.server.time.monotonic', return_value=10.2): - self.assertEqual(e._admission_slot([9999], since)[0], 4) + # Slot 1 is empty, but using it would pad over the protected history at 0: the same loss as taking slot 0. + self.assertEqual(self.place(e, [9999]).slot, 0) def test_padding_invalidity_survives_completion_of_higher_row(self): e = self.e() @@ -743,9 +719,8 @@ def tearDown(self): slot_policy.configure({}) def test_environment_overrides_and_bad_values_keep_defaults(self): - got = slot_policy.configure({"STRATA_SLOT_TTL_MAX_S": "240", "STRATA_SLOT_PROTECT_WAIT_S": "2", - "STRATA_SLOT_PARK_MIN_TOKENS": "x"}) - self.assertEqual(got, {"ttl_max_s": 240.0, "protect_wait_s": 2.0, "park_min_tokens": 8192, + got = slot_policy.configure({"STRATA_SLOT_TTL_MAX_S": "240", "STRATA_SLOT_PARK_MIN_TOKENS": "x"}) + self.assertEqual(got, {"ttl_max_s": 240.0, "park_min_tokens": 8192, "expired_keep_score": 0.01, "idle_value_zero_s": 1800.0, "park_value_ratio": 2.0, "park_min_value": 0.01}) self.assertAlmostEqual(slot_policy.cache_ttl(100000), 180.0) # the whole curve doubles From 9b0425968853ac20a308fb57243e2df16d668709 Mon Sep 17 00:00:00 2001 From: andyhrc <12491949+andyhrc@users.noreply.github.com> Date: Fri, 16 Oct 2026 09:01:01 +0800 Subject: [PATCH 19/21] serve: STRATA_SLOT_PARK_MIN_TOKENS is gone - the park's value threshold decides alone A park skipped every main-session conversation under STRATA_SLOT_PARK_MIN_TOKENS (8192 tokens) before its value was looked at: a fixed length cut-off beside the score. At the default STRATA_SLOT_PARK_MIN_VALUE (0.01) the score alone skips conversations worth less than about 2.6K tokens at expiry; at 0.1 (production) about 18K, so the cut-off either decided nothing or overrode the score. The constant, its variable and live.slot_policy.park_min_tokens are removed (the server's check went with the mechanism's previous commit). A 5000-token conversation is now parked at the default threshold and a 2000-token one is not (main_low_value); the test says so. docs/BATCHING.md: the park bullet names the value threshold only. Co-Authored-By: Claude Opus 5.5 --- docs/BATCHING.md | 6 +++--- serve/slot_policy.py | 7 ++----- serve/test_slot_alloc.py | 18 +++++++++++------- 3 files changed, 16 insertions(+), 15 deletions(-) diff --git a/docs/BATCHING.md b/docs/BATCHING.md index 5bf1ac850e8..67fff347966 100644 --- a/docs/BATCHING.md +++ b/docs/BATCHING.md @@ -493,9 +493,9 @@ tokens. The policy weighs length and age, and every threshold is a parameter: `1 - x^2`, where x runs from 0 at the end of its protection to 1 at a total idle age of `STRATA_SLOT_IDLE_VALUE_ZERO_S` (default 1800 s; 0 disables decay). Reclaiming and parking use this value; it never changes whether a history can match. -- **Park** (with `slot_padding_safe=1`): not when the main session's conversation is shorter than - `STRATA_SLOT_PARK_MIN_TOKENS` (default 8192), when its value at its idle age is below `STRATA_SLOT_PARK_MIN_VALUE` - (default 0.01), or when a slot holds 90% of it already. Then its own older copy, else an idle expired slot in the +- **Park** (with `slot_padding_safe=1`): not when the main session's conversation's value at its idle age is below + `STRATA_SLOT_PARK_MIN_VALUE` (default 0.01, about 2.6K tokens at expiry; no separate length cut-off), or when a + slot holds 90% of it already. Then its own older copy, else an idle expired slot in the reclaim order above (not worth keeping first, then group load, then the lowest value); a history worth keeping is taken only when the main session's conversation is worth at least `STRATA_SLOT_PARK_VALUE_RATIO` (default 2) times more. Never another conversation's protected slot. The parked copy keeps the main session's last-use time. diff --git a/serve/slot_policy.py b/serve/slot_policy.py index 11f110bd521..b08caf19d6a 100644 --- a/serve/slot_policy.py +++ b/serve/slot_policy.py @@ -7,7 +7,6 @@ MAX_CACHE_TOKENS = 262144 # Tunables (environment, read by configure() when the engine starts; the menu sets them per profile under # config_overrides.env). Defaults are the values the policy was specified with. -PARK_MIN_TOKENS = 8192 # STRATA_SLOT_PARK_MIN_TOKENS: a shorter main-session conversation is not parked EXPIRED_KEEP_SCORE = 0.01 # STRATA_SLOT_EXPIRED_KEEP_SCORE: an expired history scoring at least this (retention at # expiry, then decayed; 0.01 is about 2.5K tokens at expiry) goes last PARK_VALUE_RATIO = 2.0 # STRATA_SLOT_PARK_VALUE_RATIO: a park takes an expired history worth keeping only when the @@ -23,7 +22,7 @@ def configure(env) -> dict: """Read the tunables from `env` (the engine's environment); bad or missing values keep the defaults.""" - global PARK_MIN_TOKENS, TTL_MAX_S, TTL_POINTS, EXPIRED_KEEP_SCORE, IDLE_VALUE_ZERO_S, \ + global TTL_MAX_S, TTL_POINTS, EXPIRED_KEEP_SCORE, IDLE_VALUE_ZERO_S, \ PARK_VALUE_RATIO, PARK_MIN_VALUE def value(name, default): @@ -36,15 +35,13 @@ def value(name, default): EXPIRED_KEEP_SCORE = value("STRATA_SLOT_EXPIRED_KEEP_SCORE", 0.01) IDLE_VALUE_ZERO_S = value("STRATA_SLOT_IDLE_VALUE_ZERO_S", 1800.0) TTL_POINTS = tuple((n, s * TTL_MAX_S / 120.0) for n, s in _TTL_SHAPE) - PARK_MIN_TOKENS = int(value("STRATA_SLOT_PARK_MIN_TOKENS", 8192)) PARK_VALUE_RATIO = value("STRATA_SLOT_PARK_VALUE_RATIO", 2.0) PARK_MIN_VALUE = value("STRATA_SLOT_PARK_MIN_VALUE", 0.01) return settings() def settings() -> dict: - return {"ttl_max_s": TTL_MAX_S, "park_min_tokens": PARK_MIN_TOKENS, - "expired_keep_score": EXPIRED_KEEP_SCORE, + return {"ttl_max_s": TTL_MAX_S, "expired_keep_score": EXPIRED_KEEP_SCORE, "idle_value_zero_s": IDLE_VALUE_ZERO_S, "park_value_ratio": PARK_VALUE_RATIO, "park_min_value": PARK_MIN_VALUE} diff --git a/serve/test_slot_alloc.py b/serve/test_slot_alloc.py index a1153c2217b..2410e44715e 100644 --- a/serve/test_slot_alloc.py +++ b/serve/test_slot_alloc.py @@ -545,11 +545,15 @@ def test_side_request_claims_a_blank_slot_and_parked_line_records_it(self): e._on_parked(e._park_claims and next(iter(e._park_claims)) or 0, 50002) self.assertIn(50000, [len(h) for h in e.slot_held]) - def test_continuing_short_or_already_kept_conversations_are_not_parked(self): + def test_continuing_low_value_or_already_kept_conversations_are_not_parked(self): e = self.e() self.assertEqual(e._claim_park(e.main_held + [1, 2, 3]), "") # the same conversation continues - e.main_held = [7] * 5000 - self.assertEqual(e._claim_park([1] * 60), "") # short: cheap to read again + e.main_held = [7] * 2000 + self.assertEqual(e._claim_park([1] * 60), "") # worth less than PARK_MIN_VALUE (no length cut) + self.assertEqual(e.cache_events_view()[0]["reason"], "main_low_value") + e.main_held = [7] * 5000 # short, but worth more: parked + self.assertTrue(e._claim_park([1] * 60).startswith(" park=")) + e = self.e() e.main_held = [7] * 50000 e.slot_held[3] = [7] * 48000 # a slot holds 96% of it already self.assertEqual(e._claim_park([1] * 60), "") @@ -719,8 +723,8 @@ def tearDown(self): slot_policy.configure({}) def test_environment_overrides_and_bad_values_keep_defaults(self): - got = slot_policy.configure({"STRATA_SLOT_TTL_MAX_S": "240", "STRATA_SLOT_PARK_MIN_TOKENS": "x"}) - self.assertEqual(got, {"ttl_max_s": 240.0, "park_min_tokens": 8192, + got = slot_policy.configure({"STRATA_SLOT_TTL_MAX_S": "240", "STRATA_SLOT_PARK_MIN_VALUE": "x"}) + self.assertEqual(got, {"ttl_max_s": 240.0, "expired_keep_score": 0.01, "idle_value_zero_s": 1800.0, "park_value_ratio": 2.0, "park_min_value": 0.01}) self.assertAlmostEqual(slot_policy.cache_ttl(100000), 180.0) # the whole curve doubles @@ -729,8 +733,8 @@ def test_environment_overrides_and_bad_values_keep_defaults(self): self.assertAlmostEqual(slot_policy.cache_ttl(100000), 90.0) def test_engine_reads_them_from_its_environment(self): - e = StrataEngine("missing", [], env={"STRATA_SLOT_PARK_MIN_TOKENS": "20000"}, lazy=True) - self.assertEqual(e.slot_policy_settings["park_min_tokens"], 20000) + e = StrataEngine("missing", [], env={"STRATA_SLOT_PARK_MIN_VALUE": "0.2"}, lazy=True) + self.assertEqual(e.slot_policy_settings["park_min_value"], 0.2) def test_idle_limit_override_disable_and_invalid_values(self): ttl = slot_policy.cache_ttl(50000) From 494f2329b641de653992b5910a8ebd2265c7371d Mon Sep 17 00:00:00 2001 From: andyhrc <12491949+andyhrc@users.noreply.github.com> Date: Fri, 16 Oct 2026 10:50:25 +0800 Subject: [PATCH 20/21] serve: STRATA_SLOT_PARK_KEPT_SHARE (default 0.98) - when a slot's copy counts as already kept A park was skipped when a slot already held 90% of the main session's conversation: the missing tail is read again when the conversation comes back. That tail was up to 10%, and the copy it saves is cheap by comparison. Production parks (24, 10K-170K tokens) fit copy time = 13.2 ms per 1K tokens; a tail read on top of a long context runs at about 1,000-1,500 tok/s (a 3.8K chunk after 77K: ~1,100). The two break even at a tail of 1-2% of the conversation (3-4% at 3,000 tok/s): 100K tokens copy in about 1.2 s, while a 10K tail is read again in 7-10 s. The share is now STRATA_SLOT_PARK_KEPT_SHARE, default 0.98; below it the park goes ahead, normally into that slot (the conversation's own older copy), so no further history is given up. live.slot_policy shows it. The long PARK_MIN_VALUE comment is rewrapped to 120 columns. Co-Authored-By: Claude Opus 5.5 --- docs/BATCHING.md | 10 +++++----- serve/slot_policy.py | 15 ++++++++++----- serve/test_slot_alloc.py | 8 +++++--- 3 files changed, 20 insertions(+), 13 deletions(-) diff --git a/docs/BATCHING.md b/docs/BATCHING.md index 67fff347966..f2024b0816d 100644 --- a/docs/BATCHING.md +++ b/docs/BATCHING.md @@ -494,11 +494,11 @@ tokens. The policy weighs length and age, and every threshold is a parameter: `STRATA_SLOT_IDLE_VALUE_ZERO_S` (default 1800 s; 0 disables decay). Reclaiming and parking use this value; it never changes whether a history can match. - **Park** (with `slot_padding_safe=1`): not when the main session's conversation's value at its idle age is below - `STRATA_SLOT_PARK_MIN_VALUE` (default 0.01, about 2.6K tokens at expiry; no separate length cut-off), or when a - slot holds 90% of it already. Then its own older copy, else an idle expired slot in the - reclaim order above (not worth keeping first, then group load, then the lowest value); a history worth keeping is - taken only when the main session's conversation is worth at least `STRATA_SLOT_PARK_VALUE_RATIO` (default 2) times - more. Never another conversation's protected slot. The parked copy keeps the main session's last-use time. + `STRATA_SLOT_PARK_MIN_VALUE` (default 0.01, about 2.6K tokens at expiry; no separate length cut-off), or when a slot + holds `STRATA_SLOT_PARK_KEPT_SHARE` (default 98%) of it already. Then its own older copy, else an idle expired slot + in the reclaim order above (not worth keeping first, then group load, then the lowest value); a history worth + keeping is taken only when the main session's conversation is worth at least `STRATA_SLOT_PARK_VALUE_RATIO` (default + 2) times more. Never another conversation's protected slot. The parked copy keeps the main session's last-use time. - A busy slot is never matched or overwritten. On an engine without `slot_padding_safe=1`, a slot that would pad over a protected idle history below it costs that history's retention too. diff --git a/serve/slot_policy.py b/serve/slot_policy.py index b08caf19d6a..80d9fd49c90 100644 --- a/serve/slot_policy.py +++ b/serve/slot_policy.py @@ -12,8 +12,11 @@ PARK_VALUE_RATIO = 2.0 # STRATA_SLOT_PARK_VALUE_RATIO: a park takes an expired history worth keeping only when the # main session's conversation is worth at least this many times more; 0: never IDLE_VALUE_ZERO_S = 1800.0 # STRATA_SLOT_IDLE_VALUE_ZERO_S: idle age when expired value reaches zero; 0 disables decay -PARK_MIN_VALUE = 0.01 # STRATA_SLOT_PARK_MIN_VALUE: the main session's conversation is parked only while its value - # (stale_retention at its idle age) is at least this; 0.01 is EXPIRED_KEEP_SCORE's default +PARK_KEPT_SHARE = 0.98 # STRATA_SLOT_PARK_KEPT_SHARE: a slot holding at least this share of the main session's + # conversation already keeps it (no park); the rest is read again when it comes back +PARK_MIN_VALUE = 0.01 # STRATA_SLOT_PARK_MIN_VALUE: the main session's conversation is parked only while its + # value (stale_retention at its idle age) is at least this; 0.01 is EXPIRED_KEEP_SCORE's + # default TTL_MAX_S = 120.0 # STRATA_SLOT_TTL_MAX_S: protection at 262144 tokens; the whole curve scales with it _TTL_SHAPE = ((0, 0.0), (500, 0.1), (2000, 2.0), (8000, 10.0), (32000, 45.0), (50000, 65.0), (100000, 90.0), (MAX_CACHE_TOKENS, 120.0)) @@ -23,7 +26,7 @@ def configure(env) -> dict: """Read the tunables from `env` (the engine's environment); bad or missing values keep the defaults.""" global TTL_MAX_S, TTL_POINTS, EXPIRED_KEEP_SCORE, IDLE_VALUE_ZERO_S, \ - PARK_VALUE_RATIO, PARK_MIN_VALUE + PARK_VALUE_RATIO, PARK_MIN_VALUE, PARK_KEPT_SHARE def value(name, default): try: @@ -37,13 +40,14 @@ def value(name, default): TTL_POINTS = tuple((n, s * TTL_MAX_S / 120.0) for n, s in _TTL_SHAPE) PARK_VALUE_RATIO = value("STRATA_SLOT_PARK_VALUE_RATIO", 2.0) PARK_MIN_VALUE = value("STRATA_SLOT_PARK_MIN_VALUE", 0.01) + PARK_KEPT_SHARE = value("STRATA_SLOT_PARK_KEPT_SHARE", 0.98) return settings() def settings() -> dict: return {"ttl_max_s": TTL_MAX_S, "expired_keep_score": EXPIRED_KEEP_SCORE, "idle_value_zero_s": IDLE_VALUE_ZERO_S, "park_value_ratio": PARK_VALUE_RATIO, - "park_min_value": PARK_MIN_VALUE} + "park_min_value": PARK_MIN_VALUE, "park_kept_share": PARK_KEPT_SHARE} def cache_ttl(tokens: int) -> float: @@ -158,7 +162,8 @@ def park_destination(main: list[int], busy: list[bool], held: list[list[int]], u gs = max(1, group_size) reliable = reliable_flags(busy, valid, gs, padding_safe) for b, h in enumerate(held): # already kept (all but a short tail) - in an idle slot or one decoding it - if h and reliable[b] and len(h) * 10 >= len(main) * 9 and (main[:len(h)] == h or h[:len(main)] == main): + if (h and reliable[b] and len(h) >= PARK_KEPT_SHARE * len(main) and + (main[:len(h)] == h or h[:len(main)] == main)): return None states, blank = _blank(held, busy, used, now, reliable) positions = {b: i for i, b in enumerate(order)} diff --git a/serve/test_slot_alloc.py b/serve/test_slot_alloc.py index 2410e44715e..b77f9777ba1 100644 --- a/serve/test_slot_alloc.py +++ b/serve/test_slot_alloc.py @@ -555,8 +555,10 @@ def test_continuing_low_value_or_already_kept_conversations_are_not_parked(self) self.assertTrue(e._claim_park([1] * 60).startswith(" park=")) e = self.e() e.main_held = [7] * 50000 - e.slot_held[3] = [7] * 48000 # a slot holds 96% of it already + e.slot_held[3] = [7] * 49000 # a slot holds 98% of it already self.assertEqual(e._claim_park([1] * 60), "") + e.slot_held[3] = [7] * 48000 # 96%: the copy is cheaper than the 2K tail + self.assertEqual(e._claim_park([1] * 60), " park=3") # into its own older copy def test_parking_inherits_the_main_history_last_use_and_does_not_renew_its_value(self): e = self.e() @@ -624,7 +626,7 @@ def test_zero_value_main_can_still_continue_without_a_park_or_deletion(self): def test_an_invalid_copy_does_not_count_as_already_kept(self): e = self.e() - e.slot_held[3] = [7] * 48000 # 96% of the main conversation, but no longer usable + e.slot_held[3] = [7] * 49000 # 98% of the main conversation, but no longer usable e.slot_cache_valid[3] = False self.assertIsNotNone(self.dest(e)) e.slot_cache_valid[3] = True @@ -726,7 +728,7 @@ def test_environment_overrides_and_bad_values_keep_defaults(self): got = slot_policy.configure({"STRATA_SLOT_TTL_MAX_S": "240", "STRATA_SLOT_PARK_MIN_VALUE": "x"}) self.assertEqual(got, {"ttl_max_s": 240.0, "expired_keep_score": 0.01, "idle_value_zero_s": 1800.0, "park_value_ratio": 2.0, - "park_min_value": 0.01}) + "park_min_value": 0.01, "park_kept_share": 0.98}) self.assertAlmostEqual(slot_policy.cache_ttl(100000), 180.0) # the whole curve doubles self.assertAlmostEqual(slot_policy.cache_ttl(10 ** 6), 240.0) self.assertEqual(slot_policy.configure({})["ttl_max_s"], 120.0) From df1d87bd23e6e796d29677a2f2b28e74223b8b52 Mon Sep 17 00:00:00 2001 From: andyhrc <12491949+andyhrc@users.noreply.github.com> Date: Sat, 17 Oct 2026 03:45:50 +0800 Subject: [PATCH 21/21] serve: STRATA_SLOT_SIBLING_SHARE (default 0.98) - an older version of the same conversation is replaced in place An idle history that shares at least this share of its own length with the prompt (or with the main session's conversation, for a park) but is not its start is the same conversation with another end: the client rewrote it. Claude Code's safety classifier does this on every query - it inserts the new tool calls before the closing text of its transcript - so no query continues the last one. Kept side by side, the versions took 4-5 of the 6 slots (2026-10-16) and pushed other conversations out, while a classifier query that ran on the main session was not parked at all (the versions in the slots were worth as much as it, so the value ratio refused them). place(): with no slot continuing the prompt, the version sharing the most is the destination (reason "sibling"), protected or not, before the reclaim order. park_destination(): after its own older copy, before the expired candidates and the value ratio. 0 turns it off. Co-Authored-By: Claude Opus 5.5 --- docs/BATCHING.md | 9 ++++++- serve/slot_policy.py | 55 ++++++++++++++++++++++++++++++++++---- serve/test_slot_alloc.py | 57 +++++++++++++++++++++++++++++++++++++++- 3 files changed, 114 insertions(+), 7 deletions(-) diff --git a/docs/BATCHING.md b/docs/BATCHING.md index f2024b0816d..cca103e8607 100644 --- a/docs/BATCHING.md +++ b/docs/BATCHING.md @@ -485,6 +485,12 @@ tokens. The policy weighs length and age, and every threshold is a parameter: expired or blank slot: first one not worth keeping (blank, or its score at freshness 0 below `STRATA_SLOT_EXPIRED_KEEP_SCORE`, default 0.01 - about 2.5K tokens), then by group load, a blank slot before a history, and the lowest score. +- **An older version of the same conversation** - an idle history that shares at least + `STRATA_SLOT_SIBLING_SHARE` (default 98%; 0: off) of its own length with the prompt but is not its start (the client + rewrote its end) - is replaced in place, protected or not, when no slot's history continues the prompt. Claude + Code's safety classifier inserts each query's new tool calls before the closing text of its transcript, so no + query continues the previous one; kept side by side, the versions took 4-5 of the 6 slots on the machine above and + pushed the other conversations out. - **No wait**: when every idle history is still protected, the request overwrites the one that loses least at once (losses within 10% go by group load). An earlier version waited up to 5 s first, once per request; with more requests than slots each wait ran its full 5 s, the slot idle meanwhile, and ended in the same overwrite (7 of 7 in @@ -495,7 +501,8 @@ tokens. The policy weighs length and age, and every threshold is a parameter: changes whether a history can match. - **Park** (with `slot_padding_safe=1`): not when the main session's conversation's value at its idle age is below `STRATA_SLOT_PARK_MIN_VALUE` (default 0.01, about 2.6K tokens at expiry; no separate length cut-off), or when a slot - holds `STRATA_SLOT_PARK_KEPT_SHARE` (default 98%) of it already. Then its own older copy, else an idle expired slot + holds `STRATA_SLOT_PARK_KEPT_SHARE` (default 98%) of it already. Then its own older copy, else an older version of + it (`STRATA_SLOT_SIBLING_SHARE` above, protected or not; the most shared first), else an idle expired slot in the reclaim order above (not worth keeping first, then group load, then the lowest value); a history worth keeping is taken only when the main session's conversation is worth at least `STRATA_SLOT_PARK_VALUE_RATIO` (default 2) times more. Never another conversation's protected slot. The parked copy keeps the main session's last-use time. diff --git a/serve/slot_policy.py b/serve/slot_policy.py index 80d9fd49c90..798f49fa219 100644 --- a/serve/slot_policy.py +++ b/serve/slot_policy.py @@ -14,6 +14,10 @@ IDLE_VALUE_ZERO_S = 1800.0 # STRATA_SLOT_IDLE_VALUE_ZERO_S: idle age when expired value reaches zero; 0 disables decay PARK_KEPT_SHARE = 0.98 # STRATA_SLOT_PARK_KEPT_SHARE: a slot holding at least this share of the main session's # conversation already keeps it (no park); the rest is read again when it comes back +SIBLING_SHARE = 0.98 # STRATA_SLOT_SIBLING_SHARE: an idle history that shares at least this share of its own + # length with the conversation being placed or parked is an older version of it (the + # client rewrote its end: a classifier query inserts before its closing text); it is + # replaced in place, even while protected; 0: off PARK_MIN_VALUE = 0.01 # STRATA_SLOT_PARK_MIN_VALUE: the main session's conversation is parked only while its # value (stale_retention at its idle age) is at least this; 0.01 is EXPIRED_KEEP_SCORE's # default @@ -26,7 +30,7 @@ def configure(env) -> dict: """Read the tunables from `env` (the engine's environment); bad or missing values keep the defaults.""" global TTL_MAX_S, TTL_POINTS, EXPIRED_KEEP_SCORE, IDLE_VALUE_ZERO_S, \ - PARK_VALUE_RATIO, PARK_MIN_VALUE, PARK_KEPT_SHARE + PARK_VALUE_RATIO, PARK_MIN_VALUE, PARK_KEPT_SHARE, SIBLING_SHARE def value(name, default): try: @@ -41,13 +45,14 @@ def value(name, default): PARK_VALUE_RATIO = value("STRATA_SLOT_PARK_VALUE_RATIO", 2.0) PARK_MIN_VALUE = value("STRATA_SLOT_PARK_MIN_VALUE", 0.01) PARK_KEPT_SHARE = value("STRATA_SLOT_PARK_KEPT_SHARE", 0.98) + SIBLING_SHARE = value("STRATA_SLOT_SIBLING_SHARE", 0.98) return settings() def settings() -> dict: return {"ttl_max_s": TTL_MAX_S, "expired_keep_score": EXPIRED_KEEP_SCORE, "idle_value_zero_s": IDLE_VALUE_ZERO_S, "park_value_ratio": PARK_VALUE_RATIO, - "park_min_value": PARK_MIN_VALUE, "park_kept_share": PARK_KEPT_SHARE} + "park_min_value": PARK_MIN_VALUE, "park_kept_share": PARK_KEPT_SHARE, "sibling_share": SIBLING_SHARE} def cache_ttl(tokens: int) -> float: @@ -121,6 +126,34 @@ def superseded_slots(held: list[list[int]], busy: list[bool], reliable: list[boo return out +def common_prefix(a: list[int], b: list[int]) -> int: + """How many leading tokens `a` and `b` share (each step compares only the part not yet known to match).""" + lo, hi = 0, min(len(a), len(b)) + while lo < hi: + mid = (lo + hi + 1) // 2 + if a[lo:mid] == b[lo:mid]: + lo = mid + else: + hi = mid - 1 + return lo + + +def siblings(tokens: list[int], held: list[list[int]], slots) -> dict[int, int]: + """The slots among `slots` whose history is an older version of `tokens`: it shares at least SIBLING_SHARE of + its own length with them but is not their start (a start is a cache source, or the conversation's own older + copy). Slot -> shared tokens.""" + if SIBLING_SHARE <= 0: + return {} + out = {} + for b in slots: + h = held[b] + if h and not (len(h) < len(tokens) and tokens[:len(h)] == h): + c = common_prefix(tokens, h) + if c < len(h) and c >= SIBLING_SHARE * len(h): + out[b] = c + return out + + def _blank(held, busy, used, now, reliable): superseded = superseded_slots(held, busy, reliable) states = [cache_state(len(h), used[b], now, reliable[b] and b not in superseded) for b, h in enumerate(held)] @@ -154,9 +187,10 @@ def park_destination(main: list[int], busy: list[bool], held: list[list[int]], u order: list[int], group_size: int, now: float, valid: list[bool] | None = None, padding_safe: bool = False, exclude=(), main_value: float = 0.0) -> int | None: """An idle slot to keep the main session's conversation in before another request replaces it (engine park=): - its own older copy, else an expired slot in place()'s reclaim order (reclaim_slot), where a history worth keeping - counts only when `main_value` (the main session's current value) is at least PARK_VALUE_RATIO times its own. - Never a protected slot. None: nowhere to keep it (or it is already kept).""" + its own older copy, else an older version of it (siblings(): the same conversation with another end, protected + or not), else an expired slot in place()'s reclaim order (reclaim_slot), where a history worth keeping counts + only when `main_value` (the main session's current value) is at least PARK_VALUE_RATIO times its own. Never + another conversation's protected slot. None: nowhere to keep it (or it is already kept).""" if not padding_safe or not main: return None gs = max(1, group_size) @@ -172,6 +206,9 @@ def park_destination(main: list[int], busy: list[bool], held: list[list[int]], u len(held[b]) < len(main) and main[:len(held[b])] == held[b]] if own: return max(own, key=lambda b: (len(held[b]), -positions[b])) + older = siblings(main, held, [b for b in order if not busy[b] and b not in exclude and reliable[b]]) + if older: + return max(older, key=lambda b: (older[b], -positions[b])) values = _slot_values(held, states, blank) candidates = [b for b in order if not busy[b] and b not in exclude and states[b]["expired"] and (not _worth_keeping(values[b]) or @@ -228,6 +265,14 @@ def pipeline(b): source = max(sources, key=lambda b: (matches[b], -positions[b])) return Placement(slot, source, best, near, "reuse") + # An older version of this conversation (siblings(): its client rewrote the end) is replaced in place, protected + # or not: kept beside it, each version of a classifier query took a slot of its own and pushed other conversations + # out (2026-10-16: 4-5 of 6 slots). The longest shared part first, then group load. + older = siblings(prompt, held, [b for b in free if reliable[b] and collateral(b) == 0]) + if older: + slot = min(older, key=lambda b: (-older[b], *pipeline(b)[:2], positions[b])) + return Placement(slot, reason="sibling") + safe = [b for b in free if states[b]["expired"] and collateral(b) == 0] values = _slot_values(held, states, blank) diff --git a/serve/test_slot_alloc.py b/serve/test_slot_alloc.py index b77f9777ba1..ba5faf0079c 100644 --- a/serve/test_slot_alloc.py +++ b/serve/test_slot_alloc.py @@ -353,6 +353,37 @@ def test_two_idle_conversations_do_not_overwrite_each_other(self): e.slot_held[slot] = list(h) e.slot_used[slot] = self.NOW - 300 + def test_an_older_version_of_the_prompt_is_replaced_in_place_even_while_protected(self): + # 2026-10-16: each Claude Code classifier query inserts its new tool calls before the closing text of the + # last one's transcript, so it is no slot's continuation; every version took a blank slot of its own + e = self.e() + transcript = [1] * 42000 + e.slot_held[3] = transcript + [8] * 60 + [9] * 40 # the last query, protected (just used) + e.slot_held[0] = [2] * 30000 # another chat + prompt = transcript + [8] * 60 + [7] * 300 + [9] * 40 + plan = self.place(e, prompt, padding_safe=True) + self.assertEqual((plan.slot, plan.reason), (3, "sibling")) + other = [1] * 20000 + [5] * 22000 # shares under 98% of slot 3: a blank slot + self.assertEqual(self.place(e, other, padding_safe=True).reason, "reclaim") + self.assertEqual(self.place(e, prompt + [4], padding_safe=True).slot, 3) + e.slot_held[5] = transcript[:40000] # a full-prefix source still decides first + self.assertEqual(self.place(e, prompt, padding_safe=True).reason, "reuse") + e.slot_held[5] = [] + e.slot_busy[3] = True # a busy slot is never replaced + self.assertEqual(self.place(e, prompt, padding_safe=True).reason, "reclaim") + e.slot_busy[3] = False + slot_policy.configure({"STRATA_SLOT_SIBLING_SHARE": "0"}) + try: + self.assertEqual(self.place(e, prompt, padding_safe=True).reason, "reclaim") # 0: off + finally: + slot_policy.configure({}) + + def test_common_prefix(self): + for a, b, n in (([], [1], 0), ([1, 2, 3], [1, 2, 3], 3), ([1, 2, 3], [1, 2], 2), ([1, 2, 3], [1, 5, 3], 1), + (list(range(100000)), list(range(77777)) + [0], 77777)): + self.assertEqual(slot_policy.common_prefix(a, b), n) + self.assertEqual(slot_policy.common_prefix(b, a), n) + def test_match_moves_only_into_a_blank_slot_of_a_less_loaded_group(self): e = self.e() e.slot_busy[0] = True @@ -637,6 +668,29 @@ def test_own_older_copy_is_refreshed_before_using_a_blank_slot(self): e.slot_held[5] = [7] * 30000 # this conversation, a few turns back self.assertEqual(self.dest(e), 5) + def test_an_older_version_is_replaced_by_the_park_protected_or_not(self): + # the main session ran a classifier query; slot 2 holds the previous one (the same transcript with another + # end), worth about as much as the main one, so the value ratio would never let the park take it + e = self.e() + e.slot_held = [[b] * 40000 for b in range(6)] # every slot holds another fresh conversation + e.slot_held[2] = e.main_held[:49900] + [3] * 80 + self.assertEqual(self.dest(e), 2) + e.slot_held[4] = e.main_held[:49950] + [3] * 30 # the version sharing more goes first + self.assertEqual(self.dest(e), 4) + self.assertEqual(self.dest(e, exclude=(4,)), 2) + e.slot_held[1] = e.main_held[:30000] # its own older copy is still preferred + self.assertEqual(self.dest(e), 1) + e.slot_held[1] = [1] * 40000 + e.slot_held[2] = e.main_held[:40000] + [3] * 9000 # 82%: another conversation + e.slot_held[4] = e.main_held[:40000] + [4] * 9000 + self.assertIsNone(self.dest(e)) + e.slot_held[2] = e.main_held[:49900] + [3] * 80 + slot_policy.configure({"STRATA_SLOT_SIBLING_SHARE": "0"}) + try: + self.assertIsNone(self.dest(e)) # 0: off + finally: + slot_policy.configure({}) + def test_nowhere_to_park_without_evicting(self): e = self.e() e.slot_held = [[b] * 40000 for b in range(6)] # every slot holds another fresh conversation @@ -728,7 +782,8 @@ def test_environment_overrides_and_bad_values_keep_defaults(self): got = slot_policy.configure({"STRATA_SLOT_TTL_MAX_S": "240", "STRATA_SLOT_PARK_MIN_VALUE": "x"}) self.assertEqual(got, {"ttl_max_s": 240.0, "expired_keep_score": 0.01, "idle_value_zero_s": 1800.0, "park_value_ratio": 2.0, - "park_min_value": 0.01, "park_kept_share": 0.98}) + "park_min_value": 0.01, "park_kept_share": 0.98, + "sibling_share": 0.98}) self.assertAlmostEqual(slot_policy.cache_ttl(100000), 180.0) # the whole curve doubles self.assertAlmostEqual(slot_policy.cache_ttl(10 ** 6), 240.0) self.assertEqual(slot_policy.configure({})["ttl_max_s"], 120.0)