From 637ce97211ed07e5242226e4c575145d21acf60e Mon Sep 17 00:00:00 2001 From: Pc1 Date: Sun, 4 Oct 2026 15:53:30 +0200 Subject: [PATCH 1/2] file tier: unbuffered reads on Linux too (O_DIRECT + the kernel's asynchronous reads) #286's unbuffered file tier existed on Windows alone: on Linux the experts outside the RAM copy were always read through the page cache, as page faults of the mapped GGUF - small requests on the critical path, and pages that a 32 GB PC cannot keep beside a 16 GiB RAM copy anyway, so they were read again and again. - open_direct opens the files with O_DIRECT; read_direct queues every 4 KiB-aligned window of a batch with one io_submit (linux/aio_abi.h, no library; allowed by Docker's default seccomp profile, unlike io_uring) and waits for all of them, each thread with its own context and aligned buffer. Windows merges windows up to 1 MiB apart for NTFS; here only windows that touch are merged. A refused submit or a failed read falls back to the mapping, as on Windows. - The same decision as Windows' `cache_counts = false` arm: unbuffered when the file cache could not keep the expert bytes outside the RAM copy (platform::file_cache_keeps, from MemAvailable and the tightest cgroup limit), re-checked once the copy is built (#577). STRATA_UNBUFFERED_LOAD=1 / 0 forces it either way. A pack's experts.bin works too (its Linux open now records the path). - direct_fallbacks() counts the blobs an unbuffered read could not deliver. - Tests: file_expert_source_test reads a synthetic pack unbuffered - one blob at a time, a batch of adjacent experts, copy_blob - and requires the file's bytes and no fallback (a mutation that fails every read is caught). ctest: 63 of 66 pass; the three others need a Q2_0 pack, a higher `ulimit -l` and AVX-512, as before. Measured on an RTX 3080 Ti Laptop 16 GB, i9-12900H, 30 GB RAM, KIOXIA PCIe 4 NVMe, Linux 7.0, ext4: Qwen3.8-Flash-Next Abliterated Q4_K_M in place (--native), --resident-budget-gib 16, --expert-cache auto, 8K context, greedy; three prompts (code, Russian prose, a 7.8K-token summary), each with and without MTP, two rounds of buffered (STRATA_UNBUFFERED_LOAD=0) against unbuffered, the same binary. Decode 4.38 -> 7.90 tok/s on average (x1.80; x1.49-x2.01 per prompt and mode), the 7.8K prompt read at 154/172 -> 184/290 tok/s, startup 36 -> 18-21 s. The code prompt's answer is the same text in all 16 runs; the other two differ between runs of either arm (the adaptive GPU cache changes which experts the CPU computes), as before. Co-Authored-By: Claude Opus 5.5 --- include/strata/core/expert_source.hpp | 12 +- src/core/expert_source.cpp | 227 +++++++++++++++++++++++-- src/program/generate.cpp | 6 +- tests/core/file_expert_source_test.cpp | 77 ++++++++- 4 files changed, 303 insertions(+), 19 deletions(-) diff --git a/include/strata/core/expert_source.hpp b/include/strata/core/expert_source.hpp index 8fbc4a58cd2..0cf00050ded 100644 --- a/include/strata/core/expert_source.hpp +++ b/include/strata/core/expert_source.hpp @@ -499,7 +499,8 @@ class FileExpertSource : public ExpertSource { double file_ms() const { return (double) file_us_.load(std::memory_order_relaxed) / 1000.0; } /// Threads `prefetch` reads the GGUF with (STRATA_FETCH_THREADS, default 8). void set_fetch_threads(int n) { fetch_threads_ = n < 1 ? 1 : n; } - /// #286 (Windows): read the experts straight from the drive (FILE_FLAG_NO_BUFFERING, overlapped) instead of + /// #286: read the experts straight from the drive (Windows: FILE_FLAG_NO_BUFFERING, overlapped; Linux: O_DIRECT and + /// the kernel's asynchronous reads) instead of /// through the mapped files - the GGUF in place, or a pack's experts.bin - when the file cache could not keep them /// beside `ram_bytes` (the RAM budget), cached now or not (their mapped pages would land in the working set); see /// experts_unbuffered. The mapped reads' page faults are one small request each, and the pages they bring in @@ -534,6 +535,8 @@ class FileExpertSource : public ExpertSource { /// Of the blobs the file tier read for the decode, how many had been warmed for their layer beforehand. int64_t warmed_hits() const { return warm_hits_.load(std::memory_order_relaxed); } int64_t warmed() const { return warm_count_.load(std::memory_order_relaxed); } + /// #286: blobs an unbuffered read could not deliver, read through the mapping instead (0 when all went direct). + int64_t direct_fallbacks() const { return direct_fallbacks_.load(std::memory_order_relaxed); } /// Blobs touched, for the driver to report. With `h = 0` this is `48 * k` per token and the number is only /// interesting once Phase 3 makes it not so. @@ -551,12 +554,12 @@ class FileExpertSource : public ExpertSource { struct Fill { size_t v; int64_t layer, e; uint8_t* dst; }; /// The claimed buffers' blobs: one overlapped batch when unbuffered, else the fetch threads' mapped copies. void fill_many(const std::vector& todo); - /// #286: the blobs from the drive, unbuffered: every role's 4 KiB-aligned window is read at once (overlapped) - /// into this thread's aligned buffer, then copied into place. False when a read fails. + /// #286: the blobs from the drive, unbuffered: every role's 4 KiB-aligned window is read at once (overlapped on + /// Windows, io_submit on Linux) into this thread's aligned buffer, then copied into place. False when a read fails. bool read_direct(const Fill* fills, size_t n) const; bool open_direct(std::string& why); std::vector paths_; ///< the mapped files, as maps_ - std::vector direct_; ///< #286: per file, an unbuffered overlapped handle (Windows) + std::vector direct_; ///< #286: per file, an unbuffered handle (Windows) or O_DIRECT fd (Linux) std::vector role_file_; ///< 3 x n_layers: index into maps_ / direct_ /// blobs assembled in the stage buffers (`blob` hands those out): the GGUF in place, or any unbuffered source bool staged() const { return !role_ptr_.empty() || !direct_.empty(); } @@ -590,6 +593,7 @@ class FileExpertSource : public ExpertSource { std::atomic file_blob_bytes_{0}, file_us_{0}; std::unique_ptr[]> warm_stamp_; ///< per (layer, expert): epoch_ + 1 when warmed std::atomic warm_hits_{0}, warm_count_{0}; + mutable std::atomic direct_fallbacks_{0}; std::unordered_map stage_of_; uint64_t stage_blob_ = 0; uint64_t stage_seq_ = 0; diff --git a/src/core/expert_source.cpp b/src/core/expert_source.cpp index 98e1952d67c..b5922e1c559 100644 --- a/src/core/expert_source.cpp +++ b/src/core/expert_source.cpp @@ -44,6 +44,11 @@ #include #include #endif +#if defined(__linux__) +#include +#include // the kernel's own asynchronous reads (io_submit): no library, allowed by Docker's seccomp +#include +#endif // a 64-bit seek (as in pinned.cu): the 32-bit `fseek` wraps past 4 GiB, and the spelling differs per platform #ifndef STRATA_FSEEK64 @@ -331,6 +336,49 @@ bool available_memory_bytes(uint64_t& bytes) { return bytes > 0; } +/// #286: an unbuffered file handle - a HANDLE on Windows, an O_DIRECT descriptor (as a pointer-sized integer) on Linux. +void close_direct(void* h) { +#if defined(_WIN32) + CloseHandle((HANDLE) h); +#else + ::close((int) (intptr_t) h); +#endif +} + +/// #286/#577: whether the file tier reads its `read_bytes` unbuffered, beside `arena_bytes` of RAM copy still to be +/// taken. Windows times probe reads too (experts_unbuffered); on Linux the same rule as its `cache_counts = false` +/// arm decides: whether the file cache could keep those bytes at all, from MemAvailable and the tightest cgroup +/// limit (host_available_memory). STRATA_UNBUFFERED_LOAD=1 / 0 forces it either way, as on Windows. +bool file_tier_unbuffered(const std::vector& paths, uint64_t arena_bytes, uint64_t read_bytes, + std::string& why) { +#if defined(_WIN32) + return experts_unbuffered(paths, arena_bytes, why, /*cache_counts=*/false, read_bytes); +#elif defined(__linux__) + (void) paths; + if (const char* env = std::getenv("STRATA_UNBUFFERED_LOAD"); env != nullptr && env[0] != '\0') { + why = std::string("STRATA_UNBUFFERED_LOAD=") + env; + return env[0] != '0'; + } + uint64_t avail = 0; + if (!available_memory_bytes(avail)) { + why = "the available RAM is unknown"; + return false; + } + const bool keepable = strata::platform::file_cache_keeps(avail, arena_bytes, read_bytes); + char msg[256]; + std::snprintf(msg, sizeof msg, "%.1f GiB available, %.1f GiB of it still to be taken by the RAM copy, %.1f GiB " + "of experts read from the files: the file cache %s keep them", + (double) avail / (1ull << 30), (double) arena_bytes / (1ull << 30), (double) read_bytes / (1ull << 30), + keepable ? "can" : "cannot"); + why = msg; + return !keepable; +#else + (void) paths; (void) arena_bytes; (void) read_bytes; + why = "through the file cache (no unbuffered reads on this platform)"; + return false; +#endif +} + } // namespace // ================================ THE FILE-BACKED SOURCE ================================ @@ -496,6 +544,7 @@ bool FileExpertSource::open(const std::string& pack_dir, int64_t n_layers, int64 if (view == MAP_FAILED) { ::close(fd); err = "FileExpertSource: mmap failed on " + path; return false; } fd_ = fd; base_ = (const uint8_t*) view; + paths_.assign(1, path); // #286: open_direct opens it again with O_DIRECT #endif blobs_ = (int64_t) blob_count; n_layers_ = n_layers; @@ -558,9 +607,7 @@ void FileExpertSource::close() { role_bytes_.clear(); role_file_.clear(); paths_.clear(); -#if defined(_WIN32) - for (void* h : direct_) CloseHandle((HANDLE) h); -#endif + for (void* h : direct_) close_direct(h); direct_.clear(); { std::lock_guard lk(stage_mu_); @@ -702,6 +749,7 @@ bool FileExpertSource::copy_from_files(int64_t layer, int64_t expert, uint8_t* d if (!direct_.empty()) { // #286: from the drive; a failed read falls back to the mapping below const Fill f{0, layer, expert, dst}; if (read_direct(&f, 1)) return true; + direct_fallbacks_.fetch_add(1, std::memory_order_relaxed); } if (!role_ptr_.empty()) { uint64_t at = 0; @@ -948,6 +996,23 @@ bool FileExpertSource::open_direct(std::string& why) { for (uint64_t b : layer_blob_bytes_) stage_blob_ = std::max(stage_blob_, b); } return true; +#elif defined(__linux__) + // O_DIRECT: the reads bypass the page cache, as FILE_FLAG_NO_BUFFERING does (4 KiB-aligned windows, read_direct) + for (const std::string& path : paths_) { + const int fd = ::open(path.c_str(), O_RDONLY | O_DIRECT | O_CLOEXEC); + if (fd < 0) { + why += "; cannot open " + path + " with O_DIRECT (" + std::strerror(errno) + "), read through the file cache"; + for (void* d : direct_) close_direct(d); + direct_.clear(); + return false; + } + direct_.push_back((void*) (intptr_t) fd); + } + if (role_ptr_.empty()) { // experts.bin: blob() now assembles into the stage buffers, sized for the largest blob + std::lock_guard lk(stage_mu_); + for (uint64_t b : layer_blob_bytes_) stage_blob_ = std::max(stage_blob_, b); + } + return true; #else (void) why; return false; @@ -955,7 +1020,7 @@ bool FileExpertSource::open_direct(std::string& why) { } bool FileExpertSource::set_unbuffered(uint64_t ram_bytes, std::string& why) { -#if defined(_WIN32) +#if defined(_WIN32) || defined(__linux__) if (base_ == nullptr || paths_.empty() || !direct_.empty()) { why = !direct_.empty() ? "already unbuffered" : "no expert files open"; return !direct_.empty(); @@ -964,17 +1029,17 @@ bool FileExpertSource::set_unbuffered(uint64_t ram_bytes, std::string& why) { // PLE table), and the copy holds at most every expert - the budget asked for can be more than that const uint64_t experts = expert_bytes(); const uint64_t arena = std::min(ram_bytes, experts); - if (!experts_unbuffered(paths_, arena, why, /*cache_counts=*/false, experts - arena)) return false; + if (!file_tier_unbuffered(paths_, arena, experts - arena, why)) return false; return open_direct(why); #else (void) ram_bytes; - why = "through the file cache (not Windows)"; + why = "through the file cache (no unbuffered reads on this platform)"; return false; #endif } bool FileExpertSource::recheck_unbuffered(std::string& why) { -#if defined(_WIN32) +#if defined(_WIN32) || defined(__linux__) if (const char* env = std::getenv("STRATA_UNBUFFERED_LOAD"); env != nullptr && env[0] != '\0') { why = std::string("STRATA_UNBUFFERED_LOAD=") + env; return !direct_.empty(); @@ -989,7 +1054,7 @@ bool FileExpertSource::recheck_unbuffered(std::string& why) { const uint64_t experts = expert_bytes(); const uint64_t read = experts > complement_bytes_ ? experts - complement_bytes_ : 0; std::string w; - const bool ub = experts_unbuffered(paths_, 0, w, /*cache_counts=*/false, read); + const bool ub = file_tier_unbuffered(paths_, 0, read, w); char head[96]; std::snprintf(head, sizeof head, "re-checked with the RAM copy built (%.2f GiB): ", (double) complement_bytes_ / 1073741824.0); @@ -997,11 +1062,11 @@ bool FileExpertSource::recheck_unbuffered(std::string& why) { if (ub == !direct_.empty()) return ub; if (ub) return open_direct(why); // through the file cache after all: the mapped reads take over (staged() stays true for the GGUF in place) - for (void* d : direct_) CloseHandle((HANDLE) d); + for (void* d : direct_) close_direct(d); direct_.clear(); return false; #else - why = "through the file cache (not Windows)"; + why = "through the file cache (no unbuffered reads on this platform)"; return false; #endif } @@ -1120,6 +1185,148 @@ bool FileExpertSource::read_direct(const Fill* fills, size_t n) const { std::memcpy(x.dst + x.at, sc.buf + sc.req[x.req].pos + x.in_req + x.skip, (size_t) x.n); } return true; +#elif defined(__linux__) + // Linux: O_DIRECT descriptors (open_direct) and the kernel's asynchronous reads - every window of the batch is + // queued with one io_submit before the first wait, so the drive sees the batch as one queue, as the overlapped + // ReadFiles above do. No NTFS here: windows are merged only where they touch (the 4 KiB rounding of adjacent + // slices, consecutive experts of a role), never across a gap that would be read for nothing. Each thread has its + // own context and aligned buffer, kept for its next batch, so concurrent batches never see each other's events. + constexpr uint64_t kSector = 4096, kMerge = 32ull << 20; + constexpr long kDepth = 256; // requests in flight per thread; a larger batch is read in waves of this many + struct Window { int file; uint64_t a0, size, skip, n, at; uint8_t* dst; size_t req; uint64_t in_req; }; + struct Req { int fd; uint64_t a0, size, pos; }; + struct Scratch { + uint8_t* buf = nullptr; + size_t cap = 0; + aio_context_t ctx = 0; + std::vector win; + std::vector order; + std::vector req; + std::vector got; + std::vector cb; + std::vector cbp; + std::vector ev; + ~Scratch() { + std::free(buf); + if (ctx != 0) (void) syscall(SYS_io_destroy, ctx); + } + }; + thread_local Scratch sc; + if (sc.ctx == 0 && syscall(SYS_io_setup, kDepth, &sc.ctx) != 0) { + sc.ctx = 0; + return false; // no AIO (a seccomp profile, aio-max-nr): the caller falls back to the mapped copy + } + sc.win.clear(); + for (size_t k = 0; k < n; ++k) { + const Fill& f = fills[k]; + if (f.dst == nullptr || f.layer < 0 || f.e < 0 || f.layer >= n_layers_ || f.e >= n_expert_) return false; + if (role_ptr_.empty()) { // experts.bin: the blob is one contiguous range + const uint64_t per = layer_blob_bytes_[(size_t) f.layer]; + const uint64_t off = layer_offsets_[(size_t) f.layer] + (uint64_t) f.e * per; + const uint64_t a0 = off / kSector * kSector, a1 = (off + per + kSector - 1) / kSector * kSector; + sc.win.push_back({0, a0, a1 - a0, off - a0, per, 0, f.dst, 0, 0}); + continue; + } + uint64_t at = 0; + for (int r = 0; r < 3; ++r) { + const size_t i = (size_t) (3 * f.layer + r); + const uint64_t per = role_bytes_[i]; + const Map& m = maps_[(size_t) role_file_[i]]; + const uint64_t off = (uint64_t) (role_ptr_[i] - m.base) + (uint64_t) f.e * per; + const uint64_t a0 = off / kSector * kSector, a1 = (off + per + kSector - 1) / kSector * kSector; + sc.win.push_back({role_file_[i], a0, a1 - a0, off - a0, per, at, f.dst, 0, 0}); + at += per; + } + } + sc.order.resize(sc.win.size()); + for (size_t w = 0; w < sc.win.size(); ++w) sc.order[w] = w; + std::sort(sc.order.begin(), sc.order.end(), [&](size_t a, size_t b) { + return sc.win[a].file != sc.win[b].file ? sc.win[a].file < sc.win[b].file : sc.win[a].a0 < sc.win[b].a0; + }); + sc.req.clear(); + uint64_t total = 0; + int last_file = -1; + for (size_t w : sc.order) { + Window& x = sc.win[w]; + if (!sc.req.empty() && x.file == last_file) { + Req& q = sc.req.back(); + const uint64_t end = q.a0 + q.size, xend = x.a0 + x.size; + if (x.a0 <= end && std::max(end, xend) - q.a0 <= kMerge) { + if (xend > end) { + total += xend - end; + q.size = xend - q.a0; + } + x.req = sc.req.size() - 1; + x.in_req = x.a0 - q.a0; + continue; + } + } + sc.req.push_back({(int) (intptr_t) direct_[(size_t) x.file], x.a0, x.size, total}); + total += x.size; + last_file = x.file; + x.req = sc.req.size() - 1; + x.in_req = 0; + } + if (total > sc.cap) { + std::free(sc.buf); + sc.cap = (size_t) ((total + (1u << 20) - 1) >> 20 << 20); + void* p = nullptr; + sc.buf = posix_memalign(&p, (size_t) kSector, sc.cap) == 0 ? (uint8_t*) p : nullptr; + if (sc.buf == nullptr) { sc.cap = 0; return false; } + } + sc.got.assign(sc.req.size(), -1); + sc.cb.resize(std::min(sc.req.size(), (size_t) kDepth)); + sc.cbp.resize(sc.cb.size()); + sc.ev.resize(sc.cb.size()); + bool ok = true; + for (size_t w0 = 0; ok && w0 < sc.req.size(); w0 += (size_t) kDepth) { + const size_t wn = std::min((size_t) kDepth, sc.req.size() - w0); + for (size_t j = 0; j < wn; ++j) { + const Req& r = sc.req[w0 + j]; + iocb& c = sc.cb[j]; + std::memset(&c, 0, sizeof c); + c.aio_lio_opcode = IOCB_CMD_PREAD; + c.aio_fildes = (uint32_t) r.fd; + c.aio_buf = (uint64_t) (uintptr_t) (sc.buf + r.pos); + c.aio_nbytes = r.size; + c.aio_offset = (int64_t) r.a0; + c.aio_data = (uint64_t) (w0 + j); + sc.cbp[j] = &c; + } + // every request of the wave queued; a refused submit ends the batch once what was queued has completed + // (the kernel still writes into the buffer until then) + size_t queued = 0; + while (queued < wn) { + const long r = syscall(SYS_io_submit, sc.ctx, (long) (wn - queued), sc.cbp.data() + queued); + if (r > 0) { queued += (size_t) r; continue; } + if (r < 0 && errno == EINTR) continue; + ok = false; + break; + } + for (size_t done = 0; done < queued;) { + const long r = syscall(SYS_io_getevents, sc.ctx, 1L, (long) (queued - done), sc.ev.data(), nullptr); + if (r < 0) { + if (errno == EINTR) continue; + // the context is in an unknown state: drop it (io_destroy waits for what is still in flight) + (void) syscall(SYS_io_destroy, sc.ctx); + sc.ctx = 0; + return false; + } + for (long j = 0; j < r; ++j) { + const io_event& e = sc.ev[(size_t) j]; + sc.got[(size_t) e.data] = e.res; + if (e.res < 0) ok = false; + } + done += (size_t) r; + } + } + if (!ok) return false; + for (const Window& x : sc.win) { + // a request may run past the end of the file: only the role's own bytes have to arrive + if (sc.got[x.req] < 0 || (uint64_t) sc.got[x.req] < x.in_req + x.skip + x.n) return false; + std::memcpy(x.dst + x.at, sc.buf + sc.req[x.req].pos + x.in_req + x.skip, (size_t) x.n); + } + return true; #else (void) fills; (void) n; return false; diff --git a/src/program/generate.cpp b/src/program/generate.cpp index 085c3d8ced6..f0fc1a1c507 100644 --- a/src/program/generate.cpp +++ b/src/program/generate.cpp @@ -4592,9 +4592,9 @@ int main(int argc, char** argv) { } // #577: the unbuffered choice above was made before the RAM copy existed, from the budget asked for; now the // copy is built, decide again from the RAM it really holds and the expert bytes outside it (on a 96 GB PC - // the file cache keeps those, and every refill after a prompt read the drive instead). Windows only: the - // unbuffered reads exist there alone -#if defined(_WIN32) + // the file cache keeps those, and every refill after a prompt read the drive instead). Windows and Linux: the + // unbuffered reads exist there +#if defined(_WIN32) || defined(__linux__) if (o.mmap_experts && o.resident_budget > 0) { std::string why; const bool was = src.unbuffered(); diff --git a/tests/core/file_expert_source_test.cpp b/tests/core/file_expert_source_test.cpp index 104507d1352..bc9eb30b152 100644 --- a/tests/core/file_expert_source_test.cpp +++ b/tests/core/file_expert_source_test.cpp @@ -9,6 +9,8 @@ #include #include +#include +#include #include #include #include @@ -32,9 +34,9 @@ void require(bool ok, const std::string& message) { struct TempDirectory { fs::path path; - TempDirectory() { + explicit TempDirectory(const fs::path& base = fs::temp_directory_path()) { const auto stamp = std::chrono::steady_clock::now().time_since_epoch().count(); - path = fs::temp_directory_path() / ("strata-file-source-test-" + std::to_string(stamp)); + path = base / ("strata-file-source-test-" + std::to_string(stamp)); fs::create_directories(path); } @@ -288,6 +290,76 @@ void test_cgroup_memory_budget() { "missing memory.stat counters did not fail closed"); } +void set_env(const char* name, const char* value) { +#if defined(_WIN32) + _putenv_s(name, value != nullptr ? value : ""); +#else + if (value != nullptr) setenv(name, value, 1); + else unsetenv(name); +#endif +} + +// #286: the unbuffered file tier (Windows: FILE_FLAG_NO_BUFFERING; Linux: O_DIRECT + io_submit) hands out the same +// bytes as the mapping - one blob at a time, a batch of adjacent experts (merged into one request), and copy_blob - and +// never falls back to the mapping. The pack lives in the working directory, not the temp directory: a tmpfs /tmp has +// no O_DIRECT, and there the test says so and skips. +void test_unbuffered_reads() { + using namespace strata::core; + using namespace strata::kernels::cpu; + constexpr int64_t layers = 2; + constexpr int64_t experts = 3; + const uint64_t layer_bytes = (uint64_t) experts * BLOB; + const uint64_t total = (uint64_t) layers * layer_bytes; + TempDirectory dir(fs::current_path()); + std::string err; + require(expert_layout_load(dir.path.string(), layers, experts, err), "could not load canonical layout: " + err); + std::vector bytes((size_t) total); + uint64_t x = 0x9E3779B97F4A7C15ull; + for (uint8_t& b : bytes) { + x = x * 6364136223846793005ull + 1442695040888963407ull; + b = (uint8_t) (x >> 56); + } + { + std::ofstream out(dir.path / "experts.bin", std::ios::binary | std::ios::trunc); + out.write((const char*) bytes.data(), (std::streamsize) bytes.size()); + require((bool) out, "could not write the synthetic experts.bin"); + } + FileExpertSource source; + require(source.open(dir.path.string(), layers, experts, err), "could not map the synthetic pack: " + err); + set_env("STRATA_UNBUFFERED_LOAD", "1"); + std::string why; + const bool unbuffered = source.set_unbuffered(0, why); + set_env("STRATA_UNBUFFERED_LOAD", nullptr); +#if defined(_WIN32) || defined(__linux__) + if (!unbuffered) { + std::cout << "file_expert_source_test: unbuffered reads skipped (" << why << ")\n"; + source.close(); + return; + } +#else + require(!unbuffered, "unbuffered reads on a platform without them"); + source.close(); + return; +#endif + require(source.unbuffered(), "set_unbuffered succeeded but the source is not unbuffered"); + const int64_t batch[experts] = {0, 1, 2}; + source.prefetch(1, batch, experts); // layer 1 in one batch: three adjacent blobs + for (int64_t l = 0; l < layers; ++l) + for (int64_t e = 0; e < experts; ++e) { + const uint8_t* b = source.blob(l, e); + require(b != nullptr, "an unbuffered blob lookup failed"); + require(std::memcmp(b, bytes.data() + (size_t) (l * (int64_t) layer_bytes + e * (int64_t) BLOB), + (size_t) BLOB) == 0, + "an unbuffered blob differs from the file"); + } + std::vector copy((size_t) BLOB); + require(source.copy_blob(0, 2, copy.data()) && + std::memcmp(copy.data(), bytes.data() + (size_t) (2 * BLOB), (size_t) BLOB) == 0, + "an unbuffered copy_blob differs from the file"); + require(source.direct_fallbacks() == 0, "an unbuffered read fell back to the mapping"); + source.close(); +} + // #633: the host RAM probe with fake /proc and cgroup trees: v2 (a limit, "max", a missing or malformed limit), v1 // (a limit, unlimited), and no cgroup line at all. Elsewhere than Linux it reads the machine's RAM. void test_host_memory() { @@ -348,6 +420,7 @@ int main() { test_cgroup_memory_budget(); test_host_memory(); test_canonical_layout(); + test_unbuffered_reads(); #if defined(STRATA_NATIVE_EXPERTS) test_native_variable_layout(); #endif From 2bdcf377e0d61b87f1d0bd442d5477ccf9952f1f Mon Sep 17 00:00:00 2001 From: Pc1 Date: Mon, 5 Oct 2026 00:17:47 +0200 Subject: [PATCH 2/2] file tier: STRATA_UNBUFFERED_LOAD=1 is not honoured without a RAM budget Review of #773 (2 x RTX 5060 Ti 16 GB, 32 GB RAM, --mmap-experts, no --resident-budget-gib): with no RAM copy the file cache is the expert tier, and forcing unbuffered reads there re-read 163-629 GB from the drive and halved the speed. The automatic decision never armed in that mode; now the forced one does not either, and the log says so. Co-Authored-By: Claude Opus 5.5 --- src/program/generate.cpp | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/program/generate.cpp b/src/program/generate.cpp index f0fc1a1c507..41de68240f0 100644 --- a/src/program/generate.cpp +++ b/src/program/generate.cpp @@ -3136,11 +3136,18 @@ int main(int argc, char** argv) { // #286: with a RAM budget the hottest experts live in it, and the rest are read from the drive unbuffered // when the file cache could not keep them beside the budget anyway (a 32 GB PC) - the mapped reads' page // faults are small requests on the critical path, and their pages take the RAM the budget was sized for - if (o.resident_budget > 0 || std::getenv("STRATA_UNBUFFERED_LOAD") != nullptr) { + // #773: without a RAM budget there is no RAM copy and the file cache IS the expert tier (measured on a + // 32 GB, 2 x 16 GB rig: forcing unbuffered reads there re-read 163-629 GB from the drive and halved the + // speed), so STRATA_UNBUFFERED_LOAD=1 is not honoured in that mode + if (o.resident_budget > 0) { std::string why; const bool ub = src.set_unbuffered(o.resident_budget, why); std::fprintf(stderr, "strata generate: the file tier reads %s (%s)\n", ub ? "unbuffered" : "through the file cache", why.c_str()); + } else if (const char* env = std::getenv("STRATA_UNBUFFERED_LOAD"); env != nullptr && env[0] != '\0' && + env[0] != '0') { + std::fprintf(stderr, "strata generate: the file tier reads through the file cache (STRATA_UNBUFFERED_LOAD=%s " + "ignored: without --resident-budget-gib the file cache holds the experts)\n", env); } srcp = &src; } else {