From 4636dbe3957c09a1d35483ca1d0d8c30dc9a3857 Mon Sep 17 00:00:00 2001 From: Donald Chen Date: Fri, 16 Oct 2026 20:26:21 -0700 Subject: [PATCH 1/2] poly, HIP: the expert cache gives VRAM back and takes it again on AMD (#13) On a Linux AMD card the expert cache only ever got smaller: every program that held VRAM a moment cost experts until the server was restarted. Three causes, measured on an RX 9070 XT (16 GB, Fedora, ROCm 7 wheels): - --kv-whole (which setup writes for AMD) turned VMM off for the whole engine, so the expert cache was one allocation, filled again smaller when VRAM ran short and never grown. VMM now stays on for the expert cache; --kv-whole keeps the attention caches as one allocation each (kv_alloc()). - ROCm gives a released handle's VRAM back only when the addresses it was mapped at are freed, and frees only a whole reservation: 320 MiB unmapped and released left 32 MiB more free, the addresses freed all of it, and a partial hipMemAddressFree fails. vmm_unmap() frees the reservation, reserves it again at the same address and maps the kept pieces back (their bytes stay), under STRATA_VMM_RELEASE_WITH_ADDRESSES (hip_compat/cuda.h). - hipMemGetInfo's free figure is KFD accounting, which read 924 MiB while amdgpu had 2,128 MiB free (a 1,500 MiB hipMalloc beside the engine succeeded), and at another time more than amdgpu had. On Linux the free figure is now amdgpu's mem_info_vram_total less mem_info_vram_used for the card's PCI address; STRATA_KERNEL_VRAM_FREE=0 keeps hipMemGetInfo's. With Tiel Coder UD-Q4_K_XL at 128K: another process taking 800 MiB shrank the cache 4523 -> 4005 experts and amdgpu's free VRAM rose by what was cut; when it ended, the next request grew the cache to 4453. A 256-token answer at 110-128 tokens/s and a 4.5K prompt at about 1,820 tokens/s (105.7 and 1,941 before, with 4,758 experts cached). Co-Authored-By: Claude Opus 5.5 --- include/strata/hip_compat/cuda.h | 4 +++ include/strata/hip_compat/cuda_runtime.h | 41 +++++++++++++++++++++++- src/poly/engine.cpp | 39 ++++++++++++++++++++-- 3 files changed, 80 insertions(+), 4 deletions(-) diff --git a/include/strata/hip_compat/cuda.h b/include/strata/hip_compat/cuda.h index 46629d61e..a3a9e2243 100644 --- a/include/strata/hip_compat/cuda.h +++ b/include/strata/hip_compat/cuda.h @@ -4,6 +4,10 @@ // (cuMemCreate, cuMemMap, ...) onto HIP's memory management API, which the HIP runtime provides. #include +// ROCm gives the VRAM of a released handle back only when the addresses it was mapped at are freed, and frees only +// a whole reservation (#13): Engine::vmm_unmap() reserves the addresses again for the pieces it keeps. +#define STRATA_VMM_RELEASE_WITH_ADDRESSES 1 + typedef hipError_t CUresult; typedef int CUdevice; // HIP's own CUdeviceptr/handle equivalents are pointer-typed; CUDA's are integers, and strata-poly's Mapped diff --git a/include/strata/hip_compat/cuda_runtime.h b/include/strata/hip_compat/cuda_runtime.h index c35c9ce1a..10b55da06 100644 --- a/include/strata/hip_compat/cuda_runtime.h +++ b/include/strata/hip_compat/cuda_runtime.h @@ -92,7 +92,46 @@ hipError_t mem_get_info(size_t* free_bytes, size_t* total_bytes); } #define cudaMemGetInfo ::strata::hip_compat::mem_get_info #else -#define cudaMemGetInfo hipMemGetInfo +// Linux: hipMemGetInfo's free figure is the kernel driver's KFD accounting, which on an RX 9070 XT (16 GB, Fedora, +// ROCm 7 wheels) read 924 MiB while amdgpu's own counters had 2,128 MiB free, and a 1,500 MiB hipMalloc beside the +// engine succeeded. The expert cache, sized and grown by that figure, left more than a GiB of the card unused and +// did not grow back after another program freed VRAM. The free figure is amdgpu's (mem_info_vram_total less +// mem_info_vram_used, for the card's PCI address) when the files are there (STRATA_KERNEL_VRAM_FREE=0: as it is). +#include +#include +#include +#include +namespace strata::hip_compat { +inline bool read_sysfs_u64(const char* path, unsigned long long* v) { + FILE* f = std::fopen(path, "r"); + if (!f) return false; + const bool ok = std::fscanf(f, "%llu", v) == 1; + std::fclose(f); + return ok; +} +inline hipError_t mem_get_info(size_t* free_bytes, size_t* total_bytes) { + const hipError_t e = hipMemGetInfo(free_bytes, total_bytes); + static const bool off = [] { + const char* v = std::getenv("STRATA_KERNEL_VRAM_FREE"); + return v && std::strcmp(v, "0") == 0; + }(); + int dev = 0; + char bus[64] = {}; + if (e != hipSuccess || off || hipGetDevice(&dev) != hipSuccess || + hipDeviceGetPCIBusId(bus, (int) sizeof(bus), dev) != hipSuccess) + return e; + for (char* c = bus; *c; ++c) *c = (char) std::tolower((unsigned char) *c); + char path[160]; + unsigned long long total = 0, used = 0; + std::snprintf(path, sizeof(path), "/sys/bus/pci/devices/%s/mem_info_vram_total", bus); + if (!read_sysfs_u64(path, &total)) return e; + std::snprintf(path, sizeof(path), "/sys/bus/pci/devices/%s/mem_info_vram_used", bus); + if (!read_sysfs_u64(path, &used) || used > total || total == 0) return e; + *free_bytes = (size_t) (total - used); + return e; +} +} // namespace strata::hip_compat +#define cudaMemGetInfo ::strata::hip_compat::mem_get_info #endif #define cudaMemcpy hipMemcpy #define cudaMemcpy2D hipMemcpy2D diff --git a/src/poly/engine.cpp b/src/poly/engine.cpp index 3a2a13791..3f953b37d 100644 --- a/src/poly/engine.cpp +++ b/src/poly/engine.cpp @@ -496,6 +496,7 @@ void Engine::vmm_map(Mapped& m, size_t bytes, size_t piece, const char* what) { // Gives back the pieces of m that lie past its first `bytes`. The card must be done with them. void Engine::vmm_unmap(Mapped& m, size_t bytes) { + const size_t had = m.mapped; while (!m.pieces.empty() && m.mapped - m.pieces.back().second >= bytes) { const auto [h, n] = m.pieces.back(); m.mapped -= n; @@ -503,6 +504,36 @@ void Engine::vmm_unmap(Mapped& m, size_t bytes) { cu_ck(cuMemUnmap(m.base + m.mapped, n), "unmapping VRAM"); cu_ck(cuMemRelease(h), "unmapping VRAM"); } +#ifdef STRATA_VMM_RELEASE_WITH_ADDRESSES + // On ROCm the released pieces stay the process's while their addresses are reserved (an RX 9070 XT: 320 MiB + // unmapped and released, 32 MiB more free; the addresses freed, all of it). The kept pieces leave their + // addresses, which are freed and reserved again where they were, and come back with their bytes. + if (m.mapped == had || !m.base) return; + for (size_t at = m.mapped, i = m.pieces.size(); i > 0; --i) { + at -= m.pieces[i - 1].second; + cu_ck(cuMemUnmap(m.base + at, m.pieces[i - 1].second), "unmapping VRAM"); + } + const CUdeviceptr was = m.base; + cu_ck(cuMemAddressFree(m.base, m.reserved), "unmapping VRAM"); + CUdeviceptr p = 0; + cu_ck(cuMemAddressReserve(&p, m.reserved, 0, was, 0), "reserving addresses for VRAM"); + if (p != was) { // the slots and the recorded launches point into the old addresses + (void) cuMemAddressFree(p, m.reserved); + throw std::runtime_error("unmapping VRAM: the addresses could not be reserved again where they were"); + } + CUmemAccessDesc access = {}; + access.location.type = CU_MEM_LOCATION_TYPE_DEVICE; + access.location.id = device_; + access.flags = CU_MEM_ACCESS_FLAGS_PROT_READWRITE; + size_t at = 0; + for (const auto& [h, n] : m.pieces) { + cu_ck(cuMemMap(m.base + at, n, 0, h, 0), "mapping VRAM"); + cu_ck(cuMemSetAccess(m.base + at, n, &access, 1), "mapping VRAM"); + at += n; + } +#else + (void) had; +#endif } // For the destructor: what is left of m, and its addresses, without throwing. @@ -520,7 +551,7 @@ void* Engine::kv_alloc(size_t bytes, int per) { if (again_ != SIZE_MAX) return bufs_.at(again_++).p; const size_t all = (size_t) ((opt_.max_context + per - 1) / per) * bytes; void* p = nullptr; - if (!vmm_) { + if (!vmm_ || !opt_.kv_grow) { p = dmalloc(all, "the attention cache"); } else { Kv k; @@ -1525,11 +1556,13 @@ void Engine::load(const Options& o) { ev_ = ev_row_[0]; blas_ = blas_create(st_); // The card maps VRAM onto reserved addresses a piece at a time: the attention caches grow with the sequence, - // and the expert cache has what a short one leaves (cover()). + // and the expert cache has what a short one leaves (cover()). With --kv-whole the attention caches are one + // allocation each (kv_alloc()) and the expert cache still gives VRAM to other programs and takes it back + // (mind_vram()): refilled in one allocation instead, it only ever got smaller. ck(cudaGetDevice(&device_), "cudaGetDevice"); CUdevice dev = 0; int vmm = 0; - if (opt_.kv_grow && cuDeviceGet(&dev, device_) == CUDA_SUCCESS && + if (cuDeviceGet(&dev, device_) == CUDA_SUCCESS && cuDeviceGetAttribute(&vmm, CU_DEVICE_ATTRIBUTE_VIRTUAL_MEMORY_MANAGEMENT_SUPPORTED, dev) == CUDA_SUCCESS && vmm) { CUmemAllocationProp prop = {}; prop.type = CU_MEM_ALLOCATION_TYPE_PINNED; From 8dd220504e1d543620f35f81752a4ad29d899de7 Mon Sep 17 00:00:00 2001 From: Donald Chen Date: Fri, 16 Oct 2026 21:09:46 -0700 Subject: [PATCH 2/2] poly, HIP: review fixes for the elastic expert cache - An integrated GPU keeps hipMemGetInfo's free figure: amdgpu's mem_info_vram files count only its BIOS carve-out (a Ryzen 7 9800X3D's Radeon, gfx1036: 512 MiB) while HIP gives it system RAM (15,536 MiB free), so an APU would have lost almost its whole expert cache. - A step of vmm_unmap()'s rebind that fails once the kept pieces have left their addresses ends the engine with a message instead of throwing: the throw reached the request's catch in main.cpp and the next request read VRAM that was no longer mapped. The server starts the engine again for the next request. - vmm_unmap() waits until amdgpu counts the VRAM it gave back as free (2 to 5 ms, at most 100 ms): the log line after a shrink read 248 MiB left where there were 1,012. - The rebind is Linux only, where it was measured; Windows' HIP runtime keeps the plain release. RX 9070 XT, Tiel Coder UD-Q4_K_XL: 800 MiB taken by another process shrank the cache 4694 -> 4180 experts with 1012 MiB left (amdgpu: 1013), and it grew to 4623 when that process ended. Co-Authored-By: Claude Opus 5.5 --- include/strata/hip_compat/cuda.h | 5 +++- include/strata/hip_compat/cuda_runtime.h | 5 +++- src/poly/engine.cpp | 37 +++++++++++++++++++----- 3 files changed, 37 insertions(+), 10 deletions(-) diff --git a/include/strata/hip_compat/cuda.h b/include/strata/hip_compat/cuda.h index a3a9e2243..61b332cf6 100644 --- a/include/strata/hip_compat/cuda.h +++ b/include/strata/hip_compat/cuda.h @@ -5,8 +5,11 @@ #include // ROCm gives the VRAM of a released handle back only when the addresses it was mapped at are freed, and frees only -// a whole reservation (#13): Engine::vmm_unmap() reserves the addresses again for the pieces it keeps. +// a whole reservation (#13): Engine::vmm_unmap() reserves the addresses again for the pieces it keeps. Measured on +// Linux; Windows' HIP runtime is another implementation and keeps the plain release until it is measured there. +#if !defined(_WIN32) #define STRATA_VMM_RELEASE_WITH_ADDRESSES 1 +#endif typedef hipError_t CUresult; typedef int CUdevice; diff --git a/include/strata/hip_compat/cuda_runtime.h b/include/strata/hip_compat/cuda_runtime.h index 10b55da06..b614a58d2 100644 --- a/include/strata/hip_compat/cuda_runtime.h +++ b/include/strata/hip_compat/cuda_runtime.h @@ -97,6 +97,8 @@ hipError_t mem_get_info(size_t* free_bytes, size_t* total_bytes); // engine succeeded. The expert cache, sized and grown by that figure, left more than a GiB of the card unused and // did not grow back after another program freed VRAM. The free figure is amdgpu's (mem_info_vram_total less // mem_info_vram_used, for the card's PCI address) when the files are there (STRATA_KERNEL_VRAM_FREE=0: as it is). +// Not on an integrated GPU: its files count only the BIOS carve-out (a Ryzen 7 9800X3D's Radeon: 512 MiB), while HIP +// gives it system RAM (15,536 MiB free by hipMemGetInfo). #include #include #include @@ -115,9 +117,10 @@ inline hipError_t mem_get_info(size_t* free_bytes, size_t* total_bytes) { const char* v = std::getenv("STRATA_KERNEL_VRAM_FREE"); return v && std::strcmp(v, "0") == 0; }(); - int dev = 0; + int dev = 0, integrated = 1; char bus[64] = {}; if (e != hipSuccess || off || hipGetDevice(&dev) != hipSuccess || + hipDeviceGetAttribute(&integrated, hipDeviceAttributeIntegrated, dev) != hipSuccess || integrated || hipDeviceGetPCIBusId(bus, (int) sizeof(bus), dev) != hipSuccess) return e; for (char* c = bus; *c; ++c) *c = (char) std::tolower((unsigned char) *c); diff --git a/src/poly/engine.cpp b/src/poly/engine.cpp index 3f953b37d..83e67d46a 100644 --- a/src/poly/engine.cpp +++ b/src/poly/engine.cpp @@ -23,6 +23,7 @@ #include #include #include +#include #include #include #include @@ -507,19 +508,32 @@ void Engine::vmm_unmap(Mapped& m, size_t bytes) { #ifdef STRATA_VMM_RELEASE_WITH_ADDRESSES // On ROCm the released pieces stay the process's while their addresses are reserved (an RX 9070 XT: 320 MiB // unmapped and released, 32 MiB more free; the addresses freed, all of it). The kept pieces leave their - // addresses, which are freed and reserved again where they were, and come back with their bytes. + // addresses, which are freed and reserved again where they were, and come back with their bytes. Once the first + // has left, the slots, the recorded launches and m point at VRAM that is not there: a step that fails from + // there on ends the engine (the server starts it again for the next request) rather than reading that VRAM. if (m.mapped == had || !m.base) return; + const auto must = [](CUresult r, const char* what) { + if (r == CUDA_SUCCESS) return; + const char* s = nullptr; + (void) cuGetErrorString(r, &s); + std::fprintf(stderr, "strata-poly: %s while giving VRAM back: %s; the caches are no longer mapped, so the " + "engine stops\n", what, s ? s : "unknown error"); + std::abort(); + }; for (size_t at = m.mapped, i = m.pieces.size(); i > 0; --i) { at -= m.pieces[i - 1].second; - cu_ck(cuMemUnmap(m.base + at, m.pieces[i - 1].second), "unmapping VRAM"); + must(cuMemUnmap(m.base + at, m.pieces[i - 1].second), "unmapping the kept VRAM"); } + size_t free_b = 0, total_b = 0; + (void) cudaMemGetInfo(&free_b, &total_b); const CUdeviceptr was = m.base; - cu_ck(cuMemAddressFree(m.base, m.reserved), "unmapping VRAM"); + must(cuMemAddressFree(m.base, m.reserved), "freeing the addresses"); CUdeviceptr p = 0; - cu_ck(cuMemAddressReserve(&p, m.reserved, 0, was, 0), "reserving addresses for VRAM"); + must(cuMemAddressReserve(&p, m.reserved, 0, was, 0), "reserving the addresses again"); if (p != was) { // the slots and the recorded launches point into the old addresses - (void) cuMemAddressFree(p, m.reserved); - throw std::runtime_error("unmapping VRAM: the addresses could not be reserved again where they were"); + std::fprintf(stderr, "strata-poly: the VRAM addresses could not be reserved again where they were, so the " + "engine stops\n"); + std::abort(); } CUmemAccessDesc access = {}; access.location.type = CU_MEM_LOCATION_TYPE_DEVICE; @@ -527,10 +541,17 @@ void Engine::vmm_unmap(Mapped& m, size_t bytes) { access.flags = CU_MEM_ACCESS_FLAGS_PROT_READWRITE; size_t at = 0; for (const auto& [h, n] : m.pieces) { - cu_ck(cuMemMap(m.base + at, n, 0, h, 0), "mapping VRAM"); - cu_ck(cuMemSetAccess(m.base + at, n, &access, 1), "mapping VRAM"); + must(cuMemMap(m.base + at, n, 0, h, 0), "mapping VRAM again"); + must(cuMemSetAccess(m.base + at, n, &access, 1), "mapping VRAM again"); at += n; } + // amdgpu counts the VRAM as free 2 to 5 ms later: a reading before that (mind_vram()'s log line, cover()) is + // short by what was given back. At most 100 ms, as another program may take VRAM meanwhile. + const size_t back = (had - m.mapped) / 8 * 7, want = free_b + back; + for (int i = 0; i < 100 && free_b < want; ++i) { + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + (void) cudaMemGetInfo(&free_b, &total_b); + } #else (void) had; #endif