perf(api): stream mapping eviction to fix OOM from whole-index loading

go.mod

@@ -127,7 +127,6 @@ require (
 )
 
 require (
-	github.com/darkweak/go-esi v0.0.5
 	github.com/dustin/go-humanize v1.0.1 // indirect
 	github.com/klauspost/compress v1.18.4 // indirect
 	github.com/miekg/dns v1.1.72 // indirect
@@ -136,3 +135,5 @@ require (
 	golang.org/x/tools v0.42.0 // indirect
 	google.golang.org/grpc v1.79.1 // indirect
 )
+
+replace github.com/darkweak/storages/core => github.com/mohammed90/storages/core v0.0.0-20260707230756-6f41a049943a

go.sum

@@ -105,10 +105,6 @@ github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwc
 github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
 github.com/cpuguy83/go-md2man/v2 v2.0.7 h1:zbFlGlXEAKlwXpmvle3d8Oe3YnkKIK4xSRTd3sHPnBo=
 github.com/cpuguy83/go-md2man/v2 v2.0.7/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
-github.com/darkweak/go-esi v0.0.5 h1:b9LHI8Tz46R+i6p8avKPHAIBRQUCZDebNmKm5w/Zrns=
-github.com/darkweak/go-esi v0.0.5/go.mod h1:koCJqwum1u6mslyZuq/Phm6hfG1K3ZK5Y7jrUBTH654=
-github.com/darkweak/storages/core v0.0.20-0.20260314133624-4df176921261 h1:yD3MyDBr12X2YwZrv6eoHgNygDwUcoMPBs6z42MVC9M=
-github.com/darkweak/storages/core v0.0.20-0.20260314133624-4df176921261/go.mod h1:h36sNSwQ36fdHFMUIqD3j403kv29R4FVJJDaLbyvofM=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -229,6 +225,8 @@ github.com/mitchellh/go-ps v1.0.0/go.mod h1:J4lOc8z8yJs6vUwklHw2XEIiT4z4C40KtWVN
 github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
 github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ=
 github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw=
+github.com/mohammed90/storages/core v0.0.0-20260707230756-6f41a049943a h1:DuxwkyCqbr/t46q7m3NsCWX2D3ArHq3GNjAQfK4UDj8=
+github.com/mohammed90/storages/core v0.0.0-20260707230756-6f41a049943a/go.mod h1:h36sNSwQ36fdHFMUIqD3j403kv29R4FVJJDaLbyvofM=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2DcNVpwGmV9E1BkGknEliJkfwQj0=

pkg/api/souin.go

@@ -168,18 +168,30 @@
 	types.DefaultStorageName: types.OneYearDuration,
 }
 
+// maxMappingValueSize bounds the mapping values the eviction is willing to
+// decode. Anything larger is dropped undecoded: unmarshalling a pathological
+// value would materialize it in memory, and its response keys expire through
+// their own TTLs anyway.
+const maxMappingValueSize = 1 << 20
+
 func EvictMapping(current types.Storer) {
-	values := current.MapKeys(core.MappingKeyPrefix)
 	now := time.Now()
 	infiniteStoreDuration := storageToInfiniteTTLMap[current.Name()]
 
-	for k, v := range values {
+	processMapping := func(k string, v []byte) bool {
+		if len(v) > maxMappingValueSize {
+			fmt.Println("Deleting the oversized mapping", core.MappingKeyPrefix+k)
+			current.Delete(core.MappingKeyPrefix + k)
+
+			return true
+		}
+
 		mapping := &core.StorageMapper{}
 
-		e := proto.Unmarshal([]byte(v), mapping)
+		e := proto.Unmarshal(v, mapping)
 		if e != nil {
 			current.Delete(core.MappingKeyPrefix + k)
-			continue
+			return true
 		}
 
 		updated := false
@@ -205,6 +217,20 @@
 		if len(mapping.GetMapping()) == 0 {
 			current.Delete(core.MappingKeyPrefix + k)
 		}
+
+		return true
+	}
+
+	// Stream the mapping index in bounded batches when the storer supports
+	// it, so the whole index is never materialized in memory at once.
+	if walker, ok := current.(core.MappingWalker); ok {
+		_ = walker.WalkMappings(core.MappingKeyPrefix, processMapping)
+
+		return
+	}
+
+	for k, v := range current.MapKeys(core.MappingKeyPrefix) {
+		processMapping(k, []byte(v))
 	}
 }
 

pkg/api/souin_internal_test.go

@@ -0,0 +1,46 @@
+package api
+
+import (
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/darkweak/souin/pkg/storage"
+	"github.com/darkweak/souin/tests"
+	"github.com/darkweak/storages/core"
+	"google.golang.org/protobuf/proto"
+	"google.golang.org/protobuf/types/known/timestamppb"
+)
+
+func TestEvictMapping_OversizedValue(t *testing.T) {
+	memoryStorer, err := storage.Factory(tests.MockConfiguration(tests.BaseConfiguration))
+	if err != nil {
+		t.Fatalf("Impossible to create the storer, %v given", err)
+	}
+
+	oversized := strings.Repeat("a", maxMappingValueSize+1)
+	if err := memoryStorer.Set(core.MappingKeyPrefix+"oversized", []byte(oversized), time.Minute); err != nil {
+		t.Fatalf("Impossible to store the value, %v given", err)
+	}
+
+	expired, _ := proto.Marshal(&core.StorageMapper{Mapping: map[string]*core.KeyIndex{
+		"expired-key": {
+			FreshTime: timestamppb.New(time.Now().Add(-time.Hour)),
+			StaleTime: timestamppb.New(time.Now().Add(-time.Hour)),
+			RealKey:   "expired-key",
+		},
+	}})
+	if err := memoryStorer.Set(core.MappingKeyPrefix+"valid", expired, time.Minute); err != nil {
+		t.Fatalf("Impossible to store the value, %v given", err)
+	}
+
+	EvictMapping(memoryStorer)
+
+	if res := memoryStorer.Get(core.MappingKeyPrefix + "oversized"); len(res) != 0 {
+		t.Error("The oversized mapping should be deleted without being decoded")
+	}
+
+	if res := memoryStorer.Get(core.MappingKeyPrefix + "valid"); len(res) != 0 {
+		t.Error("The fully expired mapping should be deleted")
+	}
+}

pkg/middleware/middleware.go

@@ -75,6 +75,10 @@ func tryAcquireEvictionLock(storer types.Storer) bool {
 			if err == nil && now.Before(lockedUntil) {
 				// Lock is still valid - check if we own it
 				if holderID == evictionLockHolder {
+					// Extend the expiry so walks longer than the TTL keep ownership.
+					renewed := evictionLockHolder + "|" + now.Add(evictionLockTTL).Format(time.RFC3339)
+					_ = storer.Set(evictionLockKey, []byte(renewed), evictionLockTTL)
+
 					return true
 				}
 				return false
@@ -96,26 +100,58 @@ func tryAcquireEvictionLock(storer types.Storer) bool {
 	return string(verifyValue) == lockValue
 }
 
+// evictionRegistry tracks storers that already have an eviction goroutine so
+// re-instantiated handlers (e.g. on config reload or multiple cache blocks)
+// don't spawn concurrent walkers for the same storer.
+var evictionRegistry = sync.Map{}
+
 func registerMappingKeysEviction(ctx baseCtx.Context, logger core.Logger, storers []types.Storer, interval time.Duration) {
 	for _, storer := range storers {
+		if _, alreadyRegistered := evictionRegistry.LoadOrStore(storer.Name()+"-"+storer.Uuid(), true); alreadyRegistered {
+			logger.Debugf("mapping eviction already registered for storer %s", storer.Name())
+
+			continue
+		}
+
 		logger.Debugf("registering mapping eviction for storer %s (interval: %s)", storer.Name(), interval)
 		go func(current types.Storer, currentInterval time.Duration) {
-			ticker := time.NewTicker(interval)
-			defer ticker.Stop()
+			// A timer reset after each walk guarantees a full interval of
+			// rest between walks; a ticker would fire immediately after a
+			// walk longer than the interval.
+			timer := time.NewTimer(currentInterval)
+			defer timer.Stop()
 
 			for {
 				select {
 				case <-ctx.Done():
 					return
-				case <-ticker.C:
-					if !tryAcquireEvictionLock(current) {
-						logger.Debugf("skipping mapping eviction for storer %s, another instance holds the lock", current.Name())
+				case <-timer.C:
+					if tryAcquireEvictionLock(current) {
+						// Keep extending the lock while the walk runs so other
+						// replicas don't join once the TTL elapses mid-walk.
+						walkDone := make(chan struct{})
+						go func() {
+							keepalive := time.NewTicker(evictionLockTTL / 2)
+							defer keepalive.Stop()
+
+							for {
+								select {
+								case <-walkDone:
+									return
+								case <-keepalive.C:
+									_ = tryAcquireEvictionLock(current)
+								}
+							}
+						}()
 
-						continue
+						logger.Debugf("run mapping eviction for storer %s", current.Name())
+						api.EvictMapping(current)
+						close(walkDone)
+					} else {
+						logger.Debugf("skipping mapping eviction for storer %s, another instance holds the lock", current.Name())
 					}
 
-					logger.Debugf("run mapping eviction for storer %s", current.Name())
-					api.EvictMapping(current)
+					timer.Reset(currentInterval)
 				}
 			}
 		}(storer, interval)

plugins/caddy/go.mod

@@ -37,7 +37,6 @@ require (
 	github.com/cloudflare/circl v1.6.3 // indirect
 	github.com/coreos/go-oidc/v3 v3.17.0 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.7 // indirect
-	github.com/darkweak/go-esi v0.0.6 // indirect
 	github.com/dgraph-io/badger v1.6.2 // indirect
 	github.com/dgraph-io/badger/v2 v2.2007.4 // indirect
 	github.com/dgraph-io/ristretto v0.2.0 // indirect
@@ -177,3 +176,5 @@ require (
 )
 
 replace github.com/darkweak/souin v1.7.8 => ../..
+
+replace github.com/darkweak/storages/core => github.com/mohammed90/storages/core v0.0.0-20260707230756-6f41a049943a

plugins/caddy/go.sum

@@ -119,10 +119,6 @@ github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwc
 github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
 github.com/cpuguy83/go-md2man/v2 v2.0.7 h1:zbFlGlXEAKlwXpmvle3d8Oe3YnkKIK4xSRTd3sHPnBo=
 github.com/cpuguy83/go-md2man/v2 v2.0.7/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
-github.com/darkweak/go-esi v0.0.6 h1:eVHCJfqrZwOHPfRK7JTlSYG9F8lfpX/d4lz/41RQkd8=
-github.com/darkweak/go-esi v0.0.6/go.mod h1:IJSayeQZDUh5R5ayyDC3wUEBykti12aUa0eUxZZeodk=
-github.com/darkweak/storages/core v0.0.20-0.20260314133624-4df176921261 h1:yD3MyDBr12X2YwZrv6eoHgNygDwUcoMPBs6z42MVC9M=
-github.com/darkweak/storages/core v0.0.20-0.20260314133624-4df176921261/go.mod h1:h36sNSwQ36fdHFMUIqD3j403kv29R4FVJJDaLbyvofM=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -258,6 +254,8 @@ github.com/mitchellh/go-ps v1.0.0/go.mod h1:J4lOc8z8yJs6vUwklHw2XEIiT4z4C40KtWVN
 github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
 github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ=
 github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw=
+github.com/mohammed90/storages/core v0.0.0-20260707230756-6f41a049943a h1:DuxwkyCqbr/t46q7m3NsCWX2D3ArHq3GNjAQfK4UDj8=
+github.com/mohammed90/storages/core v0.0.0-20260707230756-6f41a049943a/go.mod h1:h36sNSwQ36fdHFMUIqD3j403kv29R4FVJJDaLbyvofM=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2DcNVpwGmV9E1BkGknEliJkfwQj0=