From 72c5646cf8cb734a2ea2bdfd11e11b244d727cac Mon Sep 17 00:00:00 2001 From: bussyjd Date: Wed, 3 Jun 2026 10:24:30 +0400 Subject: [PATCH 1/7] fix(controller): repin serviceoffer-controller to rc9 image with Secret create-only The pinned serviceoffer-controller image (f5d94fc) was a side-branch build that predated the change making Secret create-only in the reconciler. The tightened ClusterRole grants no secrets update/patch verb, so the deployed binary 403s when it Updates the per-agent hermes-api-server / remote-signer-keystore Secrets on re-reconcile, and per-agent provisioning never converges. Repin to 503016b@sha256:bec62ea0 (rc9 commit 503016bf, image 0.10.0-rc9), whose reconciler treats Secret as create-only and matches the shipped RBAC. Add a tripwire test mirroring the x402-verifier one so a future downgrade can't silently re-ship the bug. The short-SHA tag keeps the dev-mode :latest rewrite and production pin invariants intact. --- internal/embed/embed_crd_test.go | 22 +++++++++++++++++++ internal/embed/embed_image_pin_test.go | 6 ++++- .../infrastructure/base/templates/x402.yaml | 10 ++++++++- 3 files changed, 36 insertions(+), 2 deletions(-) diff --git a/internal/embed/embed_crd_test.go b/internal/embed/embed_crd_test.go index 87c654a0..13e3a41c 100644 --- a/internal/embed/embed_crd_test.go +++ b/internal/embed/embed_crd_test.go @@ -791,6 +791,28 @@ func TestX402VerifierImage_CarriesAgentAuthFix(t *testing.T) { } } +// TestServiceOfferControllerImage_CarriesSecretCreateOnlyFix is the tripwire +// for the per-agent provisioning 403 (GA blocker). The controller ClusterRole +// in x402.yaml grants no secrets update/patch verb because the reconciler +// treats Secret as create-only (isCreateOnlyKind). The image MUST therefore be +// built from source that has that behaviour — the prior f5d94fc side-branch pin +// did not, so the deployed binary Updated the per-agent Secrets on re-reconcile +// and 403'd. 503016b (rc9 commit 503016bf, image 0.10.0-rc9) carries the fix. +// Bumping this pin requires a conscious, documented change here so a future +// downgrade can't silently re-ship the bug. +func TestServiceOfferControllerImage_CarriesSecretCreateOnlyFix(t *testing.T) { + data, err := ReadInfrastructureFile("base/templates/x402.yaml") + if err != nil { + t.Fatalf("ReadInfrastructureFile: %v", err) + } + + const ref = "ghcr.io/obolnetwork/serviceoffer-controller:503016b@sha256:bec62ea04842caf62980b529a89f5d553987a106c3167eb45209a8b278121957" + if !strings.Contains(string(data), "image: "+ref) { + t.Fatalf("serviceoffer-controller image must carry the Secret-create-only reconciler fix "+ + "(else per-agent provisioning 403s under the no-update/patch Secret RBAC): %s", ref) + } +} + // TestServiceOfferControllerSecretRBAC_Scoped guards the controller's Secret // access against the actual code paths in internal/serviceoffercontroller. // The reconciler touches exactly three Secrets by name (litellm-secrets, diff --git a/internal/embed/embed_image_pin_test.go b/internal/embed/embed_image_pin_test.go index 92ff5ec1..10aae737 100644 --- a/internal/embed/embed_image_pin_test.go +++ b/internal/embed/embed_image_pin_test.go @@ -230,8 +230,12 @@ func TestEmbeddedImages_X402ControllerAndBuyerUseFixPins(t *testing.T) { ref string }{ { + // Repinned off the f5d94fc side-branch build (which predated the + // Secret-create-only reconciler change) to the rc9 release image + // (commit 503016b, image 0.10.0-rc9). + // See TestServiceOfferControllerImage_CarriesSecretCreateOnlyFix. file: "base/templates/x402.yaml", - ref: "ghcr.io/obolnetwork/serviceoffer-controller:f5d94fc@sha256:c6aa6259e3a6bc61a5f4f7203d8c68cfdd861a8d365f9629d234d13b949bf48e", + ref: "ghcr.io/obolnetwork/serviceoffer-controller:503016b@sha256:bec62ea04842caf62980b529a89f5d553987a106c3167eb45209a8b278121957", }, { file: "base/templates/llm.yaml", diff --git a/internal/embed/infrastructure/base/templates/x402.yaml b/internal/embed/infrastructure/base/templates/x402.yaml index 56a603ca..7bef232e 100644 --- a/internal/embed/infrastructure/base/templates/x402.yaml +++ b/internal/embed/infrastructure/base/templates/x402.yaml @@ -343,7 +343,15 @@ spec: type: RuntimeDefault containers: - name: controller - image: ghcr.io/obolnetwork/serviceoffer-controller:f5d94fc@sha256:c6aa6259e3a6bc61a5f4f7203d8c68cfdd861a8d365f9629d234d13b949bf48e + # MUST be an image whose reconciler treats Secret as create-only + # (isCreateOnlyKind includes "Secret"). The ClusterRole above grants + # NO secrets update/patch verb, so a controller that Updates the + # per-agent hermes-api-server / remote-signer-keystore Secrets on + # re-reconcile 403s and per-agent provisioning never converges. The + # prior pin (f5d94fc) predated that source change and shipped the + # bug; 503016b (rc9 commit 503016bf, image 0.10.0-rc9) carries it. + # See TestServiceOfferControllerImage_CarriesSecretCreateOnlyFix. + image: ghcr.io/obolnetwork/serviceoffer-controller:503016b@sha256:bec62ea04842caf62980b529a89f5d553987a106c3167eb45209a8b278121957 imagePullPolicy: IfNotPresent securityContext: allowPrivilegeEscalation: false From 149e11f21cb4b0e18e5a6d253517c59b1dae1bf7 Mon Sep 17 00:00:00 2001 From: bussyjd Date: Wed, 3 Jun 2026 10:24:30 +0400 Subject: [PATCH 2/7] fix(x402-buyer): drop expired pre-signed auths before signing HoldSign popped s.auths[0] with no deadline check. A pre-signed Permit2 (OBOL) batch shares one ~5-min deadline, so once expired the buyer served the whole batch auth-by-auth, each returning 503 invalid_payment_expired from the verifier before reaching a fresh auth. Add authDeadlineUnix (covering the Permit2 deadline, nested ERC-3009 validBefore, and legacy flat field) and skip expired auths at pick time. USDC vouchers use a year-2106 validBefore and are never dropped. --- internal/x402/buyer/signer.go | 106 +++++++++++++++++++++++++++++ internal/x402/buyer/signer_test.go | 106 ++++++++++++++++++++++++++++- 2 files changed, 211 insertions(+), 1 deletion(-) diff --git a/internal/x402/buyer/signer.go b/internal/x402/buyer/signer.go index b41c8f9d..835277a4 100644 --- a/internal/x402/buyer/signer.go +++ b/internal/x402/buyer/signer.go @@ -3,13 +3,109 @@ package buyer import ( "encoding/json" "fmt" + "log" "math/big" + "strconv" "strings" "sync" + "time" x402types "github.com/coinbase/x402/go/types" ) +// authExpirySafetyMarginSec is how far before its on-chain deadline an auth is +// considered already expired for selection. The window between picking an auth +// and the facilitator settling it is a few seconds; dropping auths inside this +// margin avoids racing a voucher that expires mid-flight (which the facilitator +// rejects as invalid_payment_expired, surfacing as a 503 to the caller). +const authExpirySafetyMarginSec int64 = 10 + +// authDeadlineUnix returns the unix expiry of a pre-signed auth and whether one +// was found. Permit2 vouchers (OBOL) carry a real "deadline" (~5 min out); +// ERC-3009 vouchers (USDC) carry "validBefore". The value lives in the v2 +// payment payload; the legacy flat ValidBefore field is a fallback. Auths with +// no discoverable deadline are treated as non-expiring (ok=false) and never +// dropped — only an explicitly-past deadline removes an auth from the pool. +func authDeadlineUnix(a *PreSignedAuth) (int64, bool) { + if a == nil { + return 0, false + } + if a.Payment != nil { + if d, ok := payloadDeadlineUnix(a.Payment.Payload); ok { + return d, true + } + } + return parseUnixValue(a.ValidBefore) +} + +// payloadDeadlineUnix extracts the expiry from a v2 payment payload, covering +// both the Permit2 (permit2Authorization.deadline) and ERC-3009 +// (authorization.validBefore) shapes. +func payloadDeadlineUnix(payload map[string]any) (int64, bool) { + if payload == nil { + return 0, false + } + if p2, ok := payload["permit2Authorization"].(map[string]any); ok { + if d, ok := parseUnixValue(p2["deadline"]); ok { + return d, true + } + } + if authz, ok := payload["authorization"].(map[string]any); ok { + if d, ok := parseUnixValue(authz["validBefore"]); ok { + return d, true + } + } + return 0, false +} + +// parseUnixValue coerces a unix-timestamp field that may arrive as a JSON +// string, float64, json.Number, or integer into an int64. +func parseUnixValue(v any) (int64, bool) { + switch t := v.(type) { + case string: + if t == "" { + return 0, false + } + n, err := strconv.ParseInt(t, 10, 64) + if err != nil { + return 0, false + } + return n, true + case json.Number: + n, err := t.Int64() + if err != nil { + return 0, false + } + return n, true + case float64: + return int64(t), true + case int64: + return t, true + case int: + return int64(t), true + default: + return 0, false + } +} + +// dropExpiredAuthsLocked removes auths whose deadline is at or inside the safety +// margin from the head of the pool. Caller must hold s.mu. It returns the number +// dropped. Auths with no discoverable deadline (e.g. USDC validBefore in 2106) +// are kept. +func (s *PreSignedSigner) dropExpiredAuthsLocked(now int64) int { + dropped := 0 + for len(s.auths) > 0 { + dl, ok := authDeadlineUnix(s.auths[0]) + if ok && dl <= now+authExpirySafetyMarginSec { + s.auths = s.auths[1:] + dropped++ + continue + } + break + } + return dropped +} + // PreSignedSigner implements Signer using pre-signed ERC-3009 // TransferWithAuthorization vouchers. It pops one auth from the pool per // Sign() call. The pool is finite — once exhausted, CanSign returns false. @@ -118,6 +214,16 @@ func (s *PreSignedSigner) HoldSign(req *x402types.PaymentRequirements) (*x402typ s.mu.Lock() defer s.mu.Unlock() + // Discard auths that have expired (or expire within the safety margin) + // before picking. The pool is FIFO and a pre-signed batch shares roughly + // one deadline, so without this an expired batch is served auth-by-auth, + // each returning a 503 invalid_payment_expired from the verifier, until the + // whole expired batch is burned through. USDC vouchers carry a far-future + // validBefore and are never dropped here. + if dropped := s.dropExpiredAuthsLocked(time.Now().Unix()); dropped > 0 { + log.Printf("x402-buyer: dropped %d expired pre-signed auth(s) before signing (%d remaining)", dropped, len(s.auths)) + } + if len(s.auths) == 0 { return nil, nil, fmt.Errorf("pre-signed auth pool exhausted (spent %d): %w", s.spent, ErrNoValidSigner) diff --git a/internal/x402/buyer/signer_test.go b/internal/x402/buyer/signer_test.go index 04950485..373e6a19 100644 --- a/internal/x402/buyer/signer_test.go +++ b/internal/x402/buyer/signer_test.go @@ -2,8 +2,10 @@ package buyer import ( "encoding/json" + "strconv" "sync" "testing" + "time" x402types "github.com/coinbase/x402/go/types" ) @@ -340,7 +342,7 @@ func TestPreSignedSigner_SignGenericPayment(t *testing.T) { "from": "0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266", "spender": "0x402085c248EeA27D92E8b30b2C58ed07f9E20001", "nonce": "42", - "deadline": "99", + "deadline": "99999999999", "permitted": { "token": "0x036CbD53842c5426634e7929541eC2318f3dCF7e", "amount": "1000" @@ -399,3 +401,105 @@ func makeAuth(sig string) *PreSignedAuth { Nonce: "0xdeadbeef" + sig, } } + +// makePermit2Auth builds a Permit2 (OBOL) pre-signed auth with the given +// on-chain deadline (unix seconds) carried in the v2 payment payload — the +// shape buy.py emits for the OBOL path. +func makePermit2Auth(id string, deadline int64) *PreSignedAuth { + payment := &x402types.PaymentPayload{ + X402Version: 2, + Accepted: x402types.PaymentRequirements{ + Scheme: "exact", + Network: "eip155:84532", + Amount: "1000", + PayTo: "0x70997970C51812dc3A010C7d01b50e0d17dc79C8", + Asset: "0x036CbD53842c5426634e7929541eC2318f3dCF7e", + }, + Payload: map[string]any{ + "signature": "0x" + id, + "permit2Authorization": map[string]any{ + "from": "0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266", + "nonce": id, + "deadline": strconv.FormatInt(deadline, 10), + }, + }, + } + return &PreSignedAuth{ID: id, Payment: payment} +} + +// TestAuthDeadlineUnix covers expiry extraction across the Permit2, nested +// ERC-3009, and legacy-flat shapes — the load-bearing helper behind the +// expired-auth filter. +func TestAuthDeadlineUnix(t *testing.T) { + if d, ok := authDeadlineUnix(makePermit2Auth("1", 1234567890)); !ok || d != 1234567890 { + t.Errorf("permit2 deadline: got (%d,%v), want (1234567890,true)", d, ok) + } + // Legacy flat ERC-3009 validBefore (USDC path) — far future, parsed. + if d, ok := authDeadlineUnix(makeAuth("a")); !ok || d != 4294967295 { + t.Errorf("flat validBefore: got (%d,%v), want (4294967295,true)", d, ok) + } + // Nested ERC-3009 authorization.validBefore. + nested := &PreSignedAuth{Payment: &x402types.PaymentPayload{Payload: map[string]any{ + "authorization": map[string]any{"validBefore": "1700000000"}, + }}} + if d, ok := authDeadlineUnix(nested); !ok || d != 1700000000 { + t.Errorf("nested validBefore: got (%d,%v), want (1700000000,true)", d, ok) + } + // No deadline anywhere -> not found (never dropped). + if _, ok := authDeadlineUnix(&PreSignedAuth{Payment: &x402types.PaymentPayload{Payload: map[string]any{}}}); ok { + t.Error("auth with no deadline must report ok=false") + } + if _, ok := authDeadlineUnix(nil); ok { + t.Error("nil auth must report ok=false") + } +} + +// TestPreSignedSigner_DropsExpiredAuths is the regression for the 503 +// invalid_payment_expired cascade: HoldSign must skip expired Permit2 vouchers +// at the head of the FIFO pool instead of serving them. +func TestPreSignedSigner_DropsExpiredAuths(t *testing.T) { + now := time.Now().Unix() + expiredA := makePermit2Auth("expA", now-60) + expiredB := makePermit2Auth("expB", now-5) // inside the safety margin + fresh := makePermit2Auth("fresh", now+3600) + + signer := NewPreSignedSigner( + "base-sepolia", + "0x70997970C51812dc3A010C7d01b50e0d17dc79C8", + "0x036CbD53842c5426634e7929541eC2318f3dCF7e", + "1000", "OBOL", 18, + []*PreSignedAuth{expiredA, expiredB, fresh}, + 0, nil, + ) + + req := &x402types.PaymentRequirements{ + Network: "eip155:84532", + PayTo: "0x70997970C51812dc3A010C7d01b50e0d17dc79C8", + Asset: "0x036CbD53842c5426634e7929541eC2318f3dCF7e", + Amount: "1000", + } + + _, held, err := signer.HoldSign(req) + if err != nil { + t.Fatalf("HoldSign: %v", err) + } + if held.ID != "fresh" { + t.Fatalf("expected the fresh auth to be served, got %q (expired auths were not skipped)", held.ID) + } + if r := signer.Remaining(); r != 0 { + t.Fatalf("expected pool drained to 0 after serving the only fresh auth, got %d", r) + } + + // A pool of only-expired auths must exhaust, not serve an expired voucher. + expiredOnly := NewPreSignedSigner( + "base-sepolia", + "0x70997970C51812dc3A010C7d01b50e0d17dc79C8", + "0x036CbD53842c5426634e7929541eC2318f3dCF7e", + "1000", "OBOL", 18, + []*PreSignedAuth{makePermit2Auth("x", now-100), makePermit2Auth("y", now-100)}, + 0, nil, + ) + if _, _, err := expiredOnly.HoldSign(req); err == nil { + t.Fatal("expected exhausted-pool error when all auths are expired, got nil") + } +} From c412f24b3d675fd98fe3db3d1150a8edfc7a70c0 Mon Sep 17 00:00:00 2001 From: bussyjd Date: Wed, 3 Jun 2026 10:24:30 +0400 Subject: [PATCH 3/7] fix(controller): finalize purchase delete-drain when sidecar upstream is gone reconcileDeletingPurchase routed the 'not found in sidecar status' error into the Configured&&Remaining>0 branch, which kept Remaining>0 and requeued every 5s forever, stranding the PurchaseRequest in Terminating until its finalizer was force-removed. That signal means the sidecar has nothing left to drain. Add a case (via isSidecarUpstreamGone) that collapses Remaining to 0 so cleanup and finalizer removal proceed, consistent with the terminal not-found check already present later in the function. Transient errors still requeue. --- internal/serviceoffercontroller/purchase.go | 23 +++++++++++++- .../purchase_pure_test.go | 30 +++++++++++++++++++ 2 files changed, 52 insertions(+), 1 deletion(-) diff --git a/internal/serviceoffercontroller/purchase.go b/internal/serviceoffercontroller/purchase.go index f7c3d60d..787ab942 100644 --- a/internal/serviceoffercontroller/purchase.go +++ b/internal/serviceoffercontroller/purchase.go @@ -95,6 +95,16 @@ func (c *Controller) reconcilePurchase(ctx context.Context, key string) error { return nil } +// isSidecarUpstreamGone reports whether a checkBuyerStatus error means the +// x402-buyer sidecar no longer lists the upstream at all. That is the signal +// that a delete-drain is complete: there is nothing left for the sidecar to +// consume. It is deliberately a string match on the error checkBuyerStatus +// returns (purchase_helpers.go) so both the entry switch and the terminal +// cleanup check in reconcileDeletingPurchase agree on what "drained" means. +func isSidecarUpstreamGone(err error) bool { + return err != nil && strings.Contains(err.Error(), "not found in sidecar status") +} + func (c *Controller) reconcileDeletingPurchase(ctx context.Context, pr *monetizeapi.PurchaseRequest, raw *unstructured.Unstructured) error { buyerNS := pr.EffectiveBuyerNamespace() status := pr.Status @@ -105,6 +115,17 @@ func (c *Controller) reconcileDeletingPurchase(ctx context.Context, pr *monetize case err == nil: status.Remaining = remaining status.Spent = spent + case isSidecarUpstreamGone(err): + // The sidecar no longer reports this upstream. That deleted-upstream + // signal IS the drain-done signal: collapse Remaining to 0 so cleanup + // and finalizer removal proceed below. Without this case, a Configured + // purchase with Remaining>0 fell through to the next case, kept + // Remaining>0, and requeued every 5s forever — stranding the CR in + // Terminating until its finalizer was force-removed by hand. The + // terminal not-found check at the end of this function already treats + // the same signal as drain-complete; this keeps the entry switch + // consistent with it. + status.Remaining = 0 case purchaseConditionIsTrue(status.Conditions, "Configured") && status.Remaining > 0: log.Printf("purchase %s/%s: delete drain waiting for sidecar status: %v", pr.Namespace, pr.Name, err) default: @@ -146,7 +167,7 @@ func (c *Controller) reconcileDeletingPurchase(ctx context.Context, pr *monetize } c.purchaseQueue.AddAfter(pr.Namespace+"/"+pr.Name, 5*time.Second) return nil - } else if !strings.Contains(err.Error(), "not found in sidecar status") { + } else if !isSidecarUpstreamGone(err) { setPurchaseCondition( &status.Conditions, "Deleting", diff --git a/internal/serviceoffercontroller/purchase_pure_test.go b/internal/serviceoffercontroller/purchase_pure_test.go index a05fd5e9..22498180 100644 --- a/internal/serviceoffercontroller/purchase_pure_test.go +++ b/internal/serviceoffercontroller/purchase_pure_test.go @@ -1,12 +1,42 @@ package serviceoffercontroller import ( + "errors" + "fmt" "strings" "testing" "github.com/ObolNetwork/obol-stack/internal/monetizeapi" ) +// --- isSidecarUpstreamGone (delete-drain finalizer hang regression) ---------- + +// TestIsSidecarUpstreamGone locks in the fix for the delete-drain hang: a +// PurchaseRequest stuck in Terminating because reconcileDeletingPurchase kept +// Remaining>0 forever when checkBuyerStatus reported the upstream gone. Only +// the specific "not found in sidecar status" signal must count as drained; +// transient errors (e.g. the litellm pod unreachable) must NOT. +func TestIsSidecarUpstreamGone(t *testing.T) { + cases := []struct { + name string + err error + want bool + }{ + {"nil error is not gone", nil, false}, + {"upstream gone (bare)", errors.New(`upstream "docuseal" not found in sidecar status`), true}, + {"upstream gone (wrapped)", fmt.Errorf("checkBuyerStatus: %w", errors.New(`upstream "x" not found in sidecar status`)), true}, + {"transient: no litellm pods is NOT gone", errors.New("no litellm pods in llm"), false}, + {"transient: connection refused is NOT gone", errors.New("dial tcp 10.0.0.1:8402: connect: connection refused"), false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := isSidecarUpstreamGone(tc.err); got != tc.want { + t.Errorf("isSidecarUpstreamGone(%v) = %v, want %v", tc.err, got, tc.want) + } + }) + } +} + // --- purchaseConditionIsTrue ------------------------------------------------ func TestPurchaseConditionIsTrue(t *testing.T) { From 2bcf7098331c18e8e67cae656502b86f54c44286 Mon Sep 17 00:00:00 2001 From: bussyjd Date: Wed, 3 Jun 2026 10:24:30 +0400 Subject: [PATCH 4/7] fix(buy-x402): make status/list 'remaining' expiry-aware buy.py status/list showed the raw sidecar 'remaining' count, so an all-expired Permit2 auth pool read as ready to spend. Add _auth_deadline / _count_valid_auths and surface expired auths in both commands so an operator or agent tops up instead of burning expired vouchers into 503s. --- internal/embed/skills/buy-x402/scripts/buy.py | 65 ++++++++++++++++++- 1 file changed, 63 insertions(+), 2 deletions(-) diff --git a/internal/embed/skills/buy-x402/scripts/buy.py b/internal/embed/skills/buy-x402/scripts/buy.py index 0b5e3765..7c482983 100644 --- a/internal/embed/skills/buy-x402/scripts/buy.py +++ b/internal/embed/skills/buy-x402/scripts/buy.py @@ -499,6 +499,60 @@ def _active_auth_pool(existing_auths, live_status): return auths[-remaining:] +def _auth_deadline(auth): + """Return the unix expiry of a pre-signed auth, or None if none is set. + + Permit2 (OBOL) vouchers carry permit2Authorization.deadline; ERC-3009 (USDC) + vouchers carry authorization.validBefore (USDC uses a far-future value). The + legacy flat validBefore is a fallback. Mirrors authDeadlineUnix in the Go + buyer sidecar (internal/x402/buyer/signer.go).""" + auth = auth or {} + payload = (auth.get("payment") or {}).get("payload") or {} + for value in ( + (payload.get("permit2Authorization") or {}).get("deadline"), + (payload.get("authorization") or {}).get("validBefore"), + auth.get("validBefore"), + ): + if value is None: + continue + try: + return int(value) + except (TypeError, ValueError): + continue + return None + + +def _count_valid_auths(auths, now=None): + """Split an auth pool into (valid, expired) counts by on-chain deadline. + + Auths with no discoverable deadline count as valid (USDC validBefore is + ~year 2106). Used to make the displayed "remaining" expiry-aware so an + operator/agent does not treat an all-expired pool as ready to spend.""" + if now is None: + now = int(time.time()) + valid = expired = 0 + for a in auths or []: + deadline = _auth_deadline(a) + if deadline is not None and deadline <= now: + expired += 1 + else: + valid += 1 + return valid, expired + + +def _expired_in_active_pool(spec, live_status): + """Count expired auths among the currently-active pool, defensively. + + Falls back to the full preSignedAuths list if the live remaining count + can't be reconciled with the pool (e.g. sidecar mid-reload).""" + try: + pool = _active_auth_pool(spec.get("preSignedAuths"), live_status) + except ValueError: + pool = spec.get("preSignedAuths") or [] + _, expired = _count_valid_auths(pool) + return expired + + def _build_active_auth_pool(existing_auths, live_status, new_auths): return _active_auth_pool(existing_auths, live_status) + list(new_auths or []) @@ -1558,9 +1612,11 @@ def cmd_list(): alias = live.get("public_model") or status.get("publicModel") or f"paid/{spec.get('model', name)}" price = (spec.get("payment") or {}).get("price", "?") chain = live.get("network") or (spec.get("payment") or {}).get("network", "?") + expired = _expired_in_active_pool(spec, live) + remaining_col = f"{remaining} ({expired} expired)" if expired else f"{remaining}" print(f"{name:<20} {alias:<32} " f"{str(price):<12} {chain:<15} " - f"{remaining}") + f"{remaining_col}") # --------------------------------------------------------------------------- @@ -1585,7 +1641,12 @@ def cmd_status(name): print(f"Endpoint: {live_status.get('url') or _normalize_endpoint(spec.get('endpoint', '?'))}") print(f"Model: {live_status.get('remote_model') or spec.get('model', '?')}") print(f"Chain: {live_status.get('network') or (spec.get('payment') or {}).get('network', '?')}") - print(f"Auths remaining: {live_status.get('remaining', status.get('remaining', 0))}") + remaining_display = live_status.get('remaining', status.get('remaining', 0)) + expired = _expired_in_active_pool(spec, live_status) + print(f"Auths remaining: {remaining_display}") + if expired: + print(f" WARNING: {expired} of {remaining_display} remaining auth(s) are EXPIRED and unusable; " + f"run `buy {name} ...` to top up with fresh authorizations") print(f"Auths spent: {live_status.get('spent', status.get('spent', 0))}") print() From 158e2a277b7662b5399d04fffe525702e324d488 Mon Sep 17 00:00:00 2001 From: bussyjd Date: Wed, 3 Jun 2026 10:24:30 +0400 Subject: [PATCH 5/7] fix(x402): suppress verifyOnly=false warning on the in-process settle path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit HandleProxy (and the standalone inference gateway) rebuild the ForwardAuth middleware per request with VerifyOnly=false by design — they proxy to the real upstream and settle only after a <400 response — so the verifyOnly=false warning fired on every paid request telling operators to 'fix' correct config. Add a SettlesInProcess flag that suppresses the warning on those paths while leaving the genuinely-dangerous Traefik ForwardAuth path loud. --- internal/inference/gateway.go | 4 ++++ internal/x402/forwardauth.go | 11 ++++++++++- internal/x402/forwardauth_test.go | 33 +++++++++++++++++++++++++++++++ internal/x402/verifier.go | 7 ++++++- 4 files changed, 53 insertions(+), 2 deletions(-) diff --git a/internal/inference/gateway.go b/internal/inference/gateway.go index 9bfb1e6e..0c8f6b33 100644 --- a/internal/inference/gateway.go +++ b/internal/inference/gateway.go @@ -181,6 +181,10 @@ func (g *Gateway) buildHandler(upstreamURL string) (http.Handler, error) { paymentMiddleware := x402pkg.NewForwardAuthMiddleware(x402pkg.ForwardAuthConfig{ FacilitatorURL: g.config.FacilitatorURL, VerifyOnly: g.config.VerifyOnly, + // The standalone inference gateway is in-process and settlement-aware, + // so a configured VerifyOnly=false is correct by design — suppress the + // misleading per-request warning on this path. + SettlesInProcess: true, }, []x402types.PaymentRequirements{requirement}) // Initialise key backend: TEE (Linux) or SE (macOS), mutually exclusive. diff --git a/internal/x402/forwardauth.go b/internal/x402/forwardauth.go index 107f960f..00e24d2b 100644 --- a/internal/x402/forwardauth.go +++ b/internal/x402/forwardauth.go @@ -47,6 +47,15 @@ type ForwardAuthConfig struct { // "ways to pay" prompts) while x402-aware clients keep getting JSON. // Nil keeps today's behaviour: every 402 is JSON. SendPaymentRequired SendPaymentRequiredFunc + + // SettlesInProcess marks the in-process seller-gateway path (HandleProxy / + // obol sell inference) where VerifyOnly=false is correct BY DESIGN — the + // middleware proxies to the real upstream and settles only after a <400 + // response, so the verifyOnly=false warning would be misleading noise on + // every paid request. When true, that warning is suppressed. It does NOT + // change settlement behaviour; the genuinely-dangerous Traefik ForwardAuth + // path leaves this false and still warns if an operator flips VerifyOnly. + SettlesInProcess bool } // facilitatorVerifyRequest is the JSON body sent to POST /verify and /settle. @@ -97,7 +106,7 @@ func NewForwardAuthMiddleware(cfg ForwardAuthConfig, requirements []x402types.Pa verifyClient := &http.Client{Timeout: facilitatorVerifyTimeout} settleClient := &http.Client{Timeout: facilitatorSettleTimeout} - if !cfg.VerifyOnly { + if !cfg.VerifyOnly && !cfg.SettlesInProcess { log.Printf("x402: WARNING verifyOnly=false — settlement will run after upstream success. " + "This is ONLY safe for in-process middleware (e.g. obol sell inference) that sees " + "the real upstream status. Behind Traefik ForwardAuth this debits the payer before " + diff --git a/internal/x402/forwardauth_test.go b/internal/x402/forwardauth_test.go index 5cee3fc9..dd36619e 100644 --- a/internal/x402/forwardauth_test.go +++ b/internal/x402/forwardauth_test.go @@ -497,3 +497,36 @@ func TestForwardAuth_VerifyOnlyTrue_NoStartupWarning(t *testing.T) { t.Fatalf("did not expect verifyOnly warning when VerifyOnly=true, got:\n%s", gotLog) } } + +// TestForwardAuth_SettlesInProcess_SuppressesWarning pins the fix for the +// per-request log spam on the in-process seller gateway (HandleProxy / obol +// sell inference): that path sets VerifyOnly=false BY DESIGN (it proxies to the +// real upstream and settles after a <400 response), so the verifyOnly=false +// warning is misleading there. SettlesInProcess=true must silence it while +// leaving the dangerous Traefik ForwardAuth path (SettlesInProcess=false) loud. +func TestForwardAuth_SettlesInProcess_SuppressesWarning(t *testing.T) { + var buf bytes.Buffer + + origFlags := log.Flags() + origOutput := log.Writer() + + log.SetFlags(0) + log.SetOutput(&buf) + t.Cleanup(func() { + log.SetFlags(origFlags) + log.SetOutput(origOutput) + }) + + _ = NewForwardAuthMiddleware(ForwardAuthConfig{ + FacilitatorURL: "http://example.invalid", + VerifyOnly: false, + SettlesInProcess: true, + }, []x402types.PaymentRequirements{{ + Scheme: "exact", + Network: "eip155:84532", + }}) + + if gotLog := buf.String(); strings.Contains(gotLog, "verifyOnly=false") { + t.Fatalf("SettlesInProcess=true must suppress the verifyOnly=false warning, got:\n%s", gotLog) + } +} diff --git a/internal/x402/verifier.go b/internal/x402/verifier.go index 295460dd..ab75fe18 100644 --- a/internal/x402/verifier.go +++ b/internal/x402/verifier.go @@ -239,8 +239,13 @@ func (v *Verifier) HandleProxy(w http.ResponseWriter, r *http.Request) { display := buildPaymentDisplay(rule, chain, asset, wallet, requirement.Amount) middleware := NewForwardAuthMiddleware(ForwardAuthConfig{ - FacilitatorURL: cfg.FacilitatorURL, + FacilitatorURL: cfg.FacilitatorURL, + // HandleProxy is the in-process seller gateway: it proxies to the real + // upstream and settles only after a <400 response, so verifyOnly=false + // is correct here. SettlesInProcess suppresses the (otherwise + // per-request) verifyOnly=false warning on this safe path. VerifyOnly: false, + SettlesInProcess: true, Extensions: extensions, SendPaymentRequired: NewHTMLAwarePaymentRequired(display), }, []x402types.PaymentRequirements{requirement}) From b90118fd71c130963a48f76f187fe84ac6f310d4 Mon Sep 17 00:00:00 2001 From: bussyjd Date: Wed, 3 Jun 2026 10:24:31 +0400 Subject: [PATCH 6/7] fix(agent): validate --model against LiteLLM at agent creation obol agent new --model X provisioned cleanly for an unknown model, then every chat call failed with 'no healthy deployments for this model'. Add a preflight in createCRDAgent that checks a non-empty --model against the LiteLLM registry and fails fast with the available models. A transient list error warns and continues; an empty model still lets the controller auto-pin. --- cmd/obol/agent_crd.go | 45 ++++++++++++++++++++++++++++++++++++++++++ cmd/obol/agent_test.go | 26 ++++++++++++++++++++++++ 2 files changed, 71 insertions(+) diff --git a/cmd/obol/agent_crd.go b/cmd/obol/agent_crd.go index 7892982e..82d7c6b2 100644 --- a/cmd/obol/agent_crd.go +++ b/cmd/obol/agent_crd.go @@ -7,6 +7,7 @@ import ( "github.com/ObolNetwork/obol-stack/internal/agentcrd" "github.com/ObolNetwork/obol-stack/internal/config" "github.com/ObolNetwork/obol-stack/internal/kubectl" + "github.com/ObolNetwork/obol-stack/internal/model" "github.com/ObolNetwork/obol-stack/internal/ui" "github.com/urfave/cli/v3" ) @@ -40,6 +41,43 @@ type createCRDAgentOptions struct { Interactive bool // when true, prompt for any missing fields with sensible defaults } +// validatePinnedModel rejects a non-empty --model that LiteLLM doesn't serve, +// so `obol agent new --model X` fails at creation time instead of provisioning +// an agent whose every chat call returns "no healthy deployments for this +// model". A transient registry-list error is a warning, not a hard failure; +// an empty model is always allowed (the controller auto-pins the cluster +// default at first reconcile). +func validatePinnedModel(cfg *config.Config, u *ui.UI, modelName string) error { + if strings.TrimSpace(modelName) == "" { + return nil + } + configured, err := model.GetConfiguredModels(cfg) + if err != nil { + u.Dim(fmt.Sprintf("Could not verify model %q against LiteLLM (%v); continuing", modelName, err)) + return nil + } + if len(configured) > 0 && !isModelConfigured(modelName, configured) { + return fmt.Errorf("model %q is not configured in LiteLLM\n Available: %s\n Run `obol model list` to see models, or omit --model to let the controller auto-pin the cluster default", + modelName, strings.Join(configured, ", ")) + } + return nil +} + +// isModelConfigured reports whether name is an exact entry in the LiteLLM model +// list. The `paid/*` wildcard is a routing namespace, not a callable model, so +// it is never accepted as a pin (mirrors pickAgentDefault in sell_agent.go). +func isModelConfigured(name string, configured []string) bool { + if name == "paid/*" { + return false + } + for _, m := range configured { + if m == name { + return true + } + } + return false +} + // createCRDAgent does the actual host-seed + kubectl-apply work. Returns // when the Agent CR is in place; the controller takes over from there. func createCRDAgent(cfg *config.Config, u *ui.UI, opts createCRDAgentOptions) error { @@ -73,6 +111,13 @@ func createCRDAgent(cfg *config.Config, u *ui.UI, opts createCRDAgentOptions) er createWallet = !strings.EqualFold(ans, "n") && !strings.EqualFold(ans, "no") } + // Fail fast on a pinned model LiteLLM doesn't serve. Without this the Agent + // CR provisions cleanly but every chat call returns "no healthy deployments + // for this model", which is hard to trace back to the typo. + if err := validatePinnedModel(cfg, u, model); err != nil { + return err + } + skills, err := agentcrd.ParseSkills(skillsCSV) if err != nil { return err diff --git a/cmd/obol/agent_test.go b/cmd/obol/agent_test.go index 43665ca7..4e7f5e52 100644 --- a/cmd/obol/agent_test.go +++ b/cmd/obol/agent_test.go @@ -398,3 +398,29 @@ func mkdirAgentInstance(t *testing.T, cfg *config.Config, runtime agentruntime.R t.Fatalf("create %s instance %q: %v", runtime, id, err) } } + +// TestIsModelConfigured covers the membership check behind the `obol agent new +// --model X` preflight: known models pass, unknown ones are rejected, and the +// `paid/*` wildcard meta route is never accepted as a pin. +func TestIsModelConfigured(t *testing.T) { + configured := []string{"qwen3.6", "qwen3.5:9b", "paid/*", "paid/aeon"} + cases := []struct { + name string + model string + want bool + }{ + {"known model", "qwen3.6", true}, + {"known tagged model", "qwen3.5:9b", true}, + {"concrete paid model", "paid/aeon", true}, + {"unknown model", "gpt-9000", false}, + {"wildcard meta route is not a pin", "paid/*", false}, + {"empty is not a member", "", false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := isModelConfigured(tc.model, configured); got != tc.want { + t.Errorf("isModelConfigured(%q) = %v, want %v", tc.model, got, tc.want) + } + }) + } +} From 8cfd0ca121d89212d19b9701003171960bad66b4 Mon Sep 17 00:00:00 2001 From: bussyjd Date: Wed, 3 Jun 2026 13:34:51 +0400 Subject: [PATCH 7/7] test(flows): retry dual-stack stack-up on transient k3d/Docker mount races MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Docker Desktop on macOS intermittently fails to create the gRPC-FUSE mount source for a k3d node's workspace data dir under sustained cluster-churn ("error while creating mount source path ...: no such file or directory"), so the k3s node never reports ready and k3d rolls the cluster back. The host dir exists; it's a daemon-side file-sharing race. The dual-stack stack-up loop already retries port-bind and image/Helm transients — extend it to retry this mount race (a fresh cluster on retry clears it) so the release smoke isn't flaked by an environment-side Docker hiccup. --- flows/lib-dual-stack.sh | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/flows/lib-dual-stack.sh b/flows/lib-dual-stack.sh index e650bc7a..3dc73091 100644 --- a/flows/lib-dual-stack.sh +++ b/flows/lib-dual-stack.sh @@ -139,6 +139,18 @@ stack_init_and_up_with_retry() { sleep 10 continue fi + # Docker Desktop (macOS) intermittently fails to create the gRPC-FUSE + # mount source for a k3d node's workspace dir under sustained + # cluster-churn, so the k3s node never reports ready and k3d rolls the + # cluster back. The host dir exists (reset_flow_workspace mkdir's it); + # this is a daemon-side file-sharing race. A fresh cluster on retry + # clears it. + if [ "$attempt" -lt 3 ] && echo "$out" | grep -qiE "error while creating mount source path|failed to get ready: error waiting for log line|status=restarting|Cluster creation FAILED"; then + echo " $label stack up hit a transient k3d/Docker mount race; retrying (attempt $((attempt + 1))/3)" + "$runner" stack down >/dev/null 2>&1 || true + sleep 10 + continue + fi fail "$label: stack up failed (exit $rc)" emit_metrics