fix(rc9): GA blockers + monetize buy-side fixes from the v0.10.0-rc9 report

cmd/obol/agent_crd.go

@@ -7,6 +7,7 @@ import (
 	"github.com/ObolNetwork/obol-stack/internal/agentcrd"
 	"github.com/ObolNetwork/obol-stack/internal/config"
 	"github.com/ObolNetwork/obol-stack/internal/kubectl"
+	"github.com/ObolNetwork/obol-stack/internal/model"
 	"github.com/ObolNetwork/obol-stack/internal/ui"
 	"github.com/urfave/cli/v3"
 )
@@ -40,6 +41,43 @@ type createCRDAgentOptions struct {
 	Interactive  bool // when true, prompt for any missing fields with sensible defaults
 }
 
+// validatePinnedModel rejects a non-empty --model that LiteLLM doesn't serve,
+// so `obol agent new --model X` fails at creation time instead of provisioning
+// an agent whose every chat call returns "no healthy deployments for this
+// model". A transient registry-list error is a warning, not a hard failure;
+// an empty model is always allowed (the controller auto-pins the cluster
+// default at first reconcile).
+func validatePinnedModel(cfg *config.Config, u *ui.UI, modelName string) error {
+	if strings.TrimSpace(modelName) == "" {
+		return nil
+	}
+	configured, err := model.GetConfiguredModels(cfg)
+	if err != nil {
+		u.Dim(fmt.Sprintf("Could not verify model %q against LiteLLM (%v); continuing", modelName, err))
+		return nil
+	}
+	if len(configured) > 0 && !isModelConfigured(modelName, configured) {
+		return fmt.Errorf("model %q is not configured in LiteLLM\n  Available: %s\n  Run `obol model list` to see models, or omit --model to let the controller auto-pin the cluster default",
+			modelName, strings.Join(configured, ", "))
+	}
+	return nil
+}
+
+// isModelConfigured reports whether name is an exact entry in the LiteLLM model
+// list. The `paid/*` wildcard is a routing namespace, not a callable model, so
+// it is never accepted as a pin (mirrors pickAgentDefault in sell_agent.go).
+func isModelConfigured(name string, configured []string) bool {
+	if name == "paid/*" {
+		return false
+	}
+	for _, m := range configured {
+		if m == name {
+			return true
+		}
+	}
+	return false
+}
+
 // createCRDAgent does the actual host-seed + kubectl-apply work. Returns
 // when the Agent CR is in place; the controller takes over from there.
 func createCRDAgent(cfg *config.Config, u *ui.UI, opts createCRDAgentOptions) error {
@@ -73,6 +111,13 @@ func createCRDAgent(cfg *config.Config, u *ui.UI, opts createCRDAgentOptions) er
 		createWallet = !strings.EqualFold(ans, "n") && !strings.EqualFold(ans, "no")
 	}
 
+	// Fail fast on a pinned model LiteLLM doesn't serve. Without this the Agent
+	// CR provisions cleanly but every chat call returns "no healthy deployments
+	// for this model", which is hard to trace back to the typo.
+	if err := validatePinnedModel(cfg, u, model); err != nil {
+		return err
+	}
+
 	skills, err := agentcrd.ParseSkills(skillsCSV)
 	if err != nil {
 		return err

cmd/obol/agent_test.go

@@ -398,3 +398,29 @@ func mkdirAgentInstance(t *testing.T, cfg *config.Config, runtime agentruntime.R
 		t.Fatalf("create %s instance %q: %v", runtime, id, err)
 	}
 }
+
+// TestIsModelConfigured covers the membership check behind the `obol agent new
+// --model X` preflight: known models pass, unknown ones are rejected, and the
+// `paid/*` wildcard meta route is never accepted as a pin.
+func TestIsModelConfigured(t *testing.T) {
+	configured := []string{"qwen3.6", "qwen3.5:9b", "paid/*", "paid/aeon"}
+	cases := []struct {
+		name  string
+		model string
+		want  bool
+	}{
+		{"known model", "qwen3.6", true},
+		{"known tagged model", "qwen3.5:9b", true},
+		{"concrete paid model", "paid/aeon", true},
+		{"unknown model", "gpt-9000", false},
+		{"wildcard meta route is not a pin", "paid/*", false},
+		{"empty is not a member", "", false},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			if got := isModelConfigured(tc.model, configured); got != tc.want {
+				t.Errorf("isModelConfigured(%q) = %v, want %v", tc.model, got, tc.want)
+			}
+		})
+	}
+}

flows/lib-dual-stack.sh

@@ -139,6 +139,18 @@ stack_init_and_up_with_retry() {
             sleep 10
             continue
         fi
+        # Docker Desktop (macOS) intermittently fails to create the gRPC-FUSE
+        # mount source for a k3d node's workspace dir under sustained
+        # cluster-churn, so the k3s node never reports ready and k3d rolls the
+        # cluster back. The host dir exists (reset_flow_workspace mkdir's it);
+        # this is a daemon-side file-sharing race. A fresh cluster on retry
+        # clears it.
+        if [ "$attempt" -lt 3 ] && echo "$out" | grep -qiE "error while creating mount source path|failed to get ready: error waiting for log line|status=restarting|Cluster creation FAILED"; then
+            echo "  $label stack up hit a transient k3d/Docker mount race; retrying (attempt $((attempt + 1))/3)"
+            "$runner" stack down >/dev/null 2>&1 || true
+            sleep 10
+            continue
+        fi
 
         fail "$label: stack up failed (exit $rc)"
         emit_metrics

internal/embed/embed_crd_test.go

@@ -791,6 +791,28 @@ func TestX402VerifierImage_CarriesAgentAuthFix(t *testing.T) {
 	}
 }
 
+// TestServiceOfferControllerImage_CarriesSecretCreateOnlyFix is the tripwire
+// for the per-agent provisioning 403 (GA blocker). The controller ClusterRole
+// in x402.yaml grants no secrets update/patch verb because the reconciler
+// treats Secret as create-only (isCreateOnlyKind). The image MUST therefore be
+// built from source that has that behaviour — the prior f5d94fc side-branch pin
+// did not, so the deployed binary Updated the per-agent Secrets on re-reconcile
+// and 403'd. 503016b (rc9 commit 503016bf, image 0.10.0-rc9) carries the fix.
+// Bumping this pin requires a conscious, documented change here so a future
+// downgrade can't silently re-ship the bug.
+func TestServiceOfferControllerImage_CarriesSecretCreateOnlyFix(t *testing.T) {
+	data, err := ReadInfrastructureFile("base/templates/x402.yaml")
+	if err != nil {
+		t.Fatalf("ReadInfrastructureFile: %v", err)
+	}
+
+	const ref = "ghcr.io/obolnetwork/serviceoffer-controller:503016b@sha256:bec62ea04842caf62980b529a89f5d553987a106c3167eb45209a8b278121957"
+	if !strings.Contains(string(data), "image: "+ref) {
+		t.Fatalf("serviceoffer-controller image must carry the Secret-create-only reconciler fix "+
+			"(else per-agent provisioning 403s under the no-update/patch Secret RBAC): %s", ref)
+	}
+}
+
 // TestServiceOfferControllerSecretRBAC_Scoped guards the controller's Secret
 // access against the actual code paths in internal/serviceoffercontroller.
 // The reconciler touches exactly three Secrets by name (litellm-secrets,

internal/embed/embed_image_pin_test.go

@@ -230,8 +230,12 @@ func TestEmbeddedImages_X402ControllerAndBuyerUseFixPins(t *testing.T) {
 		ref  string
 	}{
 		{
+			// Repinned off the f5d94fc side-branch build (which predated the
+			// Secret-create-only reconciler change) to the rc9 release image
+			// (commit 503016b, image 0.10.0-rc9).
+			// See TestServiceOfferControllerImage_CarriesSecretCreateOnlyFix.
 			file: "base/templates/x402.yaml",
-			ref:  "ghcr.io/obolnetwork/serviceoffer-controller:f5d94fc@sha256:c6aa6259e3a6bc61a5f4f7203d8c68cfdd861a8d365f9629d234d13b949bf48e",
+			ref:  "ghcr.io/obolnetwork/serviceoffer-controller:503016b@sha256:bec62ea04842caf62980b529a89f5d553987a106c3167eb45209a8b278121957",
 		},
 		{
 			file: "base/templates/llm.yaml",

internal/embed/infrastructure/base/templates/x402.yaml

@@ -343,7 +343,15 @@ spec:
           type: RuntimeDefault
       containers:
         - name: controller
-          image: ghcr.io/obolnetwork/serviceoffer-controller:f5d94fc@sha256:c6aa6259e3a6bc61a5f4f7203d8c68cfdd861a8d365f9629d234d13b949bf48e
+          # MUST be an image whose reconciler treats Secret as create-only
+          # (isCreateOnlyKind includes "Secret"). The ClusterRole above grants
+          # NO secrets update/patch verb, so a controller that Updates the
+          # per-agent hermes-api-server / remote-signer-keystore Secrets on
+          # re-reconcile 403s and per-agent provisioning never converges. The
+          # prior pin (f5d94fc) predated that source change and shipped the
+          # bug; 503016b (rc9 commit 503016bf, image 0.10.0-rc9) carries it.
+          # See TestServiceOfferControllerImage_CarriesSecretCreateOnlyFix.
+          image: ghcr.io/obolnetwork/serviceoffer-controller:503016b@sha256:bec62ea04842caf62980b529a89f5d553987a106c3167eb45209a8b278121957
           imagePullPolicy: IfNotPresent
           securityContext:
             allowPrivilegeEscalation: false

internal/embed/skills/buy-x402/scripts/buy.py

@@ -499,6 +499,60 @@ def _active_auth_pool(existing_auths, live_status):
     return auths[-remaining:]
 
 
+def _auth_deadline(auth):
+    """Return the unix expiry of a pre-signed auth, or None if none is set.
+
+    Permit2 (OBOL) vouchers carry permit2Authorization.deadline; ERC-3009 (USDC)
+    vouchers carry authorization.validBefore (USDC uses a far-future value). The
+    legacy flat validBefore is a fallback. Mirrors authDeadlineUnix in the Go
+    buyer sidecar (internal/x402/buyer/signer.go)."""
+    auth = auth or {}
+    payload = (auth.get("payment") or {}).get("payload") or {}
+    for value in (
+        (payload.get("permit2Authorization") or {}).get("deadline"),
+        (payload.get("authorization") or {}).get("validBefore"),
+        auth.get("validBefore"),
+    ):
+        if value is None:
+            continue
+        try:
+            return int(value)
+        except (TypeError, ValueError):
+            continue
+    return None
+
+
+def _count_valid_auths(auths, now=None):
+    """Split an auth pool into (valid, expired) counts by on-chain deadline.
+
+    Auths with no discoverable deadline count as valid (USDC validBefore is
+    ~year 2106). Used to make the displayed "remaining" expiry-aware so an
+    operator/agent does not treat an all-expired pool as ready to spend."""
+    if now is None:
+        now = int(time.time())
+    valid = expired = 0
+    for a in auths or []:
+        deadline = _auth_deadline(a)
+        if deadline is not None and deadline <= now:
+            expired += 1
+        else:
+            valid += 1
+    return valid, expired
+
+
+def _expired_in_active_pool(spec, live_status):
+    """Count expired auths among the currently-active pool, defensively.
+
+    Falls back to the full preSignedAuths list if the live remaining count
+    can't be reconciled with the pool (e.g. sidecar mid-reload)."""
+    try:
+        pool = _active_auth_pool(spec.get("preSignedAuths"), live_status)
+    except ValueError:
+        pool = spec.get("preSignedAuths") or []
+    _, expired = _count_valid_auths(pool)
+    return expired
+
+
 def _build_active_auth_pool(existing_auths, live_status, new_auths):
     return _active_auth_pool(existing_auths, live_status) + list(new_auths or [])
 
@@ -1558,9 +1612,11 @@ def cmd_list():
         alias = live.get("public_model") or status.get("publicModel") or f"paid/{spec.get('model', name)}"
         price = (spec.get("payment") or {}).get("price", "?")
         chain = live.get("network") or (spec.get("payment") or {}).get("network", "?")
+        expired = _expired_in_active_pool(spec, live)
+        remaining_col = f"{remaining} ({expired} expired)" if expired else f"{remaining}"
         print(f"{name:<20} {alias:<32} "
               f"{str(price):<12} {chain:<15} "
-              f"{remaining}")
+              f"{remaining_col}")
 
 
 # ---------------------------------------------------------------------------
@@ -1585,7 +1641,12 @@ def cmd_status(name):
     print(f"Endpoint: {live_status.get('url') or _normalize_endpoint(spec.get('endpoint', '?'))}")
     print(f"Model:    {live_status.get('remote_model') or spec.get('model', '?')}")
     print(f"Chain:    {live_status.get('network') or (spec.get('payment') or {}).get('network', '?')}")
-    print(f"Auths remaining: {live_status.get('remaining', status.get('remaining', 0))}")
+    remaining_display = live_status.get('remaining', status.get('remaining', 0))
+    expired = _expired_in_active_pool(spec, live_status)
+    print(f"Auths remaining: {remaining_display}")
+    if expired:
+        print(f"  WARNING: {expired} of {remaining_display} remaining auth(s) are EXPIRED and unusable; "
+              f"run `buy {name} ...` to top up with fresh authorizations")
     print(f"Auths spent:     {live_status.get('spent', status.get('spent', 0))}")
     print()
 

internal/inference/gateway.go

@@ -181,6 +181,10 @@ func (g *Gateway) buildHandler(upstreamURL string) (http.Handler, error) {
 	paymentMiddleware := x402pkg.NewForwardAuthMiddleware(x402pkg.ForwardAuthConfig{
 		FacilitatorURL: g.config.FacilitatorURL,
 		VerifyOnly:     g.config.VerifyOnly,
+		// The standalone inference gateway is in-process and settlement-aware,
+		// so a configured VerifyOnly=false is correct by design — suppress the
+		// misleading per-request warning on this path.
+		SettlesInProcess: true,
 	}, []x402types.PaymentRequirements{requirement})
 
 	// Initialise key backend: TEE (Linux) or SE (macOS), mutually exclusive.

internal/serviceoffercontroller/purchase.go

@@ -95,6 +95,16 @@ func (c *Controller) reconcilePurchase(ctx context.Context, key string) error {
 	return nil
 }
 
+// isSidecarUpstreamGone reports whether a checkBuyerStatus error means the
+// x402-buyer sidecar no longer lists the upstream at all. That is the signal
+// that a delete-drain is complete: there is nothing left for the sidecar to
+// consume. It is deliberately a string match on the error checkBuyerStatus
+// returns (purchase_helpers.go) so both the entry switch and the terminal
+// cleanup check in reconcileDeletingPurchase agree on what "drained" means.
+func isSidecarUpstreamGone(err error) bool {
+	return err != nil && strings.Contains(err.Error(), "not found in sidecar status")
+}
+
 func (c *Controller) reconcileDeletingPurchase(ctx context.Context, pr *monetizeapi.PurchaseRequest, raw *unstructured.Unstructured) error {
 	buyerNS := pr.EffectiveBuyerNamespace()
 	status := pr.Status
@@ -105,6 +115,17 @@ func (c *Controller) reconcileDeletingPurchase(ctx context.Context, pr *monetize
 	case err == nil:
 		status.Remaining = remaining
 		status.Spent = spent
+	case isSidecarUpstreamGone(err):
+		// The sidecar no longer reports this upstream. That deleted-upstream
+		// signal IS the drain-done signal: collapse Remaining to 0 so cleanup
+		// and finalizer removal proceed below. Without this case, a Configured
+		// purchase with Remaining>0 fell through to the next case, kept
+		// Remaining>0, and requeued every 5s forever — stranding the CR in
+		// Terminating until its finalizer was force-removed by hand. The
+		// terminal not-found check at the end of this function already treats
+		// the same signal as drain-complete; this keeps the entry switch
+		// consistent with it.
+		status.Remaining = 0
 	case purchaseConditionIsTrue(status.Conditions, "Configured") && status.Remaining > 0:
 		log.Printf("purchase %s/%s: delete drain waiting for sidecar status: %v", pr.Namespace, pr.Name, err)
 	default:
@@ -146,7 +167,7 @@ func (c *Controller) reconcileDeletingPurchase(ctx context.Context, pr *monetize
 		}
 		c.purchaseQueue.AddAfter(pr.Namespace+"/"+pr.Name, 5*time.Second)
 		return nil
-	} else if !strings.Contains(err.Error(), "not found in sidecar status") {
+	} else if !isSidecarUpstreamGone(err) {
 		setPurchaseCondition(
 			&status.Conditions,
 			"Deleting",

internal/serviceoffercontroller/purchase_pure_test.go

@@ -1,12 +1,42 @@
 package serviceoffercontroller
 
 import (
+	"errors"
+	"fmt"
 	"strings"
 	"testing"
 
 	"github.com/ObolNetwork/obol-stack/internal/monetizeapi"
 )
 
+// --- isSidecarUpstreamGone (delete-drain finalizer hang regression) ----------
+
+// TestIsSidecarUpstreamGone locks in the fix for the delete-drain hang: a
+// PurchaseRequest stuck in Terminating because reconcileDeletingPurchase kept
+// Remaining>0 forever when checkBuyerStatus reported the upstream gone. Only
+// the specific "not found in sidecar status" signal must count as drained;
+// transient errors (e.g. the litellm pod unreachable) must NOT.
+func TestIsSidecarUpstreamGone(t *testing.T) {
+	cases := []struct {
+		name string
+		err  error
+		want bool
+	}{
+		{"nil error is not gone", nil, false},
+		{"upstream gone (bare)", errors.New(`upstream "docuseal" not found in sidecar status`), true},
+		{"upstream gone (wrapped)", fmt.Errorf("checkBuyerStatus: %w", errors.New(`upstream "x" not found in sidecar status`)), true},
+		{"transient: no litellm pods is NOT gone", errors.New("no litellm pods in llm"), false},
+		{"transient: connection refused is NOT gone", errors.New("dial tcp 10.0.0.1:8402: connect: connection refused"), false},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			if got := isSidecarUpstreamGone(tc.err); got != tc.want {
+				t.Errorf("isSidecarUpstreamGone(%v) = %v, want %v", tc.err, got, tc.want)
+			}
+		})
+	}
+}
+
 // --- purchaseConditionIsTrue ------------------------------------------------
 
 func TestPurchaseConditionIsTrue(t *testing.T) {