From 3adaf17b2444a73f435bef4865381d2538670179 Mon Sep 17 00:00:00 2001 From: LIghtJUNction Date: Mon, 12 Oct 2026 13:41:48 +0800 Subject: [PATCH] fix(deploy): retain schema fence for controller lifetime --- .../deploycli/deploy_merchant_store_fence.go | 12 +- .../deploy_schema_maintenance_controller.go | 5 +- ..._schema_maintenance_fence_lifetime_test.go | 113 ++++++++++++++++++ .../deploy_schema_maintenance_owner.go | 4 +- 4 files changed, 130 insertions(+), 4 deletions(-) create mode 100644 apps/api-go/internal/deploycli/deploy_schema_maintenance_fence_lifetime_test.go diff --git a/apps/api-go/internal/deploycli/deploy_merchant_store_fence.go b/apps/api-go/internal/deploycli/deploy_merchant_store_fence.go index 22f68ab64..f6216e3ca 100644 --- a/apps/api-go/internal/deploycli/deploy_merchant_store_fence.go +++ b/apps/api-go/internal/deploycli/deploy_merchant_store_fence.go @@ -139,6 +139,16 @@ func acquireMerchantStartupBaselineFence(ctx context.Context, sealedValues map[s // Both authorities use this one physical-session implementation. The ordinary // entry above still requires its complete, distinct two-provider contract. func openMerchantStoreDeploymentFence(ctx context.Context, sealedValues map[string]string, expected *productionMerchantStoreWriterContract, startupOnly bool) (*productionMerchantStoreFence, error) { + return openMerchantStoreDeploymentFenceForOwner(ctx, ctx, sealedValues, expected, startupOnly) +} + +// Acquisition still uses the individual operation's budget. Only the owning +// controller's lifetime determines whether a retained session remains valid +// after that operation returns; cancelling an RPC must not cancel its owner. +func openMerchantStoreDeploymentFenceForOwner(ctx, ownerContext context.Context, sealedValues map[string]string, expected *productionMerchantStoreWriterContract, startupOnly bool) (*productionMerchantStoreFence, error) { + if ownerContext == nil || ownerContext.Err() != nil { + return nil, errors.New("merchant deployment fence owner lifetime is unavailable") + } dsn, err := productionDatabaseURL(sealedValues) if err != nil { return nil, err @@ -162,7 +172,7 @@ func openMerchantStoreDeploymentFence(ctx context.Context, sealedValues map[stri if err != nil { return nil, errors.New("merchant deployment fence dedicated connection is unavailable") } - lease := &productionMerchantStoreFence{connection: connection, ownerContext: ctx, identity: *expected, backendPID: int32(connection.PgConn().PID()), startupOnly: startupOnly} + lease := &productionMerchantStoreFence{connection: connection, ownerContext: ownerContext, identity: *expected, backendPID: int32(connection.PgConn().PID()), startupOnly: startupOnly} var locked bool if err := connection.QueryRow(ctx, "SELECT pg_catalog.pg_try_advisory_lock_shared($1)", merchantStoreDeploymentFenceKey).Scan(&locked); err != nil || !locked { _ = lease.Close() diff --git a/apps/api-go/internal/deploycli/deploy_schema_maintenance_controller.go b/apps/api-go/internal/deploycli/deploy_schema_maintenance_controller.go index dcd161c6e..c633aa758 100644 --- a/apps/api-go/internal/deploycli/deploy_schema_maintenance_controller.go +++ b/apps/api-go/internal/deploycli/deploy_schema_maintenance_controller.go @@ -21,6 +21,7 @@ import ( type productionSchemaMaintenanceController struct { actionMu sync.Mutex runtime *productionRuntime + ownerContext context.Context plan productionSchemaMaintenancePlan planPath, planSHA string origin productionSchemaMaintenanceOriginPlan @@ -66,7 +67,7 @@ func (runtime *productionRuntime) newSchemaMaintenanceController(ctx context.Con if !bytes.Equal(append(canonical, '\n'), raw) || binding.TransitionID != plan.TransitionID || binding.PlanPath != planPath || binding.PlanSHA256 != planSHA || binding.Host != origin.Host || binding.CapsulePath != origin.OldCapsule.Path || binding.CapsuleSHA256 != origin.OldCapsule.SHA256 || binding.Workspace != origin.Workspace || binding.GuardianSocket != origin.Guardian.Socket || binding.GuardianBindingPath != origin.Guardian.BindingPath || binding.GuardianProtocol != origin.Guardian.Protocol { return nil, errors.New("schema controller host projection differs from the canonical pre-change plan") } - c := &productionSchemaMaintenanceController{runtime: runtime, plan: plan, planPath: planPath, planSHA: planSHA, origin: origin} + c := &productionSchemaMaintenanceController{runtime: runtime, ownerContext: ctx, plan: plan, planPath: planPath, planSHA: planSHA, origin: origin} if err := c.checkPlan(ctx); err != nil { return nil, err } @@ -527,7 +528,7 @@ func (c *productionSchemaMaintenanceController) handle(ctx context.Context, requ if err != nil { return schemaMaintenanceRPCReply{}, err } - authority := productionSchemaMaintenanceAuthority{TransitionID: c.plan.TransitionID, PlanSHA256: c.planSHA, Workspace: c.origin.Workspace, Artifacts: artifacts, OldCapsule: old, OldCapsuleSHA256: c.origin.OldCapsule.SHA256, CheckPlan: c.checkNormalPlan, CheckStoppedOrigins: c.checkStoppedOrigins, CheckRecoveryOrigins: c.checkRecoveryOrigins, CheckBackup: c.checkBackup} + authority := productionSchemaMaintenanceAuthority{OwnerContext: c.ownerContext, TransitionID: c.plan.TransitionID, PlanSHA256: c.planSHA, Workspace: c.origin.Workspace, Artifacts: artifacts, OldCapsule: old, OldCapsuleSHA256: c.origin.OldCapsule.SHA256, CheckPlan: c.checkNormalPlan, CheckStoppedOrigins: c.checkStoppedOrigins, CheckRecoveryOrigins: c.checkRecoveryOrigins, CheckBackup: c.checkBackup} intent := map[string]string{"format": "lmm-schema-maintenance-claim-intent-v1", "transition_id": c.plan.TransitionID, "plan_sha256": c.planSHA} if err := c.runtime.startupBaselineNewFile(filepath.Join(c.origin.Workspace, "state", "schema-claim-intent.json"), intent); err != nil { return schemaMaintenanceRPCReply{}, errors.New("schema claim was already attempted; reconcile the exact original owner without repeating claim") diff --git a/apps/api-go/internal/deploycli/deploy_schema_maintenance_fence_lifetime_test.go b/apps/api-go/internal/deploycli/deploy_schema_maintenance_fence_lifetime_test.go new file mode 100644 index 000000000..f9880845d --- /dev/null +++ b/apps/api-go/internal/deploycli/deploy_schema_maintenance_fence_lifetime_test.go @@ -0,0 +1,113 @@ +//go:build !windows + +package deploycli + +import ( + "context" + "os" + "strings" + "testing" + "time" +) + +func TestSchemaMaintenanceFenceLifetimeSurvivesClaimRPCActualPostgres(t *testing.T) { + // Reuse the owned no-TCP cluster, never a configured application DSN. + f := newSchemaMaintenanceDeadPreWorkerPGFixture(t) + if err := f.lease.Close(); err != nil { + t.Fatal(err) + } + values := map[string]string{"SQL_DSN": strings.Replace(f.adminDSN, "lmm_dead_pre_admin", "lmm_dead_pre_business", 1)} + daemonContext, cancelDaemon := context.WithCancel(f.ctx) + defer cancelDaemon() + var lease *productionMerchantStoreFence + var owner productionMerchantStoreFenceOwner + var claimContext context.Context + // Match RPCHandle: cancel the bounded request immediately after its claim + // handler returns, before the next request observes or uses the same owner. + claimRPC := func() error { + var cancel context.CancelFunc + claimContext, cancel = context.WithTimeout(daemonContext, time.Minute) + defer cancel() + var err error + lease, err = openMerchantStoreDeploymentFenceForOwner(claimContext, daemonContext, values, f.contract, false) + if err != nil { + return err + } + owner = testSchemaMaintenanceOwner() + owner.HolderPID, owner.BackendPID = os.Getpid(), lease.backendPID + owner.SystemIdentifier, owner.Database, owner.DatabaseOID = f.contract.SystemIdentifier, f.contract.Database, f.contract.DatabaseOID + owner.Schema, owner.SchemaOID, owner.Role = f.contract.Schema, f.contract.SchemaOID, f.contract.Role + owner.ContractSHA256, owner.ProviderSHA256 = merchantStoreFenceContractSHA(f.contract), f.contract.Candidate.PayloadSHA256 + toolSHA, err := sha256File("/proc/self/exe") + if err != nil { + return err + } + return lease.claimOwner(claimContext, owner, toolSHA, nil) + } + if err := claimRPC(); err != nil { + t.Fatal(err) + } + defer lease.Close() + if claimContext.Err() != context.Canceled || daemonContext.Err() != nil { + t.Fatal("fixture did not cancel only the completed claim RPC") + } + physicalSession := lease.backendPID + fence := &productionSchemaMaintenanceFence{lease: lease, localLease: &schemaMaintenanceTestLease{}, authority: productionSchemaMaintenanceAuthority{ + OwnerContext: daemonContext, CheckPlan: func(context.Context) error { return nil }, CheckStoppedOrigins: func(context.Context) error { return nil }, + }} + for _, name := range []string{"first worker check", "next status or worker check"} { + t.Run(name, func(t *testing.T) { + requestContext, cancel := context.WithTimeout(daemonContext, time.Minute) + defer cancel() + if err := fence.Check(requestContext); err != nil { + t.Fatalf("completed claim RPC invalidated its durable owner: %v", err) + } + if lease.backendPID != physicalSession || lease.ownerKey != merchantStoreFenceOwnerKey(owner) { + t.Fatal("check replaced the original physical session or owner") + } + }) + } + // A cancelled operation still fails; the fix does not detach RPC budgets. + cancelledRequest, cancelRequest := context.WithCancel(daemonContext) + cancelRequest() + operationLease, err := openMerchantStoreDeploymentFenceForOwner(daemonContext, daemonContext, values, f.contract, false) + if err != nil { + t.Fatal(err) + } + defer operationLease.Close() + if err := operationLease.Check(cancelledRequest, f.contract); err == nil { + t.Fatal("cancelled worker request passed its physical PG check") + } + if other, err := openMerchantStoreDeploymentFenceForOwner(cancelledRequest, daemonContext, values, f.contract, false); err == nil { + _ = other.Close() + t.Fatal("cancelled claim request acquired a new physical session") + } + for _, ownerContext := range []context.Context{nil, cancelledRequest} { + if other, err := openMerchantStoreDeploymentFenceForOwner(f.ctx, ownerContext, values, f.contract, false); err == nil { + _ = other.Close() + t.Fatal("missing or cancelled controller lifetime acquired a fence") + } + } + // Ordinary/start callers retain the existing single-context semantics. + ordinaryContext, cancelOrdinary := context.WithCancel(f.ctx) + ordinary, err := openMerchantStoreDeploymentFence(ordinaryContext, values, f.contract, false) + if err != nil { + cancelOrdinary() + t.Fatal(err) + } + defer ordinary.Close() + cancelOrdinary() + if err := ordinary.Check(f.ctx, f.contract); err == nil { + t.Fatal("ordinary fence stopped following its own lifetime") + } + cancelDaemon() + if err := fence.Check(f.ctx); err == nil { + t.Fatal("cancelled daemon lifetime still authorized a worker") + } + admin := f.admin() + defer admin.Close(context.Background()) + var actual string + if err := admin.QueryRow(f.ctx, `SELECT value FROM dead_pre.options WHERE key=$1`, lease.ownerKey).Scan(&actual); err != nil || actual != lease.ownerValue { + t.Fatalf("lifetime cancellation deleted or changed the durable ACTIVE owner: %v", err) + } +} diff --git a/apps/api-go/internal/deploycli/deploy_schema_maintenance_owner.go b/apps/api-go/internal/deploycli/deploy_schema_maintenance_owner.go index a64f8d090..06740615a 100644 --- a/apps/api-go/internal/deploycli/deploy_schema_maintenance_owner.go +++ b/apps/api-go/internal/deploycli/deploy_schema_maintenance_owner.go @@ -25,6 +25,7 @@ func schemaMaintenanceHolderUnit(id string) string { // this authority after validating its immutable maintenance plan and observing // the fixed two origins. A caller boolean is not a closure or recovery proof. type productionSchemaMaintenanceAuthority struct { + OwnerContext context.Context TransitionID string PlanSHA256 string Workspace string @@ -52,6 +53,7 @@ func (runtime *productionRuntime) claimSchemaMaintenanceFence(ctx context.Contex return nil, err } if !productionIDPattern.MatchString(a.TransitionID) || !productionSHA256Pattern.MatchString(a.PlanSHA256) || + a.OwnerContext == nil || a.OwnerContext.Err() != nil || a.Workspace != schemaMaintenanceHostWorkspace(runtime.paths, a.TransitionID) || localLease == nil || a.CheckPlan == nil || a.CheckBackup == nil || a.CheckStoppedOrigins == nil || a.CheckRecoveryOrigins == nil || a.Artifacts.Format != 1 || a.Artifacts.StartupPolicy != "held" || a.OldCapsule.Format != 1 || a.OldCapsule.StartupPolicy != "per-start" { @@ -112,7 +114,7 @@ func (runtime *productionRuntime) claimSchemaMaintenanceFence(ctx context.Contex // The shared physical fence prevents activation. The sole durable typed // owner blocks ordinary/start claims, while the worker serializes owner // changes with ...0003 before taking ...0001 in its own DDL transaction. - lease, err := openMerchantStoreDeploymentFence(ctx, values, a.Artifacts.Writer, false) + lease, err := openMerchantStoreDeploymentFenceForOwner(ctx, a.OwnerContext, values, a.Artifacts.Writer, false) if err != nil { return nil, err }