diff --git a/.coverage-baseline b/.coverage-baseline index 3c2d8494..219e3542 100644 --- a/.coverage-baseline +++ b/.coverage-baseline @@ -1 +1 @@ -75.3 +75.7 diff --git a/api/v1alpha3/clusterwatchrule_types.go b/api/v1alpha3/clusterwatchrule_types.go index 4d36b183..dfb60a2d 100644 --- a/api/v1alpha3/clusterwatchrule_types.go +++ b/api/v1alpha3/clusterwatchrule_types.go @@ -108,6 +108,26 @@ type ClusterResourceRule struct { // +kubebuilder:validation:items:Pattern=`^[^/]*$` Resources []string `json:"resources"` + // ExcludeFieldManagers drops a live change whose last writer is one of these field + // managers, so a GitOps forward leg applying this branch back into the cluster does + // not have its own applies mirrored into Git. It is read from metadata.managedFields + // and needs no audit fact. Not evaluated for DELETE. See + // ResourceRule.ExcludeFieldManagers for the full semantics. + // +optional + // +kubebuilder:validation:MaxItems=32 + // +kubebuilder:validation:items:MinLength=1 + // +kubebuilder:validation:items:MaxLength=128 + ExcludeFieldManagers []string `json:"excludeFieldManagers,omitempty"` + + // ExcludeUsers drops a live change attributed to one of these identities. It requires + // author attribution and fails open when the author cannot be resolved. See + // ResourceRule.ExcludeUsers for the full semantics. + // +optional + // +kubebuilder:validation:MaxItems=32 + // +kubebuilder:validation:items:MinLength=1 + // +kubebuilder:validation:items:MaxLength=316 + ExcludeUsers []string `json:"excludeUsers,omitempty"` + // Scope defines whether this rule watches Cluster-scoped or Namespaced resources. // - "Cluster": For cluster-scoped resources (Nodes, ClusterRoles, CRDs, etc.). // - "Namespaced": For namespaced resources (Pods, Deployments, Secrets, etc.), diff --git a/api/v1alpha3/commitrequest_types.go b/api/v1alpha3/commitrequest_types.go index dc0e9c56..4da302da 100644 --- a/api/v1alpha3/commitrequest_types.go +++ b/api/v1alpha3/commitrequest_types.go @@ -42,6 +42,60 @@ type CommitRequestSpec struct { // +kubebuilder:validation:Minimum=0 // +kubebuilder:validation:Maximum=300 CloseDelaySeconds int32 `json:"closeDelaySeconds,omitempty"` + + // Author names the human this commit is for, instead of deriving them from an + // apiserver audit fact. It exists for an authenticated control plane that already + // knows who the user is — one that verified their token before impersonating them — + // and for any cluster whose apiserver audit flags the operator cannot set. + // + // Asserting an author is a privilege, not a field anyone may set. It is honored only + // when the requester holds the `assert-author` verb on the named GitTarget, in the + // style of `bind`, `escalate` and `impersonate`: + // + // rules: + // - apiGroups: ["configbutler.ai"] + // resources: ["gittargets"] + // resourceNames: ["tenants"] + // verbs: ["assert-author"] + // + // The check runs in the validate-operator-types admission webhook, which denies an + // unauthorized create. Because that webhook is failurePolicy: Ignore by design, the + // controller is the real gate: it honors this field only when an admission record + // exists for the object AND that record carries the authorized verdict. Without one — + // the webhook is off, was bypassed, or Redis is not configured — the assertion is + // ignored, the commit is authored by the configured committer, and the request reports + // AuthorAttributed=False with reason AuthorAssertionUnverified. + // + // An asserted author attaches to any open commit window for the target, not only to + // one whose audit-derived author matches: the assertion is a statement about the + // commit being made, not a claim to be the actor the audit stream recorded. It becomes + // the commit's author signature; the committer stays the operator's configured + // identity. + // +optional + Author *CommitAuthor `json:"author,omitempty"` +} + +// CommitAuthor is the Git author identity a privileged client asserts for a commit. +// Neither field is verified to correspond to a real identity: they are what the trusted +// control plane says they are. Granting `assert-author` grants the ability to write any +// author into the repository's history — treat it exactly like granting `impersonate`. +type CommitAuthor struct { + // Name is the Git author name, used verbatim in the commit's author header. + // ASCII control characters, angle brackets and newlines are rejected because they + // would let a name forge a signature header. + // +required + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:MaxLength=128 + // +kubebuilder:validation:Pattern=`^[^\x00-\x1F\x7F<>]*$` + Name string `json:"name"` + + // Email is the Git author email. When omitted, a stable synthetic address is derived + // from Name, exactly as it is for an audit-attributed author with no email claim. + // +optional + // +kubebuilder:validation:MinLength=3 + // +kubebuilder:validation:MaxLength=254 + // +kubebuilder:validation:Pattern=`^[a-zA-Z0-9._%+\-]+@[a-zA-Z0-9.\-]+\.[a-zA-Z]{2,}$` + Email string `json:"email,omitempty"` } // CommitRequestStatus defines the observed state of CommitRequest. Progress and @@ -54,10 +108,13 @@ type CommitRequestSpec struct { // - Reconciling / Stalled: the kstatus progress / blocked pair. Reconciling=True // while finalizing; Stalled=True when the finalize failed and needs attention. // - AuthorAttributed (domain): binary and settled immediately. True -// (AttributedFromAdmission) when the submitter captured at admission named the -// commit author; False (CommitterFallback) when no admission record exists — the -// validate-operator-types webhook is not configured — and the commit is authored by the -// configured committer. False is not a failure and does not affect Ready. +// (AuthorAsserted) when spec.author named the commit author and the requester was +// authorized to assert it; True (AttributedFromAdmission) when the submitter captured +// at admission named the commit author; False (CommitterFallback) when no admission +// record exists — the validate-operator-types webhook is not configured — and the +// commit is authored by the configured committer; False (AuthorAssertionUnverified) +// when spec.author was set but no authorized admission record backs it, so the +// assertion was ignored. False is not a failure and does not affect Ready. // - Pushed (domain): True once the commit is in the remote repository. type CommitRequestStatus struct { // ObservedGeneration is the most recent generation observed by the controller. diff --git a/api/v1alpha3/gittarget_types.go b/api/v1alpha3/gittarget_types.go index 5c8c8b4d..fd4edf1e 100644 --- a/api/v1alpha3/gittarget_types.go +++ b/api/v1alpha3/gittarget_types.go @@ -33,27 +33,44 @@ type GitProviderReference struct { // GitTargetSpec defines the desired state of GitTarget. // -// The destination fields — providerRef, branch, and path — are immutable. A -// GitTarget materializes the watched resources at exactly one (provider, branch, -// folder); changing where it writes would orphan the old materialization and require -// migrating manifests between repositories/branches/folders. Instead of reconciling -// that move, the destination is fixed: to relocate a GitTarget, delete it and create a -// new one. This keeps the one-owner-per-folder invariant and the initial-snapshot gate -// simple — a successful snapshot can never be silently invalidated by a destination -// change. +// A GitTarget materializes the watched resources at exactly one (provider, branch, +// folder). branch and path are mutable: changing either is a supported "retarget", in +// which the controller tears the old materialization down and rebuilds the folder from a +// fresh full snapshot at the new destination. The old folder's files are never deleted — +// deleting from Git is the one irreversible thing this operator can do, and a destination +// change is the moment an operator is least sure of what they meant. See +// docs/design/multi-tenant/gittarget-retarget.md. // -// +kubebuilder:validation:XValidation:rule="self.providerRef == oldSelf.providerRef",message="spec.providerRef is immutable; delete and recreate the GitTarget to change its destination" -// +kubebuilder:validation:XValidation:rule="self.branch == oldSelf.branch",message="spec.branch is immutable; delete and recreate the GitTarget to change its destination" -// +kubebuilder:validation:XValidation:rule="self.path == oldSelf.path",message="spec.path is immutable; delete and recreate the GitTarget to change its destination" +// providerRef stays immutable: pointing at a different repository is not a move, it is a +// different object. There is nothing to migrate and nothing to observe. +// +// The initial-snapshot gate is preserved by making the destination a thing status +// OBSERVES rather than a thing the spec cannot change: a successful snapshot is valid for +// the destination recorded in status.observedDestination. When spec and +// status.observedDestination disagree, the snapshot is by definition stale, and the +// GitTarget reports Retargeting=True until the new folder is built. +// +// +kubebuilder:validation:XValidation:rule="self.providerRef == oldSelf.providerRef",message="spec.providerRef is immutable; delete and recreate the GitTarget to change its repository (spec.branch and spec.path are mutable — the controller retargets, see docs/design/multi-tenant/gittarget-retarget.md)" type GitTargetSpec struct { // ProviderRef references the GitProvider that backs this target. - // Immutable: delete and recreate the GitTarget to change its destination. + // Immutable: delete and recreate the GitTarget to write to a different repository. // +required ProviderRef GitProviderReference `json:"providerRef"` // Branch to use for this target. // Must be one of the allowed branches in the provider. - // Immutable: delete and recreate the GitTarget to change its destination. + // + // Mutable: changing it retargets the GitTarget. The old branch's folder is left + // behind as ordinary, unmanaged Git content; the new branch's folder is built from a + // fresh full snapshot. status.observedDestination names the destination the current + // materialization belongs to. + // + // A retarget tears the old materialization down BEFORE the new destination is + // validated, because the writer resolves spec.path fresh on every write while the + // branch worker is bound to the branch its event stream was registered against — so a + // live event arriving mid-move would otherwise land at the new path on the old branch. + // A retarget onto an invalid or occupied destination therefore leaves the GitTarget + // mirroring NOTHING until a valid one is chosen. It does not fall back. // +required // +kubebuilder:validation:MinLength=1 Branch string `json:"branch"` @@ -65,11 +82,44 @@ type GitTargetSpec struct { // rejected because it is too easy to leave blank by accident to be a deliberate // root choice. Any leading slash (absolute path) and ".." are rejected, and a // trailing slash is normalized away. - // Immutable: delete and recreate the GitTarget to change its destination. + // + // Mutable: changing it retargets the GitTarget, exactly as changing branch does. The + // new path must not overlap another GitTarget on the same provider and branch. A + // retargeting GitTarget is a newcomer to the folder it is moving into, whatever its + // age, so it loses the overlap conflict to whoever is already materialized there — + // and, because the teardown precedes validation, it then mirrors nothing until a free + // path is chosen. // +required // +kubebuilder:validation:MinLength=1 Path string `json:"path"` + // SourceCluster names the cluster this GitTarget mirrors FROM. Omitted means the + // cluster the operator runs in, which is the single-cluster default and needs no + // configuration. + // + // Setting it separates the two jobs one kubeconfig used to serve: the operator reads + // its own CRs and Git credentials from the cluster it runs in (the config plane) and + // watches resources on the cluster named here. Nothing but the watched resources then + // has to live on the watched cluster — no Secret, no configbutler.ai CRDs at all — + // and one operator can mirror many clusters, because each GitTarget carries its own + // source. It is deliberately shaped like Flux's `Kustomization.spec.kubeConfig`. + // + // It sits on GitTarget rather than on WatchRule because a GitTarget already owns + // exactly one materialization. Adding the source cluster makes that one + // (cluster, provider, branch, folder) — still one owner, one folder, one desired + // state. On WatchRule, two rules could name different clusters for one folder and the + // mark-and-sweep would alternately delete each cluster's objects. + // + // WatchRule keeps its meaning: it watches the namespace it lives in, resolved on the + // source cluster. A WatchRule in config-plane namespace "team-a" watches namespace + // "team-a" on the source cluster. A ClusterWatchRule watches the whole source cluster. + // + // Mutable: changing it retargets the GitTarget, exactly as changing branch or path + // does — the folder's contents would otherwise mean something different. Rotating the + // Secret's CONTENTS is transparent and is not a retarget. + // +optional + SourceCluster *SourceClusterSpec `json:"sourceCluster,omitempty"` + // Encryption defines encryption settings for Secret resource writes. // +optional Encryption *EncryptionSpec `json:"encryption,omitempty"` @@ -82,6 +132,32 @@ type GitTargetSpec struct { Placement *GitTargetPlacementSpec `json:"placement,omitempty"` } +// SourceClusterSpec names a remote cluster to mirror from, by the Secret holding its +// kubeconfig. The Secret is read from the GitTarget's own namespace — the config plane — +// so the credential for a cluster never has to live on that cluster. +type SourceClusterSpec struct { + // KubeConfigSecretRef names the Secret holding the kubeconfig for the source cluster. + // +required + KubeConfigSecretRef SecretKeyReference `json:"kubeConfigSecretRef"` +} + +// SecretKeyReference points at one key inside a Secret in the referring object's namespace. +type SecretKeyReference struct { + // Name of the Secret, in the same namespace as the referring object. + // +required + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:MaxLength=253 + Name string `json:"name"` + + // Key within the Secret's data holding the kubeconfig. Defaults to "value.yaml", + // Flux's convention, so a Secret produced for a Flux Kustomization works unchanged. + // +optional + // +kubebuilder:default="value.yaml" + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:MaxLength=253 + Key string `json:"key,omitempty"` +} + // GitTargetPlacementSpec declares where NEW resources are written when no document // for their identity exists yet in Git — one exact-type map plus a fallback // default template (Option B2 of @@ -117,12 +193,70 @@ type GitTargetPlacementSpec struct { Default string `json:"default,omitempty"` } +// GitTargetDestination is one materialization site: the source cluster whose resources a +// GitTarget mirrors, and the branch and folder they live at. +type GitTargetDestination struct { + // Branch the materialization lives on. + // +required + Branch string `json:"branch"` + + // Path the materialization lives under, relative to the repository root. + // +required + Path string `json:"path"` + + // SourceCluster names the kubeconfig Secret whose cluster this materialization was + // mirrored from. Empty means the cluster the operator runs in. + // +optional + SourceCluster string `json:"sourceCluster,omitempty"` +} + +// SourceClusterID identifies the cluster a GitTarget mirrors from, as the watch data plane +// keys its per-cluster clients, catalogs, and type registries. The empty string is the +// cluster the operator runs in. +// +// The Secret's key is part of the identity: two GitTargets naming the same Secret under +// different keys are pointed at different kubeconfigs, and so at different clusters. +func (g *GitTarget) SourceClusterID() string { + if g.Spec.SourceCluster == nil { + return "" + } + ref := g.Spec.SourceCluster.KubeConfigSecretRef + key := ref.Key + if key == "" { + key = DefaultKubeConfigSecretKey + } + return g.Namespace + "/" + ref.Name + "/" + key +} + +// DefaultKubeConfigSecretKey is the Secret data key a source-cluster kubeconfig is read +// from when none is given. It matches Flux's convention. +const DefaultKubeConfigSecretKey = "value.yaml" + // GitTargetStatus defines the observed state of GitTarget. type GitTargetStatus struct { // ObservedGeneration is the latest generation observed by the controller. // +optional ObservedGeneration int64 `json:"observedGeneration,omitempty"` + // ObservedDestination is the destination the current materialization belongs to. It + // is written only once a snapshot has actually landed there and the path was + // accepted, so it is the answer to "which folder is this GitTarget's content in?" — + // which is not always what spec says. + // + // Absent means nothing has been materialized yet. When it disagrees with spec, the + // GitTarget is retargeting and Retargeting=True; the folder it names is the one being + // abandoned, so an operator can `git rm` it deliberately once the move settles. + // +optional + ObservedDestination *GitTargetDestination `json:"observedDestination,omitempty"` + + // RetargetingTo is the destination a retarget in progress is building, set while + // Retargeting=True and cleared once the move settles. It exists so a move is + // self-describing, and so a SECOND destination change arriving mid-move can name the + // intermediate folder the first move had already begun writing — otherwise that folder + // would be orphaned without ever being named. See the RetargetSuperseded event. + // +optional + RetargetingTo *GitTargetDestination `json:"retargetingTo,omitempty"` + // Conditions represent the latest available observations of an object's state // +optional // +listType=map @@ -179,6 +313,8 @@ type GitTargetStreamsStatus struct { // +kubebuilder:printcolumn:name="Streams",type=string,JSONPath=`.status.streams.summary` // +kubebuilder:printcolumn:name="GitPathAccepted",type=string,JSONPath=`.status.conditions[?(@.type=="GitPathAccepted")].status`,priority=1 // +kubebuilder:printcolumn:name="StreamsRunning",type=string,JSONPath=`.status.conditions[?(@.type=="StreamsRunning")].status`,priority=1 +// +kubebuilder:printcolumn:name="Retargeting",type=string,JSONPath=`.status.conditions[?(@.type=="Retargeting")].status`,priority=1 +// +kubebuilder:printcolumn:name="ObservedPath",type=string,JSONPath=`.status.observedDestination.path`,priority=1 // +kubebuilder:printcolumn:name="Status",type=string,JSONPath=`.status.conditions[?(@.type=="Ready")].message`,priority=1 // +kubebuilder:printcolumn:name="Encryption",type=string,JSONPath=`.spec.encryption.provider`,priority=1 // +kubebuilder:printcolumn:name="Age",type=date,JSONPath=`.metadata.creationTimestamp` diff --git a/api/v1alpha3/watchrule_types.go b/api/v1alpha3/watchrule_types.go index 8619cfe2..fdc3d56a 100644 --- a/api/v1alpha3/watchrule_types.go +++ b/api/v1alpha3/watchrule_types.go @@ -52,6 +52,11 @@ type WatchRuleSpec struct { // Rules define which resources to watch within this namespace. // Multiple rules create a logical OR - a resource matching ANY rule is watched. // Each rule can specify operations, API groups, versions, and resource types. + // + // An exclusion (excludeUsers / excludeFieldManagers) is a veto WITHIN its own rule, + // not a global filter: a change is mirrored when at least one rule both selects it + // and does not exclude its writer. Adding an unrestricted rule for a type therefore + // re-admits everything another rule excluded for that type. // +required // +kubebuilder:validation:MinItems=1 Rules []ResourceRule `json:"rules"` @@ -115,6 +120,54 @@ type ResourceRule struct { // +kubebuilder:validation:items:MinLength=1 // +kubebuilder:validation:items:Pattern=`^[^/]*$` Resources []string `json:"resources"` + + // ExcludeFieldManagers drops a live change whose last writer is one of these + // field managers, so a GitOps forward leg (Flux, Argo CD) applying this branch + // back into the cluster does not have its own applies mirrored into Git. + // + // Reach for this rather than ExcludeUsers: it reads metadata.managedFields off + // the object, so it needs no audit fact, cannot race the attribution grace + // window, and works in configured-author mode. + // + // The "last writer" is the manager of the newest managedFields entry. When + // several entries share that timestamp the change is excluded only if every + // tied manager is listed — when in doubt, the change is mirrored. An object + // with no managedFields is never excluded. + // + // It is NOT evaluated for DELETE: managedFields names who last wrote an object, + // not who deleted it, so excluding a delete this way would silently ignore a + // human deleting a Flux-managed resource. Use ExcludeUsers for deletes. + // + // Examples: + // - ["kustomize-controller"] ignores writes applied by Flux's kustomize-controller + // - ["argocd-controller"] ignores writes applied by Argo CD + // +optional + // +kubebuilder:validation:MaxItems=32 + // +kubebuilder:validation:items:MinLength=1 + // +kubebuilder:validation:items:MaxLength=128 + ExcludeFieldManagers []string `json:"excludeFieldManagers,omitempty"` + + // ExcludeUsers drops a live change attributed to one of these identities: the + // impersonated user when impersonation is in play, otherwise the authenticated + // user, exactly as the audit event records it. + // + // It therefore requires author attribution (--author-attribution) and a working + // audit webhook. When the author cannot be resolved — attribution disabled, or the + // grace window elapsed with no matching fact — the change is mirrored rather than + // dropped: losing a human's edit because we failed to identify its author is worse + // than mirroring one machine write. Prefer ExcludeFieldManagers, which has no such + // failure mode. + // + // Unlike ExcludeFieldManagers this does apply to DELETE, because the audit fact + // names the actor who issued the delete. + // + // Example: + // - ["system:serviceaccount:flux-system:kustomize-controller"] + // +optional + // +kubebuilder:validation:MaxItems=32 + // +kubebuilder:validation:items:MinLength=1 + // +kubebuilder:validation:items:MaxLength=316 + ExcludeUsers []string `json:"excludeUsers,omitempty"` } // WatchRuleStatus defines the observed state of WatchRule. diff --git a/api/v1alpha3/zz_generated.deepcopy.go b/api/v1alpha3/zz_generated.deepcopy.go index 86340c09..33e48130 100644 --- a/api/v1alpha3/zz_generated.deepcopy.go +++ b/api/v1alpha3/zz_generated.deepcopy.go @@ -70,6 +70,16 @@ func (in *ClusterResourceRule) DeepCopyInto(out *ClusterResourceRule) { *out = make([]string, len(*in)) copy(*out, *in) } + if in.ExcludeFieldManagers != nil { + in, out := &in.ExcludeFieldManagers, &out.ExcludeFieldManagers + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.ExcludeUsers != nil { + in, out := &in.ExcludeUsers, &out.ExcludeUsers + *out = make([]string, len(*in)) + copy(*out, *in) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterResourceRule. @@ -191,6 +201,21 @@ func (in *ClusterWatchRuleStatus) DeepCopy() *ClusterWatchRuleStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CommitAuthor) DeepCopyInto(out *CommitAuthor) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CommitAuthor. +func (in *CommitAuthor) DeepCopy() *CommitAuthor { + if in == nil { + return nil + } + out := new(CommitAuthor) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CommitMessageSpec) DeepCopyInto(out *CommitMessageSpec) { *out = *in @@ -211,7 +236,7 @@ func (in *CommitRequest) DeepCopyInto(out *CommitRequest) { *out = *in out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) - out.Spec = in.Spec + in.Spec.DeepCopyInto(&out.Spec) in.Status.DeepCopyInto(&out.Status) } @@ -269,6 +294,11 @@ func (in *CommitRequestList) DeepCopyObject() runtime.Object { func (in *CommitRequestSpec) DeepCopyInto(out *CommitRequestSpec) { *out = *in out.TargetRef = in.TargetRef + if in.Author != nil { + in, out := &in.Author, &out.Author + *out = new(CommitAuthor) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CommitRequestSpec. @@ -548,6 +578,21 @@ func (in *GitTarget) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GitTargetDestination) DeepCopyInto(out *GitTargetDestination) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GitTargetDestination. +func (in *GitTargetDestination) DeepCopy() *GitTargetDestination { + if in == nil { + return nil + } + out := new(GitTargetDestination) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *GitTargetList) DeepCopyInto(out *GitTargetList) { *out = *in @@ -606,6 +651,11 @@ func (in *GitTargetPlacementSpec) DeepCopy() *GitTargetPlacementSpec { func (in *GitTargetSpec) DeepCopyInto(out *GitTargetSpec) { *out = *in out.ProviderRef = in.ProviderRef + if in.SourceCluster != nil { + in, out := &in.SourceCluster, &out.SourceCluster + *out = new(SourceClusterSpec) + **out = **in + } if in.Encryption != nil { in, out := &in.Encryption, &out.Encryption *out = new(EncryptionSpec) @@ -631,6 +681,16 @@ func (in *GitTargetSpec) DeepCopy() *GitTargetSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *GitTargetStatus) DeepCopyInto(out *GitTargetStatus) { *out = *in + if in.ObservedDestination != nil { + in, out := &in.ObservedDestination, &out.ObservedDestination + *out = new(GitTargetDestination) + **out = **in + } + if in.RetargetingTo != nil { + in, out := &in.RetargetingTo, &out.RetargetingTo + *out = new(GitTargetDestination) + **out = **in + } if in.Conditions != nil { in, out := &in.Conditions, &out.Conditions *out = make([]v1.Condition, len(*in)) @@ -782,6 +842,16 @@ func (in *ResourceRule) DeepCopyInto(out *ResourceRule) { *out = make([]string, len(*in)) copy(*out, *in) } + if in.ExcludeFieldManagers != nil { + in, out := &in.ExcludeFieldManagers, &out.ExcludeFieldManagers + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.ExcludeUsers != nil { + in, out := &in.ExcludeUsers, &out.ExcludeUsers + *out = make([]string, len(*in)) + copy(*out, *in) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ResourceRule. @@ -794,6 +864,37 @@ func (in *ResourceRule) DeepCopy() *ResourceRule { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SecretKeyReference) DeepCopyInto(out *SecretKeyReference) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SecretKeyReference. +func (in *SecretKeyReference) DeepCopy() *SecretKeyReference { + if in == nil { + return nil + } + out := new(SecretKeyReference) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SourceClusterSpec) DeepCopyInto(out *SourceClusterSpec) { + *out = *in + out.KubeConfigSecretRef = in.KubeConfigSecretRef +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SourceClusterSpec. +func (in *SourceClusterSpec) DeepCopy() *SourceClusterSpec { + if in == nil { + return nil + } + out := new(SourceClusterSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *WatchRule) DeepCopyInto(out *WatchRule) { *out = *in diff --git a/charts/gitops-reverser/templates/deployment.yaml b/charts/gitops-reverser/templates/deployment.yaml index b54acffc..f201de08 100644 --- a/charts/gitops-reverser/templates/deployment.yaml +++ b/charts/gitops-reverser/templates/deployment.yaml @@ -71,6 +71,9 @@ spec: - --redis-username={{ . | quote }} {{- end }} - --redis-db={{ .Values.queue.redis.db }} + {{- with .Values.queue.redis.keyPrefix }} + - --redis-key-prefix={{ . | quote }} + {{- end }} {{- if not .Values.queue.redis.tls.enabled }} - --redis-insecure {{- end }} diff --git a/charts/gitops-reverser/templates/validate-operator-types-webhook.yaml b/charts/gitops-reverser/templates/validate-operator-types-webhook.yaml index 3cfdc20a..7400ef55 100644 --- a/charts/gitops-reverser/templates/validate-operator-types-webhook.yaml +++ b/charts/gitops-reverser/templates/validate-operator-types-webhook.yaml @@ -27,7 +27,10 @@ webhooks: path: /validate-operator-types port: {{ .Values.servers.admission.port }} # Ignore, not Fail: a missed author capture must degrade to the committer, never - # reject a user's CommitRequest. + # reject a user's CommitRequest. The handler does deny an unauthorized spec.author, + # and Ignore means that denial is bypassable — safely: the controller honors + # spec.author only against an admission record carrying the authorization verdict, and + # a bypassed webhook writes no record. See docs/design/multi-tenant/asserted-commit-author.md. failurePolicy: Ignore matchPolicy: Equivalent rules: diff --git a/charts/gitops-reverser/values.yaml b/charts/gitops-reverser/values.yaml index 7fecb653..36be158c 100644 --- a/charts/gitops-reverser/values.yaml +++ b/charts/gitops-reverser/values.yaml @@ -184,6 +184,12 @@ queue: # Optional Redis username for ACL-based auth (Redis 6+). Not sensitive; stays as a plain value. username: "" db: 0 + # Root namespace for every key this release writes (cursors, attribution facts, command author + # records). Give each reverser its own prefix to share one Redis/Valkey between more than the 16 + # logical databases `db` can separate. A Valkey ACL can enforce it (~:*) rather than trust + # it. Changing it orphans the keys written under the previous prefix: cursors then cold-replay + # once, which is safe. Allowed characters: [A-Za-z0-9], '-', '_', '.' and ':'. + keyPrefix: "gitops-reverser" tls: enabled: false diff --git a/cmd/main.go b/cmd/main.go index 1d6c01e4..40b4c0ab 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -28,6 +28,7 @@ import ( "k8s.io/apimachinery/pkg/runtime" utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" + authzv1client "k8s.io/client-go/kubernetes/typed/authorization/v1" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/certwatcher" "sigs.k8s.io/controller-runtime/pkg/client" @@ -64,6 +65,11 @@ const ( defaultAuditIdleTimeout = 60 * time.Second defaultAuditShutdownTimeout = 10 * time.Second defaultBranchBufferMaxSizeStr = "8Mi" + // defaultSourceClusterQPS/Burst bound the operator's request rate against a REMOTE + // source cluster. client-go's own defaults (5/10) are far too low for a discovery + // refresh plus a watch set; these match controller-runtime's manager defaults. + defaultSourceClusterQPS = 20.0 + defaultSourceClusterBurst = 30 ) func init() { @@ -130,13 +136,21 @@ func main() { workerManager.SetSSHHostKeyConfig(cfg.sshHostKeys) fatalIfErr(mgr.Add(workerManager), "unable to add worker manager to manager") - // Watch ingestion manager (placeholder, will get EventRouter set later) + // Watch ingestion manager (placeholder, will get EventRouter set later). + // + // SourceClusters is what separates the config plane from the watched cluster: the + // manager reads its own CRs and Git credentials through mgr.GetClient() (the cluster + // this pod runs in) and watches whichever cluster each GitTarget names. A GitTarget + // with no spec.sourceCluster watches the local cluster, which is every install that + // has not asked for anything else. watchMgr := &watch.Manager{ Client: mgr.GetClient(), Log: ctrl.Log.WithName("watch"), RuleStore: ruleStore, EventRouter: nil, // Will be set below SensitiveResources: cfg.sensitiveResources, + SourceClusters: watch.NewSecretSourceClusterResolver( + mgr.GetClient(), float32(cfg.sourceClusterQPS), cfg.sourceClusterBurst), } // Initialize EventRouter with all dependencies. The streaming-snapshot resync @@ -155,7 +169,10 @@ func main() { // Inject the live followability registry into the writer, so a GVR-only DELETE // event resolves to a manifest moved off its canonical path (M6 in the writer). // The registry is a stable pointer the watch manager refreshes in place. - workerManager.SetMapper(watchMgr.TypeRegistry()) + // The writer's GVK resolver is a union over every watched cluster's type registry: a + // branch worker is shared by GitTargets that may mirror different clusters, so it cannot + // hold one target's registry. In a single-cluster install it is exactly the local one. + workerManager.SetMapper(watchMgr.TypeLookup()) // Give the workers a way to surface a refused live write plan. Live events are committed // off a timer with no result channel, so without this a refusal (acceptance gate or a @@ -194,9 +211,12 @@ func main() { AuthValue: cfg.redisPassword, DB: cfg.redisDB, TLSEnabled: !cfg.redisInsecure, + KeyPrefix: cfg.redisKeyPrefix, }) fatalIfErr(err, "unable to build Redis cursor store") watchMgr.WatchCursorStore = redisStore + setupLog.Info("Redis keyspace", "addr", cfg.redisAddr, "db", cfg.redisDB, + "keyPrefix", redisStore.KeyPrefix()) redisGate = newRedisReadinessGate(redisStore) fatalIfErr(mgr.Add(redisGate), "unable to add redis readiness gate") @@ -258,6 +278,7 @@ func main() { Scheme: mgr.GetScheme(), WorkerManager: workerManager, EventRouter: eventRouter, + Recorder: mgr.GetEventRecorderFor("gittarget"), }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "GitTarget") os.Exit(1) @@ -346,11 +367,14 @@ type appConfig struct { redisUsername string redisPassword string redisDB int + redisKeyPrefix string redisInsecure bool authorAttribution bool attributionFactTTL time.Duration attributionGrace time.Duration branchBufferMaxBytes int64 + sourceClusterQPS float64 + sourceClusterBurst int sensitiveResources types.SensitiveResourcePolicy sshHostKeys git.SSHHostKeyConfig zapOpts zap.Options @@ -433,6 +457,12 @@ func parseFlagsWithArgs(fs *flag.FlagSet, args []string) (appConfig, error) { "Redis password. Prefer setting via REDIS_PASSWORD env var from a Secret.", ) fs.IntVar(&cfg.redisDB, "redis-db", 0, "Redis database index (default 0).") + fs.StringVar(&cfg.redisKeyPrefix, "redis-key-prefix", queue.DefaultKeyPrefix, + "Root namespace for every key this operator writes to Redis/Valkey (cursors, attribution "+ + "facts, command author records). Give each reverser its own prefix to share one "+ + "Redis/Valkey between more than the 16 logical databases --redis-db can separate. "+ + "Allowed characters are [A-Za-z0-9], '-', '_', '.' and ':'; a Valkey ACL can enforce "+ + "the prefix (~:*) rather than trust it.") fs.BoolVar(&cfg.redisInsecure, "redis-insecure", false, "Connect to Redis over plain TCP instead of TLS (default false; TLS). Redis carries each "+ "GitTarget's watch cursors and, when attribution is on, the audit facts — prefer TLS. Set "+ @@ -459,6 +489,11 @@ func parseFlagsWithArgs(fs *flag.FlagSet, args []string) (appConfig, error) { "", "Comma-separated additional sensitive resources in resource or group/resource form.", ) + fs.Float64Var(&cfg.sourceClusterQPS, "source-cluster-qps", defaultSourceClusterQPS, + "Client-side request rate limit per remote source cluster (GitTarget.spec.sourceCluster). "+ + "A remote cluster is reached over a network the local one is not; 0 disables throttling.") + fs.IntVar(&cfg.sourceClusterBurst, "source-cluster-burst", defaultSourceClusterBurst, + "Client-side burst allowance per remote source cluster. Ignored when --source-cluster-qps is 0.") fs.StringVar(&cfg.sshHostKeys.DefaultKnownHostsConfigMap, "default-known-hosts-configmap", "", "Optional install-level ConfigMap (in the controller's namespace) supplying SSH known_hosts "+ "for Git hosts when neither the credentials Secret nor the GitProvider's knownHostsRef does.") @@ -476,6 +511,13 @@ func parseFlagsWithArgs(fs *flag.FlagSet, args []string) (appConfig, error) { return appConfig{}, err } cfg.redisAddr = strings.TrimSpace(cfg.redisAddr) + // Validated even when redis-addr is empty: a typo'd prefix is a configuration error + // whether or not this run happens to open the connection it names. + normalizedKeyPrefix, err := queue.ValidateKeyPrefix(cfg.redisKeyPrefix) + if err != nil { + return appConfig{}, err + } + cfg.redisKeyPrefix = normalizedKeyPrefix if err := validateAuditConfig(cfg); err != nil { return appConfig{}, err } @@ -896,7 +938,8 @@ func newManager( // setupAdmissionWebhooks registers both handlers on the one admission server: the // always-allow observer (a future-policy extension point) and the validate-operator-types -// handler that captures the submitter of our own command kinds into commandAuthorStore. +// handler that captures the submitter of our own command kinds into commandAuthorStore +// and authorizes a CommitRequest's asserted commit author. func setupAdmissionWebhooks(mgr ctrl.Manager, commandAuthorStore *queue.CommandAuthorStore) { mgr.GetWebhookServer().Register( webhookhandler.ValidateAllPath, @@ -908,6 +951,16 @@ func setupAdmissionWebhooks(mgr ctrl.Manager, commandAuthorStore *queue.CommandA if commandAuthorStore != nil { operatorTypesHandler.Store = commandAuthorStore } + // The assert-author guard delegates to whatever authorizers the cluster runs (RBAC, + // webhook, node) via SubjectAccessReview. A client we cannot build leaves Authorizer + // nil, which denies every assertion — an unverifiable privilege is not a granted one. + if authzClient, err := authzv1client.NewForConfig(mgr.GetConfig()); err != nil { + setupLog.Error(err, "unable to build authorization client: CommitRequest spec.author will be denied") + } else { + operatorTypesHandler.Authorizer = webhookhandler.NewSubjectAccessReviewAuthorizer( + authzClient.SubjectAccessReviews(), + ) + } mgr.GetWebhookServer().Register( webhookhandler.ValidateOperatorTypesPath, &ctrladmission.Webhook{Handler: operatorTypesHandler}, diff --git a/cmd/main_audit_server_test.go b/cmd/main_audit_server_test.go index c6e48019..b05c6922 100644 --- a/cmd/main_audit_server_test.go +++ b/cmd/main_audit_server_test.go @@ -21,6 +21,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + + "github.com/ConfigButler/gitops-reverser/internal/queue" ) func TestParseFlagsWithArgs_Defaults(t *testing.T) { @@ -97,6 +99,7 @@ func TestParseFlagsWithArgs_CustomAuditValues(t *testing.T) { "--redis-username=user", "--redis-password=pass", "--redis-db=2", + "--redis-key-prefix=cell-a:tenant-7:", "--redis-insecure", "--author-attribution-ttl=20m", "--author-attribution-grace=750ms", @@ -119,11 +122,37 @@ func TestParseFlagsWithArgs_CustomAuditValues(t *testing.T) { assert.Equal(t, "user", cfg.redisUsername) assert.Equal(t, "pass", cfg.redisPassword) assert.Equal(t, 2, cfg.redisDB) + // The trailing colon is normalized away, so "tenant-7:" and "tenant-7" name one keyspace. + assert.Equal(t, "cell-a:tenant-7", cfg.redisKeyPrefix) assert.True(t, cfg.redisInsecure) assert.Equal(t, 20*time.Minute, cfg.attributionFactTTL) assert.Equal(t, 750*time.Millisecond, cfg.attributionGrace) } +func TestParseFlagsWithArgs_RedisKeyPrefixDefault(t *testing.T) { + fs := flag.NewFlagSet("test-redis-key-prefix-default", flag.ContinueOnError) + cfg, err := parseFlagsWithArgs(fs, []string{"--author-attribution=false"}) + require.NoError(t, err) + // An upgrade that sets nothing must keep writing the keys the previous release wrote. + assert.Equal(t, queue.DefaultKeyPrefix, cfg.redisKeyPrefix) +} + +func TestParseFlagsWithArgs_RedisKeyPrefixRejectsGlob(t *testing.T) { + fs := flag.NewFlagSet("test-redis-key-prefix-glob", flag.ContinueOnError) + // A glob metacharacter would corrupt the attribution fact-index SCAN pattern. + _, err := parseFlagsWithArgs(fs, []string{"--redis-key-prefix=tenant-*"}) + require.Error(t, err) + assert.Contains(t, err.Error(), "redis-key-prefix") +} + +func TestParseFlagsWithArgs_RedisKeyPrefixRejectsEmpty(t *testing.T) { + fs := flag.NewFlagSet("test-redis-key-prefix-empty", flag.ContinueOnError) + // Rejected even without Redis: an empty prefix silently un-namespaces the keyspace. + _, err := parseFlagsWithArgs(fs, []string{"--author-attribution=false", "--redis-addr=", "--redis-key-prefix="}) + require.Error(t, err) + assert.Contains(t, err.Error(), "non-empty") +} + func TestParseFlagsWithArgs_RedisAddrRequiredWhenAttributionEnabled(t *testing.T) { fs := flag.NewFlagSet("test-redis-required", flag.ContinueOnError) // Default --author-attribution=true, so an empty redis-addr must be rejected. diff --git a/cmd/manifest-analyzer/main.go b/cmd/manifest-analyzer/main.go index 71d64399..f7f912fb 100644 --- a/cmd/manifest-analyzer/main.go +++ b/cmd/manifest-analyzer/main.go @@ -49,6 +49,7 @@ import ( "k8s.io/client-go/tools/clientcmd" "github.com/ConfigButler/gitops-reverser/internal/manifestanalyzer" + publicanalyzer "github.com/ConfigButler/gitops-reverser/pkg/manifestanalyzer" ) // Process exit codes. @@ -237,7 +238,24 @@ func failedGroupVersions(failed map[schema.GroupVersion]error) map[string]string // refuse policy over the acceptance decision. It is structure-only: no cluster // state, so the plan is empty, but the acceptance gate is the full one — it applies // the non-API KRM allowlist and the impure-managed-file / mixed-file refusals. +// +// --format json goes through pkg/manifestanalyzer, so the CLI's machine-readable output +// and the published Go contract are the same document and cannot drift. Text output stays +// on the internal renderer, which can show the plan the public report deliberately omits. func runScan(dir, format, policy string, stdout, stderr io.Writer) int { + if format == "json" { + report, err := publicanalyzer.ScanFolder(context.Background(), dir) + if err != nil { + fmt.Fprintf(stderr, "error: %v\n", err) + return exitUsage + } + if err := report.WriteJSON(stdout); err != nil { + fmt.Fprintf(stderr, "error: %v\n", err) + return exitUsage + } + return scanExitCode(policy, report.Accepted) + } + scanPolicy := manifestanalyzer.ScanPolicy{ Acceptance: manifestanalyzer.AcceptancePolicy{Allowlist: manifestanalyzer.DefaultAllowlist()}, } @@ -246,17 +264,12 @@ func runScan(dir, format, policy string, stdout, stderr io.Writer) int { fmt.Fprintf(stderr, "error: %v\n", err) return exitUsage } + manifestanalyzer.RenderScanText(stdout, result) + return scanExitCode(policy, result.Acceptance.Accepted) +} - if format == "json" { - if err := manifestanalyzer.RenderScanJSON(stdout, result); err != nil { - fmt.Fprintf(stderr, "error: %v\n", err) - return exitUsage - } - } else { - manifestanalyzer.RenderScanText(stdout, result) - } - - if policy == "refuse" && !result.Acceptance.Accepted { +func scanExitCode(policy string, accepted bool) int { + if policy == "refuse" && !accepted { return exitRefused } return exitOK @@ -268,19 +281,24 @@ func runScan(dir, format, policy string, stdout, stderr io.Writer) int { // (exitOK, or exitUsage on an I/O error); the repo-level --policy refuse gate is // deferred per the design doc. func runRepoWalk(root, format string, stdout, stderr io.Writer) int { - rep, err := manifestanalyzer.WalkRepo(context.Background(), root) - if err != nil { - fmt.Fprintf(stderr, "error: %v\n", err) - return exitUsage - } - if format == "json" { - if err := manifestanalyzer.RenderRepoJSON(stdout, rep); err != nil { + report, err := publicanalyzer.ScanRepo(context.Background(), root) + if err != nil { fmt.Fprintf(stderr, "error: %v\n", err) return exitUsage } - } else { - manifestanalyzer.RenderRepoText(stdout, rep) + if err := report.WriteJSON(stdout); err != nil { + fmt.Fprintf(stderr, "error: %v\n", err) + return exitUsage + } + return exitOK + } + + rep, err := manifestanalyzer.WalkRepo(context.Background(), root) + if err != nil { + fmt.Fprintf(stderr, "error: %v\n", err) + return exitUsage } + manifestanalyzer.RenderRepoText(stdout, rep) return exitOK } diff --git a/config/crd/bases/configbutler.ai_clusterwatchrules.yaml b/config/crd/bases/configbutler.ai_clusterwatchrules.yaml index ba75983d..d45e3204 100644 --- a/config/crd/bases/configbutler.ai_clusterwatchrules.yaml +++ b/config/crd/bases/configbutler.ai_clusterwatchrules.yaml @@ -112,6 +112,30 @@ spec: items: type: string type: array + excludeFieldManagers: + description: |- + ExcludeFieldManagers drops a live change whose last writer is one of these field + managers, so a GitOps forward leg applying this branch back into the cluster does + not have its own applies mirrored into Git. It is read from metadata.managedFields + and needs no audit fact. Not evaluated for DELETE. See + ResourceRule.ExcludeFieldManagers for the full semantics. + items: + maxLength: 128 + minLength: 1 + type: string + maxItems: 32 + type: array + excludeUsers: + description: |- + ExcludeUsers drops a live change attributed to one of these identities. It requires + author attribution and fails open when the author cannot be resolved. See + ResourceRule.ExcludeUsers for the full semantics. + items: + maxLength: 316 + minLength: 1 + type: string + maxItems: 32 + type: array operations: description: |- Operations to watch. If empty, watches all operations (CREATE, UPDATE, DELETE). diff --git a/config/crd/bases/configbutler.ai_commitrequests.yaml b/config/crd/bases/configbutler.ai_commitrequests.yaml index 60f2e22a..d9d0a14c 100644 --- a/config/crd/bases/configbutler.ai_commitrequests.yaml +++ b/config/crd/bases/configbutler.ai_commitrequests.yaml @@ -70,6 +70,51 @@ spec: spec: description: spec defines the desired state of CommitRequest properties: + author: + description: "Author names the human this commit is for, instead of + deriving them from an\napiserver audit fact. It exists for an authenticated + control plane that already\nknows who the user is — one that verified + their token before impersonating them —\nand for any cluster whose + apiserver audit flags the operator cannot set.\n\nAsserting an author + is a privilege, not a field anyone may set. It is honored only\nwhen + the requester holds the `assert-author` verb on the named GitTarget, + in the\nstyle of `bind`, `escalate` and `impersonate`:\n\n\trules:\n\t + \ - apiGroups: [\"configbutler.ai\"]\n\t resources: [\"gittargets\"]\n\t + \ resourceNames: [\"tenants\"]\n\t verbs: [\"assert-author\"]\n\nThe + check runs in the validate-operator-types admission webhook, which + denies an\nunauthorized create. Because that webhook is failurePolicy: + Ignore by design, the\ncontroller is the real gate: it honors this + field only when an admission record\nexists for the object AND that + record carries the authorized verdict. Without one —\nthe webhook + is off, was bypassed, or Redis is not configured — the assertion + is\nignored, the commit is authored by the configured committer, + and the request reports\nAuthorAttributed=False with reason AuthorAssertionUnverified.\n\nAn + asserted author attaches to any open commit window for the target, + not only to\none whose audit-derived author matches: the assertion + is a statement about the\ncommit being made, not a claim to be the + actor the audit stream recorded. It becomes\nthe commit's author + signature; the committer stays the operator's configured\nidentity." + properties: + email: + description: |- + Email is the Git author email. When omitted, a stable synthetic address is derived + from Name, exactly as it is for an audit-attributed author with no email claim. + maxLength: 254 + minLength: 3 + pattern: ^[a-zA-Z0-9._%+\-]+@[a-zA-Z0-9.\-]+\.[a-zA-Z]{2,}$ + type: string + name: + description: |- + Name is the Git author name, used verbatim in the commit's author header. + ASCII control characters, angle brackets and newlines are rejected because they + would let a name forge a signature header. + maxLength: 128 + minLength: 1 + pattern: ^[^\x00-\x1F\x7F<>]*$ + type: string + required: + - name + type: object closeDelaySeconds: description: |- CloseDelaySeconds optionally delays closing the open commit window for this diff --git a/config/crd/bases/configbutler.ai_gittargets.yaml b/config/crd/bases/configbutler.ai_gittargets.yaml index dbdaac72..5d3c5264 100644 --- a/config/crd/bases/configbutler.ai_gittargets.yaml +++ b/config/crd/bases/configbutler.ai_gittargets.yaml @@ -41,6 +41,14 @@ spec: name: StreamsRunning priority: 1 type: string + - jsonPath: .status.conditions[?(@.type=="Retargeting")].status + name: Retargeting + priority: 1 + type: string + - jsonPath: .status.observedDestination.path + name: ObservedPath + priority: 1 + type: string - jsonPath: .status.conditions[?(@.type=="Ready")].message name: Status priority: 1 @@ -81,7 +89,18 @@ spec: description: |- Branch to use for this target. Must be one of the allowed branches in the provider. - Immutable: delete and recreate the GitTarget to change its destination. + + Mutable: changing it retargets the GitTarget. The old branch's folder is left + behind as ordinary, unmanaged Git content; the new branch's folder is built from a + fresh full snapshot. status.observedDestination names the destination the current + materialization belongs to. + + A retarget tears the old materialization down BEFORE the new destination is + validated, because the writer resolves spec.path fresh on every write while the + branch worker is bound to the branch its event stream was registered against — so a + live event arriving mid-move would otherwise land at the new path on the old branch. + A retarget onto an invalid or occupied destination therefore leaves the GitTarget + mirroring NOTHING until a valid one is chosen. It does not fall back. minLength: 1 type: string encryption: @@ -159,7 +178,13 @@ spec: rejected because it is too easy to leave blank by accident to be a deliberate root choice. Any leading slash (absolute path) and ".." are rejected, and a trailing slash is normalized away. - Immutable: delete and recreate the GitTarget to change its destination. + + Mutable: changing it retargets the GitTarget, exactly as changing branch does. The + new path must not overlap another GitTarget on the same provider and branch. A + retargeting GitTarget is a newcomer to the folder it is moving into, whatever its + age, so it loses the overlap conflict to whoever is already materialized there — + and, because the teardown precedes validation, it then mirrors nothing until a free + path is chosen. minLength: 1 type: string placement: @@ -192,7 +217,7 @@ spec: providerRef: description: |- ProviderRef references the GitProvider that backs this target. - Immutable: delete and recreate the GitTarget to change its destination. + Immutable: delete and recreate the GitTarget to write to a different repository. properties: group: default: configbutler.ai @@ -216,6 +241,57 @@ spec: required: - name type: object + sourceCluster: + description: |- + SourceCluster names the cluster this GitTarget mirrors FROM. Omitted means the + cluster the operator runs in, which is the single-cluster default and needs no + configuration. + + Setting it separates the two jobs one kubeconfig used to serve: the operator reads + its own CRs and Git credentials from the cluster it runs in (the config plane) and + watches resources on the cluster named here. Nothing but the watched resources then + has to live on the watched cluster — no Secret, no configbutler.ai CRDs at all — + and one operator can mirror many clusters, because each GitTarget carries its own + source. It is deliberately shaped like Flux's `Kustomization.spec.kubeConfig`. + + It sits on GitTarget rather than on WatchRule because a GitTarget already owns + exactly one materialization. Adding the source cluster makes that one + (cluster, provider, branch, folder) — still one owner, one folder, one desired + state. On WatchRule, two rules could name different clusters for one folder and the + mark-and-sweep would alternately delete each cluster's objects. + + WatchRule keeps its meaning: it watches the namespace it lives in, resolved on the + source cluster. A WatchRule in config-plane namespace "team-a" watches namespace + "team-a" on the source cluster. A ClusterWatchRule watches the whole source cluster. + + Mutable: changing it retargets the GitTarget, exactly as changing branch or path + does — the folder's contents would otherwise mean something different. Rotating the + Secret's CONTENTS is transparent and is not a retarget. + properties: + kubeConfigSecretRef: + description: KubeConfigSecretRef names the Secret holding the + kubeconfig for the source cluster. + properties: + key: + default: value.yaml + description: |- + Key within the Secret's data holding the kubeconfig. Defaults to "value.yaml", + Flux's convention, so a Secret produced for a Flux Kustomization works unchanged. + maxLength: 253 + minLength: 1 + type: string + name: + description: Name of the Secret, in the same namespace as + the referring object. + maxLength: 253 + minLength: 1 + type: string + required: + - name + type: object + required: + - kubeConfigSecretRef + type: object required: - branch - path @@ -223,14 +299,9 @@ spec: type: object x-kubernetes-validations: - message: spec.providerRef is immutable; delete and recreate the GitTarget - to change its destination + to change its repository (spec.branch and spec.path are mutable — + the controller retargets, see docs/design/multi-tenant/gittarget-retarget.md) rule: self.providerRef == oldSelf.providerRef - - message: spec.branch is immutable; delete and recreate the GitTarget - to change its destination - rule: self.branch == oldSelf.branch - - message: spec.path is immutable; delete and recreate the GitTarget to - change its destination - rule: self.path == oldSelf.path status: description: status defines the observed state of GitTarget properties: @@ -305,11 +376,62 @@ spec: reconcile attempt. format: date-time type: string + observedDestination: + description: |- + ObservedDestination is the destination the current materialization belongs to. It + is written only once a snapshot has actually landed there and the path was + accepted, so it is the answer to "which folder is this GitTarget's content in?" — + which is not always what spec says. + + Absent means nothing has been materialized yet. When it disagrees with spec, the + GitTarget is retargeting and Retargeting=True; the folder it names is the one being + abandoned, so an operator can `git rm` it deliberately once the move settles. + properties: + branch: + description: Branch the materialization lives on. + type: string + path: + description: Path the materialization lives under, relative to + the repository root. + type: string + sourceCluster: + description: |- + SourceCluster names the kubeconfig Secret whose cluster this materialization was + mirrored from. Empty means the cluster the operator runs in. + type: string + required: + - branch + - path + type: object observedGeneration: description: ObservedGeneration is the latest generation observed by the controller. format: int64 type: integer + retargetingTo: + description: |- + RetargetingTo is the destination a retarget in progress is building, set while + Retargeting=True and cleared once the move settles. It exists so a move is + self-describing, and so a SECOND destination change arriving mid-move can name the + intermediate folder the first move had already begun writing — otherwise that folder + would be orphaned without ever being named. See the RetargetSuperseded event. + properties: + branch: + description: Branch the materialization lives on. + type: string + path: + description: Path the materialization lives under, relative to + the repository root. + type: string + sourceCluster: + description: |- + SourceCluster names the kubeconfig Secret whose cluster this materialization was + mirrored from. Empty means the cluster the operator runs in. + type: string + required: + - branch + - path + type: object streams: description: |- Streams is the bounded data-plane roll-up over this GitTarget's tracked types. diff --git a/config/crd/bases/configbutler.ai_watchrules.yaml b/config/crd/bases/configbutler.ai_watchrules.yaml index d47d05f4..f7277a3c 100644 --- a/config/crd/bases/configbutler.ai_watchrules.yaml +++ b/config/crd/bases/configbutler.ai_watchrules.yaml @@ -76,6 +76,11 @@ spec: Rules define which resources to watch within this namespace. Multiple rules create a logical OR - a resource matching ANY rule is watched. Each rule can specify operations, API groups, versions, and resource types. + + An exclusion (excludeUsers / excludeFieldManagers) is a veto WITHIN its own rule, + not a global filter: a change is mirrored when at least one rule both selects it + and does not exclude its writer. Adding an unrestricted rule for a type therefore + re-admits everything another rule excluded for that type. items: description: |- ResourceRule defines a set of namespaced resources to watch. @@ -113,6 +118,58 @@ spec: items: type: string type: array + excludeFieldManagers: + description: |- + ExcludeFieldManagers drops a live change whose last writer is one of these + field managers, so a GitOps forward leg (Flux, Argo CD) applying this branch + back into the cluster does not have its own applies mirrored into Git. + + Reach for this rather than ExcludeUsers: it reads metadata.managedFields off + the object, so it needs no audit fact, cannot race the attribution grace + window, and works in configured-author mode. + + The "last writer" is the manager of the newest managedFields entry. When + several entries share that timestamp the change is excluded only if every + tied manager is listed — when in doubt, the change is mirrored. An object + with no managedFields is never excluded. + + It is NOT evaluated for DELETE: managedFields names who last wrote an object, + not who deleted it, so excluding a delete this way would silently ignore a + human deleting a Flux-managed resource. Use ExcludeUsers for deletes. + + Examples: + - ["kustomize-controller"] ignores writes applied by Flux's kustomize-controller + - ["argocd-controller"] ignores writes applied by Argo CD + items: + maxLength: 128 + minLength: 1 + type: string + maxItems: 32 + type: array + excludeUsers: + description: |- + ExcludeUsers drops a live change attributed to one of these identities: the + impersonated user when impersonation is in play, otherwise the authenticated + user, exactly as the audit event records it. + + It therefore requires author attribution (--author-attribution) and a working + audit webhook. When the author cannot be resolved — attribution disabled, or the + grace window elapsed with no matching fact — the change is mirrored rather than + dropped: losing a human's edit because we failed to identify its author is worse + than mirroring one machine write. Prefer ExcludeFieldManagers, which has no such + failure mode. + + Unlike ExcludeFieldManagers this does apply to DELETE, because the audit fact + names the actor who issued the delete. + + Example: + - ["system:serviceaccount:flux-system:kustomize-controller"] + items: + maxLength: 316 + minLength: 1 + type: string + maxItems: 32 + type: array operations: description: |- Operations to watch. If empty, watches all operations (CREATE, UPDATE, DELETE). diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 40ca498d..42e3c69f 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -4,6 +4,13 @@ kind: ClusterRole metadata: name: gitops-reverser rules: +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch - apiGroups: - "" resources: @@ -31,6 +38,12 @@ rules: - get - list - watch +- apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create - apiGroups: - configbutler.ai resources: diff --git a/config/webhook/validating-webhook.yaml b/config/webhook/validating-webhook.yaml index 0f7b1ea9..c988ef46 100644 --- a/config/webhook/validating-webhook.yaml +++ b/config/webhook/validating-webhook.yaml @@ -81,6 +81,14 @@ webhooks: # reject a user's CommitRequest. The handler also returns Allowed even when the Redis # write errors, so the command's success never depends on this webhook at all — # Ignore only covers the webhook being entirely unreachable. + # + # The handler DOES deny an unauthorized CommitRequest spec.author, and Ignore means that + # denial can be bypassed by taking the webhook down. That is by design and it is safe: + # the webhook records its authorization verdict on the author record, and the controller + # honors spec.author only against a record carrying it. A bypassed webhook writes no + # record, so the assertion is ignored rather than silently honored. The real gate is the + # controller; this denial is the legible error. See + # docs/design/multi-tenant/asserted-commit-author.md. failurePolicy: Ignore matchPolicy: Equivalent name: validate-operator-types.configbutler.ai diff --git a/docs/README.md b/docs/README.md index a7d738b2..be7c3d41 100644 --- a/docs/README.md +++ b/docs/README.md @@ -21,6 +21,10 @@ If you only want the supported product docs, start with the files below. - [`security-model.md`](security-model.md): controller access, trust boundaries, and the Git credentials Secret shape - [`bi-directional.md`](bi-directional.md): safe shared-path and handoff patterns +- [`design/multi-tenant/`](design/multi-tenant/): running outside a single cluster — mirroring a + remote cluster, ignoring a GitOps forward leg's own writes, sharing one Redis, moving a `GitTarget` +- [`../pkg/manifestanalyzer/`](../pkg/manifestanalyzer/): the public Go API and JSON contract for + "may this folder become a `GitTarget`?" - [`alternatives.md`](alternatives.md): nearby tools and when another approach fits better - [`UPGRADING.md`](UPGRADING.md): breaking changes and migration steps, newest first - [`../CONTRIBUTING.md`](../CONTRIBUTING.md): contributor workflow and validation commands diff --git a/docs/UPGRADING.md b/docs/UPGRADING.md index 781af79a..919e4979 100644 --- a/docs/UPGRADING.md +++ b/docs/UPGRADING.md @@ -7,6 +7,57 @@ guidance that the changelog's breaking-change entries link to. We are pre-1.0, so breaking changes bump the **minor** version (release-please is configured with `bump-minor-pre-major`) rather than the major. Read the relevant entry before upgrading across it. +## Unreleased — `GitTarget.spec.branch` and `.spec.path` are now mutable (next minor; behavior change) + +Changing either used to be rejected by the API server, and relocating a `GitTarget` meant +delete-and-recreate. Both are now mutable, and changing one is a supported **retarget**: the +controller tears the old materialization down (including its watch resume cursors) and rebuilds the +folder from a fresh full snapshot at the new destination. `spec.providerRef` stays immutable. + +- **The old folder is never deleted.** It becomes ordinary, unmanaged Git content. The + `Retargeting=False` condition message names it so you can `git rm` it deliberately. +- New `status.observedDestination` records the destination the current materialization actually + belongs to. New `Retargeting` condition is True while it disagrees with spec. +- A retarget onto a path that overlaps another `GitTarget` on the same provider and branch is + refused by the ordinary `Validated` gate, and the target keeps serving its current destination. + +**Migration** + +- Nothing to do. An operator that caught the immutable-field rejection and delete-and-recreated can + drop that code path; a plain `kubectl patch` of `spec.path` now works. +- If you relied on the API server rejecting a `spec.branch`/`spec.path` change as a guardrail — + a GitOps tool re-applying a `GitTarget` from a template, say — note that such an apply will now + retarget instead of erroring. Gate it the way you gate any other spec change. + +See [design/multi-tenant/gittarget-retarget.md](design/multi-tenant/gittarget-retarget.md). + +## Unreleased — `manifest-analyzer --format json` now emits a versioned contract (next minor; breaking) + +The analyzer's machine-readable output moved to the new public +[`pkg/manifestanalyzer`](../pkg/manifestanalyzer), which is a supported Go API and the single +definition of the JSON documents the CLI prints. Both affected modes gained a `schemaVersion` +field, and one field was dropped: + +- `--mode scan --format json` no longer carries `plan`. In scan mode the analyzer has no cluster + state and no desired resources, so `plan` was structurally always `{"counts":{},"actions":null}` — + it never carried information. The meaningful fields (`accepted`, `issues`, `retained`) are + unchanged, and `issues` now marshals as `[]` rather than `null` when there are none. +- `--mode repo-walker --format json` is otherwise unchanged. +- Retained entries now omit `identity` for a whole-file retention (an ordinary + `kustomization.yaml`, which names no resource) instead of emitting a zero-valued object. It is + still present for the refused mixed-file case, where a named resource hides inside a build + directive. +- `--mode analyze` and every `--format text` output are unchanged. + +**Migration** + +- Read `schemaVersion` and ignore fields you do not know; new fields are added within a schema major. +- If you exec'd the binary only to reach the acceptance verdict, prefer importing + `pkg/manifestanalyzer` and calling `ScanFolder` / `ScanRepo`. They run the same acceptance gate the + operator's writer enforces, so a tool built on them cannot drift from the operator that will later + adopt (or refuse) the folder. +- If you parsed `plan` from scan mode, you were reading an empty object; drop the field. + ## Unreleased — chart defaults now run Redis-free (next minor; behavior change) The Helm chart now defaults to the simple, Redis-free `configured-author` path, so a bare diff --git a/docs/architecture.md b/docs/architecture.md index f0a5f394..8f1654d8 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -87,6 +87,11 @@ The solution, in the vocabulary used throughout this document: * **One `BranchWorker` per Git branch serializes all writes.** Every write to a branch funnels through a single worker and a single commit window, which keeps concurrent GitTargets and authors from racing each other into a corrupt tree. +* **The config plane and the watched cluster are separate.** The operator reads its own CRs and + credentials from the cluster it runs in; each `GitTarget` may name the cluster it mirrors + (`spec.sourceCluster`). Every per-cluster surface — API discovery catalog, type-followability + registry, dynamic client, API-surface trigger informers — lives in one `clusterContext` per source + cluster. A single-cluster install has exactly one and behaves as it always did. *** @@ -1108,6 +1113,7 @@ Current limitations: | [internal/git/manifestedit/](../internal/git/manifestedit/) | YAML document editor | | [internal/giteaclient/](../internal/giteaclient/) | Gitea helper client | | [internal/manifestanalyzer/](../internal/manifestanalyzer/) | manifest inventory, acceptance, and resync planning | +| [pkg/manifestanalyzer/](../pkg/manifestanalyzer/) | **public**, versioned projection of the acceptance gate and repo discovery — the contract the CLI's `--format json` prints | | [internal/manifestreport/](../internal/manifestreport/) | projection of Kubernetes objects into comparable manifest reports | | [internal/queue/](../internal/queue/) | Redis attribution index (audit facts keyed for the join) and per-watch resume cursors | | [internal/reconcile/](../internal/reconcile/) | per-GitTarget event stream (watch event → branch worker) | @@ -1118,8 +1124,8 @@ Current limitations: | [internal/telemetry/](../internal/telemetry/) | metrics and OTLP setup | | [internal/types/](../internal/types/) | shared resource identity/reference and sensitivity policy | | [internal/typeset/](../internal/typeset/) | type followability registry, lookup model, and the relevance filter (controller-owned → don't mirror) | -| [internal/watch/](../internal/watch/) | discovery catalog, watch manager, per-`(GVR, scope)` raw watches with `sendInitialEvents` replay + mark-and-sweep, the author resolver, watched type tables, event router | -| [internal/webhook/](../internal/webhook/) | `/audit-webhook` ingress and attribution fact extraction (no body joiner) | +| [internal/watch/](../internal/watch/) | per-source-cluster contexts (discovery catalog, type registry, clients), watch manager, per-`(GVR, scope)` raw watches with `sendInitialEvents` replay + mark-and-sweep, the author resolver, write exclusions, watched type tables, event router | +| [internal/webhook/](../internal/webhook/) | `/audit-webhook` ingress and attribution fact extraction (no body joiner); `/validate-operator-types` command-author capture and the `assert-author` guard | *** diff --git a/docs/configuration.md b/docs/configuration.md index c06b68cc..30dffd88 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -319,10 +319,13 @@ For the platform-facing behavior behind "valid signature" versus "verified badge The important fields are: -- `spec.providerRef`: which `GitProvider` backs this target -- `spec.branch`: which allowed branch to write to +- `spec.providerRef`: which `GitProvider` backs this target. **Immutable** — a different + repository is a different object. +- `spec.branch`: which allowed branch to write to. Mutable; changing it retargets. - `spec.path`: required relative path inside the repository; use `.` only when you deliberately - want the repository root + want the repository root. Mutable; changing it retargets. +- `spec.sourceCluster`: optional; the cluster this target mirrors **from**. Omit it for the + cluster the operator runs in (see [Mirroring a remote cluster](#mirroring-a-remote-cluster-specsourcecluster)) - `spec.encryption`: how `Secret` resources should be encrypted before commit - `spec.placement`: optional policy for where **new** resources are written (see [Where new resources are written](#where-new-resources-are-written-specplacement)); omit it to follow @@ -365,10 +368,91 @@ The most useful status fields are: - `Validated` and `EncryptionConfigured`: control-plane details. - `StreamsRunning`: true when the source watches are past initial replay and routing live events. - `GitPathAccepted`: true when the target Git path is safe to materialize. +- `Retargeting`: true while `spec` and `status.observedDestination` disagree — the target is + moving (see [Moving a GitTarget](#moving-a-gittarget-retarget)). +- `status.observedDestination`: the branch, path, and source cluster the current materialization + actually belongs to. This, not `spec`, answers "which folder is this target's content in?". +- `status.retargetingTo`: where an in-flight move is building. Cleared once it settles. - `status.streams`: bounded counts for tracked, running, replaying, and blocked streams. Use conditions for automation. +### Moving a `GitTarget` (retarget) + +`spec.branch` and `spec.path` are mutable. Changing either is a supported **retarget**: the +controller tears the old materialization down — its watches, its event stream, and its durable +watch-resume cursors — and rebuilds the folder from a fresh full snapshot at the new destination. +`Retargeting=True` while that runs. + +Three things are worth knowing before you patch a path: + +- **The old folder is never deleted.** It becomes ordinary, unmanaged Git content. Deleting from + Git is the one irreversible thing this operator does, and a destination change is the moment you + are least sure of what you meant. The `Retargeting=False` message names the abandoned folder, so + you can `git rm` it deliberately. +- **The new folder is rebuilt from a full snapshot**, so the resources that already existed land in + it — not only the ones that change afterwards. +- **A retarget onto a conflicting path is refused** by the ordinary `Validated` gate, and the target + then mirrors *nothing* (`Stalled=True`, reason `TargetConflict`) until you pick a free path. It does + not fall back to the old destination: the old materialization is torn down before the new one is + validated, precisely so a live event can never be written to the new path on the old branch. + +`spec.providerRef` stays immutable. Pointing at a different repository is not a move: there is +nothing to migrate and nothing to observe. Delete and recreate the `GitTarget`. + +### Mirroring a remote cluster (`spec.sourceCluster`) + +By default a `GitTarget` mirrors the cluster the operator runs in, and nothing needs configuring. + +`spec.sourceCluster` separates the two jobs one kubeconfig used to serve: the operator reads its own +CRs and Git credentials from the cluster it runs in (**the config plane**) and watches the resources +on the cluster you name here. The shape is deliberately Flux's `Kustomization.spec.kubeConfig`. + +```yaml +apiVersion: configbutler.ai/v1alpha3 +kind: GitTarget +metadata: + name: acme + namespace: team-a +spec: + providerRef: + name: acme-provider + branch: main + path: clusters/acme + sourceCluster: + kubeConfigSecretRef: + name: acme-kubeconfig # a Secret in namespace team-a, on the config plane + key: value.yaml # optional; defaults to value.yaml, Flux's convention +``` + +What this buys: + +- The Git write credential — usually scoped far wider than the one repository a `GitTarget` names — + never has to live on the cluster you are mirroring. +- The watched cluster holds only the watched resources. It does not need the `configbutler.ai` CRDs + installed at all. +- **One operator can mirror many clusters**, because each `GitTarget` carries its own source. + +Semantics and caveats: + +- A `WatchRule` still watches the namespace **it lives in**, resolved on the source cluster. A + `WatchRule` in config-plane namespace `team-a` watches namespace `team-a` on the remote. A + `ClusterWatchRule` watches the whole source cluster. +- It sits on `GitTarget`, not `WatchRule`, because a `GitTarget` already owns exactly one + materialization. Two `WatchRule`s naming different clusters for one folder would make the + mark-and-sweep alternately delete each cluster's objects. +- Types resolve against the **source cluster's** API surface. A CRD installed only locally is not + followable for a remote target. +- The kubeconfig `Secret` is read on demand, parsed, and dropped; only its `resourceVersion` is + kept, so **rotating the Secret's contents is transparent** and rebuilds the clients once. +- `sourceCluster` is part of the destination identity: repointing it at a *different* cluster + retargets the `GitTarget`, exactly as changing the path does. +- If the Secret is missing, empty at its key, or not a usable kubeconfig, the target reports + `Validated=False` with reason `SourceClusterUnreachable` and names the Secret. + +The operator throttles its requests to a remote cluster with `--source-cluster-qps` (default `20`) +and `--source-cluster-burst` (default `30`). + ### Where new resources are written (`spec.placement`) Placement decides the file path for a resource that has **no document in Git yet**. Once a document @@ -566,6 +650,9 @@ are: across the served API surface. - `apiVersions`: a served version such as `v1`; omitted means the preferred served version. - `resources`: plural resource names such as `configmaps`, `secrets`, or `*`. +- `excludeFieldManagers`: field managers whose writes this rule declines to mirror (see + [Ignoring a GitOps forward leg's own writes](#ignoring-a-gitops-forward-legs-own-writes)). +- `excludeUsers`: attributed identities whose writes this rule declines to mirror. Subresources such as `deployments/scale` are not valid rule resources. GitOps Reverser mirrors top-level resources; selected subresource effects are handled separately by the controller. @@ -590,6 +677,72 @@ spec: Use `WatchRule` when the watched resources and the `GitTarget` live in the same namespace. +### Ignoring a GitOps forward leg's own writes + +If you run GitOps Reverser **and** a GitOps forward leg (Flux, Argo CD) on the same branch, the +reverser will commit the forward leg's own applies. The loop is: a human edits a resource, the +reverser commits it, the forward leg applies the commit back into the cluster — stamping its own +labels and `managedFields` onto the object — the reverser sees that as a live update, and commits +again. It terminates only because the content eventually stops changing. + +`excludeFieldManagers` breaks it, and is the field to reach for: + +```yaml +rules: + - resources: ["configmaps", "deployments"] + excludeFieldManagers: ["kustomize-controller"] # Flux + # excludeFieldManagers: ["argocd-controller"] # Argo CD +``` + +It reads `metadata.managedFields` off the live object, so it needs no audit fact, cannot race the +attribution grace window, and works in configured-author mode. A label selector cannot do this job: +a GitOps tool's labels *persist* on the object, so a selector would also ignore a human's later edit +of a tool-managed resource. The last writer is a property of the *write*; the labels are a property +of the *object*. + +Semantics worth knowing: + +- **The last writer is the newest `managedFields` entry.** If several entries share that timestamp, + the write is excluded only when every tied manager is excluded — when in doubt, it is mirrored. +- **`DELETE` is never excluded by field manager.** `managedFields` names who last *wrote* an object, + not who deleted it, so excluding deletes this way would silently ignore a human deleting a + Flux-managed resource. Use `excludeUsers` for deletes. +- **Exclusions are a veto within their own rule, not a global filter.** Rules are a logical OR, so a + second, unrestricted rule for the same type re-admits everything the first excluded. +- **An exclusion suppresses a *write*, not the *state*.** A replay or resync still lists the object; + otherwise the mark-and-sweep would delete its file from Git. And the next *non-excluded* write to + the same object commits its whole current state, including whatever the excluded manager changed. + The guarantee is "this manager's writes never *cause* a commit", not "this manager's changes never + reach Git". + +`excludeUsers` matches the identity the audit webhook attributed the write to (the impersonated user +when impersonation is in play, otherwise the authenticated user): + +```yaml +rules: + - resources: ["configmaps"] + excludeUsers: ["system:serviceaccount:flux-system:kustomize-controller"] +``` + +It therefore requires `--author-attribution` and a working audit webhook, and — unlike +`excludeFieldManagers` — it *does* apply to deletes. When the author cannot be resolved (attribution +disabled, or the grace window elapsed with no matching fact) the write is **mirrored, not dropped**: +losing a human's edit because we could not identify its author is worse than mirroring one machine +write. The operator logs a warning once per rule when `excludeUsers` is set without attribution. + +Dropped events are counted by +`gitopsreverser_watch_events_excluded_total{gittarget_namespace, gittarget_name, group, resource, reason}`, +where `reason` is `field_manager` or `user`. + +A zero rate does **not** by itself mean the exclusion is misconfigured. GitOps Reverser already strips +a GitOps tool's own bookkeeping from Git content — labels and annotations under +`kustomize.toolkit.fluxcd.io/`, `kro.run/`, `applyset.kubernetes.io/`, plus `managedFields` and the +server-generated metadata fields — so an apply that only stamps those carries no change to mirror and +is dropped as a no-op before any exclusion is consulted. An exclusion decides the case where the +forward leg's apply changes real content, which happens whenever Git and the cluster disagree about a +managed field. If you expect drops and see none, check that the manager name matches the object's +`metadata.managedFields`. + ## `ClusterWatchRule` `ClusterWatchRule` is the cluster-scoped variant. Use it when you need to watch: @@ -634,6 +787,8 @@ The important fields are: - `spec.message`: optional verbatim commit message - `spec.closeDelaySeconds`: optional 0-300 second delay before the open window is closed, after the request author is known — an extra collect window +- `spec.author`: optional; name the human this commit is for, instead of deriving them from an + apiserver audit fact. **A privilege** — see [Asserting a commit author](#asserting-a-commit-author). Example: @@ -652,6 +807,62 @@ spec: The entire spec is immutable. Create a new `CommitRequest` for each save attempt. +### Asserting a commit author + +Commit attribution is normally derived from an apiserver **audit** fact. That is the right default — +it names the actor who really caused a change, and cannot be forged by anyone who can create a +`CommitRequest`. But it means attribution is only available where an audit webhook can be configured, +which excludes every hosted control plane whose apiserver flags you do not own. + +`spec.author` lets a trusted client state who a commit is for: + +```yaml +spec: + targetRef: + name: tenants + author: + name: "Ada Lovelace" + email: "ada@example.com" # optional; derived from name when omitted +``` + +Asserting an author is a **privilege**, not a field anyone may set. It is authorized by an RBAC verb +on the target, in the style of `bind`, `escalate` and `impersonate`: + +```yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: gitops-api-assert-author + namespace: team-a +rules: + - apiGroups: ["configbutler.ai"] + resources: ["gittargets"] + resourceNames: ["tenants"] # scoped to one target + verbs: ["assert-author"] +``` + +The `/validate-operator-types` admission webhook issues a `SubjectAccessReview` for the requester and +**denies** an unauthorized create, naming the rule that would grant it. But that webhook is +`failurePolicy: Ignore` by design, so the *controller* is the real gate: it honors `spec.author` only +when an admission record exists for the object and carries the authorized verdict. If the webhook is +off, was bypassed, or Redis is not configured, the assertion is **ignored** — the commit is authored +by the configured committer and the request reports `AuthorAttributed=False` with reason +`AuthorAssertionUnverified`. Fail-closed, independent of `failurePolicy`. + +An authorized assertion reports `AuthorAttributed=True` with reason `AuthorAsserted`, becomes the +commit's **author** signature, and attaches to any open window for the target — the assertion is a +statement about the commit being made, not a claim to be the actor the audit stream recorded. The +**committer** stays the operator's configured identity, so a reader can always tell the reverser +committed on someone's behalf. + +Requirements: the admission webhook enabled (`servers.admission.enabled`, the chart default), Redis +configured (`queue.redis.addr`), and `create` on `subjectaccessreviews.authorization.k8s.io` for the +operator's ServiceAccount (in the chart's `ClusterRole` already). + +**Treat `assert-author` exactly like `impersonate`.** The asserted `name`/`email` are free text and +are not verified against a real identity; granting the verb grants the ability to write any author +into that repository's history. + Progress and outcome are reported through kstatus-compatible **conditions** (no `phase` string). `kubectl get commitrequest` surfaces `Ready`, `AuthorAttributed`, and `Pushed`; `kubectl wait --for=condition=Ready` blocks until the request settles: @@ -693,8 +904,33 @@ queue: auth: existingSecret: "valkey-auth" existingSecretKey: "password" + keyPrefix: "gitops-reverser" +``` + +### Sharing one Redis between several reversers + +Every key this operator writes — watch resume cursors, attribution facts, command author records — +is rooted at `queue.redis.keyPrefix` (`--redis-key-prefix`, default `gitops-reverser`). Give each +reverser its own prefix to share one Redis/Valkey between more instances than the 16 logical +databases `--redis-db` can separate: + +```yaml +queue: + redis: + addr: "valkey:6379" + keyPrefix: "cell-a:tenant-7" ``` +A Valkey ACL can *enforce* the prefix rather than trust it — pair it with `--redis-username` / +`--redis-password` and a `~:*` key pattern. + +Allowed characters are `[A-Za-z0-9]`, `-`, `_`, `.` and `:`. Glob metacharacters and `%` are rejected: +the attribution fact-index gauge `SCAN`s `:author:v1:audit:*`, and a glob in the prefix would +silently make it count the wrong keyspace. A trailing `:` is normalized away. + +Changing the prefix orphans the keys written under the previous one. Cursors then cold-replay once, +which is safe; the orphans expire on their own TTLs. + When attribution is enabled, these flags tune the join: - `--author-attribution-ttl` (default `10m`): how long an attribution fact is retained waiting for the diff --git a/docs/design/gitops-api/f8-repo-discovery-and-onboarding-scan.md b/docs/design/gitops-api/f8-repo-discovery-and-onboarding-scan.md index 1019934e..21889d52 100644 --- a/docs/design/gitops-api/f8-repo-discovery-and-onboarding-scan.md +++ b/docs/design/gitops-api/f8-repo-discovery-and-onboarding-scan.md @@ -246,6 +246,11 @@ followed — the same posture as `ScanDir`, over the whole tree. It reuses `coll `parseKustomizations`/`renderRoots`, the `Scan`/acceptance gate, and mirrors `gittarget_path_overlap` for nesting. +A product layer consuming this from Go imports [`pkg/manifestanalyzer`](../../../pkg/manifestanalyzer) +— `ScanRepo` and `ScanFolder` — rather than exec'ing the binary. That package is the +supported, versioned projection of the reports below; the engine above stays internal and +free to move. See [../multi-tenant/README.md](../multi-tenant/README.md) item 4. + **Report shape as shipped**, refining the sketch above. Per candidate: `path`, `layout`, `acceptedByOperator`, `refusalReasons[]` (`{code, detail}`), `renderRoot`, `readScope[]`, `inferredNamespace`, `resources`, `overlapsWith[]`. diff --git a/docs/design/multi-tenant/README.md b/docs/design/multi-tenant/README.md new file mode 100644 index 00000000..2c4c2b09 --- /dev/null +++ b/docs/design/multi-tenant/README.md @@ -0,0 +1,79 @@ +# Running GitOps Reverser outside a single cluster + +> Status: active workstream +> Captured: 2026-07-09 +> Related: [../../architecture.md](../../architecture.md), +> [../../security-model.md](../../security-model.md), +> [../../configuration.md](../../configuration.md) + +GitOps Reverser was designed for the shape it is most often installed in: one +operator, in one cluster, watching that cluster, writing to Git. Every +assumption that shape allows was taken — a single `rest.Config`, a single +Redis keyspace, a single API surface, one immutable destination per +`GitTarget`. + +Operators who run the reverser in a **multi-tenant or multi-cluster** setting +hit the edges of those assumptions, and so does anyone who pairs the reverser +with a GitOps **forward leg** (Flux, Argo CD) on the same branch. This +workstream collects seven changes that move those edges. They are independent; +each is useful on its own. + +## The seven + +| # | Change | Who it affects | Design | +|---|--------|----------------|--------| +| 1 | **Separate the config plane from the watched cluster** — a `GitTarget` may name the cluster it mirrors | anyone watching a *remote* cluster; unlocks multi-cluster and multi-tenant installs | [config-plane-split.md](config-plane-split.md) | +| 2 | **Ignore writes by an identity or field manager** | **every** install that pairs a reverser with a GitOps forward leg | [identity-write-exclusion.md](identity-write-exclusion.md) | +| 3 | **A Redis key prefix** | anyone running more than one reverser against one Redis/Valkey | [../../configuration.md](../../configuration.md) | +| 4 | **A public `pkg/manifestanalyzer`** | anyone building tooling on the acceptance rules | [../../../pkg/manifestanalyzer/doc.go](../../../pkg/manifestanalyzer/doc.go) | +| 5 | **A `CommitRequest` that can assert an author** | anyone on a control plane whose apiserver audit flags they cannot set | [asserted-commit-author.md](asserted-commit-author.md) | +| 6 | **A movable `GitTarget` destination** | anyone who ever repoints a target | [gittarget-retarget.md](gittarget-retarget.md) | +| 7 | **Degrade gracefully without `apiregistration.k8s.io`** | anyone on an API server that does not serve APIService aggregation | this document, below | + +## Why #2 is first + +A reverser paired with a Flux (or Argo CD) forward leg on the same branch +**commits its own forward leg's applies**. The loop is: + +1. A human edits a ConfigMap. The reverser commits it. +2. The forward leg sees the new commit and applies it back into the cluster. + The apply is not byte-identical to what the human wrote — the GitOps tool + stamps its own labels, annotations, and `managedFields` entries onto the + object. +3. That apply is a live UPDATE. The reverser mirrors it, and commits again. +4. The forward leg sees *that* commit, applies, and so on. + +The loop terminates only because the content eventually stops changing — a +convergence property standing in for an invariant nobody declared. Until it +converges it produces a run of machine-authored commits, each one re-triggering +the forward leg. + +Before this workstream there was no identity-based filter anywhere on +`GitTargetSpec`, `WatchRuleSpec`, or `ResourceRule` — the only filters were +`operations` and the type matchers. Every reverser deployed alongside a forward +leg had this behavior. + +`excludeFieldManagers` fixes it from watch state alone. See +[identity-write-exclusion.md](identity-write-exclusion.md) for why a label +selector cannot: a GitOps tool's labels *persist* on the object, so a selector +would also ignore a human's later edit of a tool-managed resource. The last +writer is a property of the write; the labels are a property of the object. + +## #7 — APIService aggregation is not universal + +The watch manager keeps its API-resource catalog fresh with two trigger +informers: one on `CustomResourceDefinition`, one on `APIService`. The +`APIService` informer was created unconditionally, so on an API server that does +not serve `apiregistration.k8s.io` — kcp, and other non-aggregating control +planes — client-go's reflector retried and logged forever. Benign, endlessly +repeated, and therefore exactly the kind of noise that hides a real error. + +Both trigger informers are now started only when discovery reports their +resource is served, and are (re-)evaluated on every catalog refresh, so an +aggregation layer installed later is picked up without a restart. When a trigger +is unavailable the manager logs one `Info` line naming what it skipped, and the +catalog still refreshes on its 30s tick. + +Discovery itself was already tolerant: a group that fails to serve is recorded +in the scan's degraded set, logged edge-triggered, and the catalog stays ready +on the remaining groups. diff --git a/docs/design/multi-tenant/asserted-commit-author.md b/docs/design/multi-tenant/asserted-commit-author.md new file mode 100644 index 00000000..061423ef --- /dev/null +++ b/docs/design/multi-tenant/asserted-commit-author.md @@ -0,0 +1,118 @@ +# A CommitRequest that can assert an author + +> Status: implemented +> Related: [README.md](README.md), +> [../commitrequest-admission-authorship.md](../commitrequest-admission-authorship.md), +> [../../attribution-setup-guide.md](../../attribution-setup-guide.md) + +## Problem + +Commit attribution is derived from an apiserver **audit** fact. That is the right +default — it names the actor who really caused a change, and it cannot be forged +by anyone who can create a `CommitRequest`. But it means attribution is only +available where an audit webhook can be configured, which excludes every hosted +control plane whose apiserver flags the operator does not own. + +An authenticated control plane in front of the API already knows who the human +is: it verified their token before impersonating them. Making it re-derive that +identity through the apiserver's audit stream is a long way around. + +`CommitRequestSpec` was `targetRef` + `message` + `closeDelaySeconds`. It +*finalizes* an open commit window; it could not *open* one with an identity. + +## Shape + +```yaml +apiVersion: configbutler.ai/v1alpha3 +kind: CommitRequest +spec: + targetRef: { name: tenants } + author: # NEW + name: "Ada Lovelace" + email: "ada@example.com" +``` + +Asserting an author is a **privilege**, not a field anyone may set. It is +authorized by an RBAC verb on the target, in the style of `bind`, `escalate`, and +`impersonate`: + +```yaml +rules: + - apiGroups: ["configbutler.ai"] + resources: ["gittargets"] + resourceNames: ["tenants"] + verbs: ["assert-author"] +``` + +## The guard, and why it does not depend on `failurePolicy` + +The `/validate-operator-types` admission webhook already captures the submitter +of a `CommitRequest` into Redis, keyed by the object's UID. It now also, when +`spec.author` is set: + +1. issues a `SubjectAccessReview` for the requester against + `{verb: assert-author, group: configbutler.ai, resource: gittargets, + namespace: , name: }`; +2. **denies** the create when the review says no — so an unauthorized caller + gets an immediate, legible error; +3. records the verdict on the admission record when it says yes. + +The webhook is `failurePolicy: Ignore` by design (a down webhook must not wedge +the API). A guard that lived only there would therefore be bypassable by taking +the webhook down. So the **controller** is the real gate: it honors `spec.author` +**only** when an admission record exists for the object's UID *and* that record +carries the authorized verdict. No record — the webhook was off, bypassed, or +Redis is not configured — means the assertion is ignored, the commit is authored +by the configured committer, and the request reports: + +``` +AuthorAttributed=False reason=AuthorAssertionUnverified +``` + +Fail-closed, and independent of `failurePolicy`. + +## Effect on the commit + +Before this change, `AttachCommitRequest.Author` was a *window-matching key*: it +selected which open commit window to finalize by comparing against the window's +author (which came from the mirrored resource's audit events), and it never +reached the Git signature. + +An asserted author is different in both respects: + +- it **matches any open window** for the target, because the assertion is a + statement about the commit being made, not a claim to be the actor the audit + stream happened to record; +- it **becomes the commit's author signature** (`name `), with the + configured committer still the committer, exactly as an audit-attributed commit + is signed. + +`AuthorAttributed=True` with reason `AuthorAsserted` distinguishes it from +`AttributedFromAdmission`. + +## Ordering against audit attribution + +When both are available, the assertion wins for the commit this `CommitRequest` +finalizes. It is the more specific, more recent statement, made by a caller who +had to hold an RBAC verb to make it. Mirrored-resource attribution +(`--author-attribution`) is unaffected for every commit *not* finalized by an +asserting `CommitRequest`. + +## Threat model + +- The verb is checked against the **named `GitTarget`**, so a tenant granted + `assert-author` on their own target cannot author commits into someone else's. +- The asserted `name`/`email` are free text and are **not** verified to + correspond to a real identity. They are what the trusted control plane says + they are. Granting `assert-author` is granting the ability to write any author + into the repository's history — treat it exactly like granting `impersonate`. +- The commit's *committer* remains the operator's configured identity, and + commit signing (when configured) still signs as the committer. A reader can + always tell that a commit was made by the reverser on someone's behalf. +- An **unverified** assertion (`AuthorAssertionUnverified`) degrades to exactly the + pre-existing committer-fallback path: the request carries no author, so it finalizes + whichever *unattributed* open window exists for its `GitTarget`, with its own message. + It cannot reach a window the audit stream attributed to a named actor, and the commit is + never signed with the rejected identity. This is the same reach a `CommitRequest` with + no `spec.author` from an unattributed submitter has always had — the guard denies the + *identity claim*, not the ability to save. diff --git a/docs/design/multi-tenant/config-plane-split.md b/docs/design/multi-tenant/config-plane-split.md new file mode 100644 index 00000000..1058234f --- /dev/null +++ b/docs/design/multi-tenant/config-plane-split.md @@ -0,0 +1,156 @@ +# Separating the config plane from the watched cluster + +> Status: implemented +> Related: [README.md](README.md), [../../security-model.md](../../security-model.md) + +## Problem + +GitOps Reverser built exactly one client: + +```go +mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ … }) +``` + +That single `rest.Config` served two jobs that are conceptually unrelated: + +- **the config plane** — where `GitProvider`, `GitTarget`, `WatchRule` and the + `secretRef` Git credentials are read; +- **the watched cluster** — where the resources it mirrors to Git actually live. + +Because `GitProvider.spec.secretRef` is a local reference and +`WatchRule.spec.targetRef` must name a same-namespace `GitTarget`, all four +objects had to sit in one namespace **on the cluster being watched**. Nothing +chose this; it fell out of having one kubeconfig. + +For an operator mirroring a cluster they also hand to someone else, that means a +Git write credential — often scoped far more broadly than the one repository a +`GitTarget` names — lives one RBAC rule away from whoever can read Secrets in +that namespace. The isolation rests on a policy decision rather than on a +boundary. + +## Shape + +`GitTarget` may name the cluster it mirrors, exactly as Flux's `Kustomization` +names the cluster it applies to: + +```yaml +apiVersion: configbutler.ai/v1alpha3 +kind: GitTarget +spec: + providerRef: { name: acme } + branch: main + path: clusters/acme + sourceCluster: # NEW — omit for "the cluster I run in" + kubeConfigSecretRef: + name: acme-workspace-kubeconfig + key: value.yaml # default: value.yaml, Flux's convention +``` + +The Secret is read from the `GitTarget`'s own namespace, on the cluster the operator +runs in. The cluster id the data plane keys on is `//` — the key +is part of the identity, because two `GitTarget`s naming one Secret under different +keys are pointed at different kubeconfigs. Its value is an ordinary kubeconfig. + +### Why `GitTarget` and not `WatchRule` + +Both were candidates. `GitTarget` wins because a `GitTarget` is already the unit +that owns exactly one materialization: one (provider, branch, folder). Adding +the source cluster makes it one (cluster, provider, branch, folder) — still one +owner, one folder, one desired state. The watch data plane is *already* keyed by +`GitTarget`, so the cluster comes along for free and no two `WatchRule`s can +disagree about which cluster a folder mirrors. + +Putting it on `WatchRule` would allow two rules pointing at different clusters to +feed one folder, and the mark-and-sweep would then alternately delete each +cluster's objects. That is not a configuration anyone wants to be able to write. + +`WatchRule` keeps its meaning: it watches the namespace **it lives in**, on the +`GitTarget`'s source cluster. A `ClusterWatchRule` watches the whole source +cluster. When the source cluster is remote, the namespace names are resolved on +the remote — a `WatchRule` in config-plane namespace `team-a` watches namespace +`team-a` on the remote cluster. + +### The source is part of the destination identity + +`sourceCluster` names where a folder's content comes *from*, so changing it changes +what the folder means. It therefore participates in the retarget lifecycle +([gittarget-retarget.md](gittarget-retarget.md)) exactly as `branch` and `path` do: +the old materialization is torn down and the folder is rebuilt from a full snapshot +of the new cluster. Rotating a kubeconfig Secret's *contents* is not a retarget — +it is the same cluster, reached with a fresh credential. + +## Implementation + +The watch manager grew a **cluster context**: the set of things that were previously +Manager-wide singletons and are in fact properties of one cluster. + +```go +type clusterContext struct { + id string // "" for the local cluster, else "//" + catalog *APIResourceCatalog + registry *typeset.Registry + restConfig *rest.Config + configVersion string // the kubeconfig Secret's resourceVersion + dynamicClient dynamic.Interface + discovery apiResourceDiscovery + triggerFactory dynamicinformer.DynamicSharedInformerFactory +} +``` + +- `Manager.clusters map[string]*clusterContext`. A zero-value Manager (unit tests, and + every single-cluster install) creates exactly one, keyed `LocalClusterID`. +- `RefreshAPIResourceCatalog` refreshes every **active** cluster — the local one plus + every cluster some compiled rule points at. It returns the *local* cluster's error + only: a remote that cannot be reached must fail its own `GitTarget`s (through its + unready registry), never the local cluster's. +- `openTargetWatch` / `openTargetList` take the cluster id, which reaches them on the + per-watch `watchFilter`, so each `(GitTarget, GVR, namespace)` watch runs against the + right cluster. +- Watched-type tables resolve a `GitTarget`'s rules against **its own cluster's** type + registry. A CRD installed only on the remote is followable only there — and, more + importantly, a type served only *locally* never resolves for a remote target. + Mirroring the wrong cluster into a folder is worse than mirroring none. + +The source cluster reaches the data plane through the rule store: a +`rulestore.TargetBinding` carries it alongside the branch and path, resolved once when a +rule compiles against its `GitTarget`. + +### The one shared surface: the GVK→GVR lookup + +`WorkerManager` holds a single `typeset.Lookup`, used when scanning the manifests already +in a Git folder to answer "what resource is this document?". Workers are keyed by +(provider, branch) and can be shared by several `GitTarget`s, so this lookup cannot be +per-target without threading it through every pending write. + +It is therefore a **union** over the live cluster registries, consulted in a stable order +(local first, then remote contexts sorted by id). GVK→GVR is derived from the served +resource name, so two clusters serving the same GVK agree on the GVR in every case short +of an outright API-group collision. First answer wins. + +## Credentials and RBAC + +The operator needs `get` on Secrets in the namespaces where `GitTarget`s live — it already +did, for `GitProvider.spec.secretRef` and the SOPS age keys. + +The kubeconfig Secret is read on demand and its contents are **not** retained: a +`rest.Config` is built, the bytes are dropped, and only the Secret's `resourceVersion` is +remembered, so a rotation rebuilds the clients exactly once. No Secret informer is added — +the same reasoning as +[../../future/secret-value-retention-plan.md](../../future/secret-value-retention-plan.md). + +A remote cluster is reached over a network the local one is not, so its clients carry +client-side throttling (`--source-cluster-qps` / `--source-cluster-burst`). + +The `GitTarget` controller reads and parses the kubeconfig before any watch opens against +it, and reports `Validated=False` / `SourceClusterUnreachable` when it cannot. That is a +legibility gate, not a security one: without it, a typo'd Secret name surfaces only as a +stalled data plane and a repeating log line. It deliberately does not *dial* the cluster — +a controller that blocked on a network round trip would stall every other `GitTarget` +behind it. + +## What this unlocks + +- Config and credentials live in the cluster the operator runs in; the watched cluster + holds only the watched resources. +- One reverser can mirror many clusters, because each `GitTarget` carries its own source. +- The watched cluster does not need the `configbutler.ai` CRDs installed at all. diff --git a/docs/design/multi-tenant/gittarget-retarget.md b/docs/design/multi-tenant/gittarget-retarget.md new file mode 100644 index 00000000..7872862a --- /dev/null +++ b/docs/design/multi-tenant/gittarget-retarget.md @@ -0,0 +1,137 @@ +# Moving a GitTarget's destination + +> Status: implemented +> Related: [README.md](README.md), [config-plane-split.md](config-plane-split.md) + +## Problem + +`spec.branch` and `spec.path` were immutable. The reasoning was sound: a +`GitTarget` materializes its watched resources at exactly one (provider, branch, +folder), and letting that move would orphan the old materialization and silently +invalidate the initial-snapshot gate — "a successful snapshot can never be +invalidated by a destination change". + +The cost fell on anyone who ever repoints a target. An operator that stamps a +`GitTarget` with a default branch or folder before the user has chosen one has +frozen the object against that default. The only recovery was to catch the +admission rejection and delete-and-recreate, which every such operator ends up +writing for itself. + +## Shape + +`spec.branch`, `spec.path` and `spec.sourceCluster` are **mutable**. +`spec.providerRef` stays immutable — pointing at a different repository is not a +move, it is a different object. + +The snapshot-gate invariant is preserved by making the destination a thing the +status *observes* rather than a thing the spec cannot change: + +```yaml +status: + observedDestination: # where the content actually is + branch: main + path: clusters/acme + sourceCluster: team-a/acme-kubeconfig/value.yaml # empty for the local cluster + retargetingTo: null # set while a move is in flight + conditions: + - type: Retargeting + status: "False" + reason: DestinationSettled +``` + +The invariant becomes: *a successful snapshot is valid for the destination +recorded in `status.observedDestination`.* When `spec` and +`status.observedDestination` disagree, the snapshot is by definition stale, and +the reverser says so. + +## Lifecycle + +On observing a destination change the controller: + +1. tears the old materialization down **before anything is validated**, and sets + `Retargeting=True` (reason `DestinationChanged`), clearing + `status.lastPushTime`. Teardown must come first: the writer reads `spec.path` + fresh on every write while the branch worker is bound to the branch its event + stream was registered against, so a live event arriving mid-move would + otherwise be written to the *new* path on the *old* branch; +2. re-runs the ordinary `Validated` gate against the **new** destination — the + branch must match the provider's `allowedBranches`, and the new path must not + overlap another `GitTarget` on the same provider+branch. A retarget onto a + conflicting path is refused exactly as a create onto one would be, and the + target stalls with `TargetConflict`. Note that because the teardown in step 1 + already happened, the target then mirrors **nothing** until a free destination + is chosen — it does not fall back to the old one. That is the price of the + ordering, and it is the right price: the alternative is a live event landing at + the new path on the old branch; +3. re-declares against the new destination, which drives a fresh full snapshot + into the new folder; +4. once the new destination reports `GitPathAccepted=True` and streams are + running, writes `status.observedDestination`, sets `Retargeting=False` (reason + `DestinationSettled`), and emits a `Retargeted` event. + +The teardown in step 1 unregisters the `GitTargetEventStream` (releasing the old +branch worker), forgets the declaration, and — the load-bearing part — drops the +per-type watch **resume cursors**. Those are keyed by `GitTarget` UID, and a +retarget keeps the same object; without dropping them the new folder would only +ever receive the changes that happen *after* the move, never the state that +already existed. + +The whole sequence is exactly what a delete-and-recreate did, minus the window in +which the object does not exist. + +It is idempotent per generation: a reconcile that finds the teardown already done +for the current generation skips it and lets the rebuild proceed. A *second* +destination change arriving mid-move bumps the generation, which makes the +recorded teardown stale, so it tears down again rather than quietly continuing to +build the first one. + +### A move that is superseded before it settles + +`status.retargetingTo` names the destination an in-flight move is building. It exists +because a *second* destination change — or a revert — arriving before the first settles +would otherwise orphan a folder nobody named: `observedDestination` still points at the +original, and the intermediate folder the first move had begun writing is left behind +silently. + +When that happens the controller emits a `RetargetSuperseded` event naming the intermediate +folder. Like every folder a retarget leaves, nothing deletes it. + +## The old folder is left alone + +**A retarget never deletes the old folder's files.** Two reasons, either +sufficient: + +- Deleting from Git is the one irreversible thing the reverser can do, and a + destination change is the moment when the operator is least sure of what they + meant. +- The path the target is leaving may already have a new owner — the overlap check + in step 2 only guards the destination it is *moving to*. + +## A mover never evicts an incumbent + +Once `spec.path` is mutable, a `GitTarget`'s creation timestamp stops being a faithful +"who claimed this folder first". The overlap resolver therefore compares **claim strength** +before age: a target is *established* when `status.observedDestination` agrees with its +spec, and *pending* when it is asking to move somewhere (or has never materialized). + +An established claim beats a pending one. So a target retargeting onto an occupied folder is +the newcomer, whatever its age, and loses — it never evicts the target already writing there. +Creation time only breaks ties between two claims of the same strength, exactly as before. + +The old folder becomes ordinary, unmanaged Git content. The controller emits a +`Retargeted` event naming the abandoned `branch:path` so the operator can `git rm` +it deliberately, and repeats it in the `Retargeting=False` condition message — +which is then left alone by later reconciles, because that message is the only +place in status where the abandoned folder is named. + +If the new path is a **subfolder or parent** of the old one, the overlap check in +step 2 fails against the target's own previous location only if another +`GitTarget` claimed it in the meantime; a target never conflicts with itself. +The files under the old path that are not under the new one simply stay. + +## What still requires delete-and-recreate + +Changing `spec.providerRef`. The materialization lives in a different repository; +there is nothing to move and nothing to observe. The CEL message says so: + +> `spec.providerRef is immutable; delete and recreate the GitTarget to change its repository (spec.branch and spec.path are mutable — the controller retargets, see docs/design/multi-tenant/gittarget-retarget.md)` diff --git a/docs/design/multi-tenant/identity-write-exclusion.md b/docs/design/multi-tenant/identity-write-exclusion.md new file mode 100644 index 00000000..c993158a --- /dev/null +++ b/docs/design/multi-tenant/identity-write-exclusion.md @@ -0,0 +1,135 @@ +# Identity-based write exclusion + +> Status: implemented +> Related: [README.md](README.md), [../../configuration.md](../../configuration.md) + +## Problem + +A reverser that mirrors a branch a GitOps forward leg (Flux, Argo CD) also +applies will commit that forward leg's own applies. Nothing in the rule model +could express *"this write is not interesting"*: a `ResourceRule` filtered on +type and operation, never on **who wrote**. + +## Shape + +Two optional fields on `ResourceRule` (`WatchRule`) and `ClusterResourceRule` +(`ClusterWatchRule`): + +```yaml +rules: + - resources: ["configmaps"] + excludeFieldManagers: ["kustomize-controller"] # from watch state alone + excludeUsers: ["system:serviceaccount:flux-system:kustomize-controller"] +``` + +`excludeFieldManagers` is the stronger form and the one to reach for. It reads +`metadata.managedFields` off the live object, so it needs no audit fact, cannot +race the attribution grace window, and works in configured-author mode. + +`excludeUsers` matches the identity the audit webhook attributed the write to +(the impersonated user when impersonation is in play, otherwise the +authenticated user). It therefore requires `--author-attribution` and a working +audit webhook. + +## Semantics + +### Rules OR, exclusions veto within a rule + +Rules are a logical OR: a resource matching **any** rule is watched. An +exclusion is a negative clause **within** one rule, not a global filter. Given + +```yaml +rules: + - resources: ["configmaps"] + excludeFieldManagers: ["kustomize-controller"] + - resources: ["configmaps"] +``` + +a write by `kustomize-controller` is still mirrored, because the second rule +admits it. Formally, for an event *e* with operation *op*, last writer *fm*, +and attributed user *u*, over the set *S* of compiled resource rules that select +the event's type and namespace: + +> route *e* ⟺ ∃ *r* ∈ *S* : *r*.operations matches *op* ∧ *fm* ∉ *r*.excludeFieldManagers ∧ *u* ∉ *r*.excludeUsers + +This is the only composition that keeps `rules` an OR. + +### "The last writer" is the newest managedFields entry + +`excludeFieldManagers` compares against the managers of the `managedFields` +entries carrying the **newest** `time`. If several entries tie on that +timestamp, the event is excluded only when **every** tied manager is excluded — +when in doubt, commit. An object with no `managedFields` at all is never +excluded. + +### DELETE is never excluded by field manager + +`managedFields` names who last *wrote* an object, not who deleted it. A human +deleting a Flux-managed ConfigMap would otherwise be silently ignored, which is +exactly the failure a label selector has. So `excludeFieldManagers` is not +evaluated for `DELETE`; `excludeUsers`, which reads the audit fact for the +delete itself, is. + +### Unresolved identity fails open + +If `excludeUsers` is set but the author cannot be attributed — attribution +disabled, or the grace window expired with no matching fact — the event is +**routed**, not dropped. Dropping a change because we failed to identify its +author would silently lose a human's edit. This is the reason to prefer +`excludeFieldManagers`. + +### Exclusion suppresses *writes*, not *state* + +An exclusion drops a live watch event. It does **not** remove the object from +the mark-and-sweep desired set that a replay or resync computes: an object a +GitOps tool manages is still an object the GitTarget mirrors, and dropping it +from the desired set would make the sweep delete its file from Git. + +The practical consequence: after the forward leg changes an object, Git keeps the +content the human last wrote, and the cluster carries the forward leg's. Git catches +up the next time that object is mirrored for any *other* reason — a replay of its +type (reconnect, restart, rule change), or the next non-excluded write to the same +object, which commits the object's whole current state including the forward leg's +change. Either way it is one idempotent write that does not re-trigger the forward +leg, because the content it commits is what the forward leg already applied. + +So `excludeFieldManagers` is not "this manager's changes never reach Git". It is +"this manager's writes never *cause* a commit". The loop is broken; the +reconciliation is not. Field-level attribution of Git content to the manager that +produced it would be a different feature entirely. + +### What is *not* a change to mirror + +`internal/sanitize` already strips a GitOps tool's own bookkeeping from Git +content: labels and annotations under `kustomize.toolkit.fluxcd.io/`, `kro.run/` +and `applyset.kubernetes.io/`, along with `managedFields`, `uid`, +`resourceVersion` and friends. + +So an apply that *only* stamps those labels produces no Git-writable change at +all, and the content dedup drops it before any exclusion is consulted. An +exclusion decides the case where the forward leg's apply changes real content — +which is what happens whenever Git and the cluster disagree about a managed field. + +This is worth knowing when reading +`gitopsreverser_watch_events_excluded_total`: a zero rate on a `GitTarget` paired +with a forward leg does not by itself prove the exclusion is misconfigured. It may +simply mean the forward leg has not written anything the operator would have +mirrored. + +## Where it is enforced + +`internal/watch/target_watch.go`, in `routeLiveTargetWatchEvent`, alongside the +existing operation filter and before the content dedup. The raw watch object is +still un-sanitized there, so `managedFields` is available (`sanitize.Sanitize` +strips it when building the Git event). + +Author attribution normally runs *after* the dedup, so a status-only update does +not pay the 3s grace window. When — and only when — some selecting rule declares +`excludeUsers`, attribution is resolved early so the exclusion can see the +identity, and the result is reused rather than looked up twice. + +## Observability + +`gitopsreverser_watch_events_excluded_total{gittarget_namespace, gittarget_name, +group, resource, reason}` counts dropped events, with `reason` one of +`field_manager` or `user`. diff --git a/docs/security-model.md b/docs/security-model.md index 8e9c0b4d..e69e5fdf 100644 --- a/docs/security-model.md +++ b/docs/security-model.md @@ -33,9 +33,46 @@ The controller does not need write access to watched resources. Its only write t |---|---| | Git credentials Secret | Grants push access to your repository. | | SOPS/age key material | Decrypts (and the public key encrypts) Secret data written to Git. | +| Source-cluster kubeconfig Secret | Grants the operator's read access to a remote cluster. | | Redis/Valkey queue | Buffers decoded audit events in transit; not an audit archive. | | Audit ingress (`/audit-webhook`) | Accepts audit traffic; protected by mutual TLS via cert-manager. | | Generated Secret material | Signing keys and generated age keys live in cluster Secrets. | +| `assert-author` RBAC verb | Lets a holder write any author into a repository's Git history. | + +## Keeping Git credentials off the cluster you watch + +Historically the operator built one client and used it for both jobs: reading its own CRs and the Git +credentials Secret, and watching the resources it mirrors. Nothing chose that; it fell out of having +one kubeconfig. The consequence is worth stating plainly: because `GitProvider.spec.secretRef` is a +same-namespace reference, the Git write credential — often scoped far more broadly than the one +repository a `GitTarget` names — had to live **on the cluster being watched**, one RBAC rule away +from whoever can read Secrets in that namespace. + +`GitTarget.spec.sourceCluster` separates them. The operator reads its own CRs and credentials from the +cluster it runs in (**the config plane**) and watches whichever cluster each `GitTarget` names. The +watched cluster then holds only the watched resources — no Git credential, and not even the +`configbutler.ai` CRDs. See +[`configuration.md`](configuration.md#mirroring-a-remote-cluster-specsourcecluster). + +The source-cluster kubeconfig Secret is read on demand from the config plane, parsed into a +`rest.Config`, and its bytes are dropped; only the Secret's `resourceVersion` is retained, so a +rotation rebuilds the clients exactly once. No Secret informer is started. + +## Asserting a commit author is a privilege + +`CommitRequest.spec.author` lets a trusted client name the human a commit is for, instead of deriving +them from an apiserver audit fact. It is gated by the `assert-author` verb on the named `GitTarget` +(checked by a `SubjectAccessReview` at admission, and re-verified by the controller against the +recorded verdict, so the check does not depend on the webhook's `failurePolicy`). + +**Treat `assert-author` exactly like `impersonate`.** The asserted `name` and `email` are free text +and are not verified against any real identity: they are what the trusted control plane says they are. +Granting the verb grants the ability to write any author into that repository's history. Scope grants +with `resourceNames` to the specific `GitTarget`s a caller owns. + +The commit's **committer** is always the operator's configured identity, and commit signing (when +configured) signs as the committer — so a reader can always tell a commit was made by the reverser on +someone's behalf, whoever the author header names. ## Secret data the controller writes to Git diff --git a/internal/controller/clusterwatchrule_controller.go b/internal/controller/clusterwatchrule_controller.go index 26c6fa1a..632cb255 100644 --- a/internal/controller/clusterwatchrule_controller.go +++ b/internal/controller/clusterwatchrule_controller.go @@ -198,14 +198,8 @@ func (r *ClusterWatchRuleReconciler) reconcileClusterWatchRuleViaTarget( // Ready check // TODO: Check GitProvider readiness - // Add rule to store with GitTarget reference and resolved values - r.RuleStore.AddOrUpdateClusterWatchRule( - *clusterRule, - target.Name, targetNS, // GitTarget reference - provider.Name, providerNS, // GitProvider reference - target.Spec.Branch, - target.Spec.Path, - ) + // Add rule to store with its resolved GitTarget binding (destination + source cluster). + r.RuleStore.AddOrUpdateClusterWatchRule(*clusterRule, rulestore.NewTargetBinding(target, provider)) // Trigger WatchManager reconciliation for new/updated rule if r.WatchManager != nil { diff --git a/internal/controller/commitrequest_asserted_author_test.go b/internal/controller/commitrequest_asserted_author_test.go new file mode 100644 index 00000000..540d88fe --- /dev/null +++ b/internal/controller/commitrequest_asserted_author_test.go @@ -0,0 +1,141 @@ +// SPDX-License-Identifier: Apache-2.0 + +package controller + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + configv1alpha3 "github.com/ConfigButler/gitops-reverser/api/v1alpha3" + "github.com/ConfigButler/gitops-reverser/internal/git" + "github.com/ConfigButler/gitops-reverser/internal/queue" +) + +func assertingCommitRequest(name string) *configv1alpha3.CommitRequest { + cr := newCommitRequest(name) + cr.Spec.Author = &configv1alpha3.CommitAuthor{Name: "Ada Lovelace", Email: "ada@example.com"} + return cr +} + +// authorizedSubmitter is the admission record the webhook writes after a SubjectAccessReview +// said the submitter holds assert-author on the GitTarget. +func authorizedSubmitter() *fakeAuthorLookup { + return &fakeAuthorLookup{ + author: queue.CommandAuthor{Author: "gitops-api", AssertAuthorAllowed: true}, + found: true, + } +} + +func TestCommitRequestReconcile_AssertedAuthorSignsTheCommit(t *testing.T) { + cr := assertingCommitRequest("save-asserted") + c := newCommitRequestClient(t, nil, cr) + f := &fakeFinalizer{ + result: git.FinalizeResult{Outcome: git.FinalizeCommitted, SHA: "abc123", Branch: "main"}, + resolved: true, + } + r := &CommitRequestReconciler{Client: c, APIReader: c, Finalizer: f, AuthorLookup: authorizedSubmitter()} + + reconcileCommitRequest(t, r, "save-asserted") + + require.Len(t, f.calls, 1) + require.NotNil(t, f.calls[0].AssertedAuthor, "the asserted identity must reach the worker") + assert.Equal(t, "Ada Lovelace", f.calls[0].AssertedAuthor.Username) + assert.Equal(t, "Ada Lovelace", f.calls[0].AssertedAuthor.DisplayName) + assert.Equal(t, "ada@example.com", f.calls[0].AssertedAuthor.Email) + assert.Equal(t, "gitops-api", f.calls[0].Author, "the submitter is still carried, for logging") + + got := fetchCommitRequest(t, c, "save-asserted") + requireCondition(t, got, ConditionTypeAuthorAttributed, metav1.ConditionTrue, crReasonAuthorAsserted) +} + +// The webhook is failurePolicy: Ignore, so a guard living only there would be bypassable +// by taking the webhook down. The controller is the real gate. +func TestCommitRequestReconcile_UnauthorizedAssertionIsIgnored(t *testing.T) { + tests := map[string]*fakeAuthorLookup{ + "no admission record at all": {found: false}, + "record without the verdict": { + author: queue.CommandAuthor{Author: "mallory", AssertAuthorAllowed: false}, + found: true, + }, + } + + for name, lookup := range tests { + t.Run(name, func(t *testing.T) { + cr := assertingCommitRequest("save-unverified") + c := newCommitRequestClient(t, nil, cr) + f := &fakeFinalizer{ + result: git.FinalizeResult{Outcome: git.FinalizeCommitted, SHA: "abc123", Branch: "main"}, + resolved: true, + } + r := &CommitRequestReconciler{Client: c, APIReader: c, Finalizer: f, AuthorLookup: lookup} + + reconcileCommitRequest(t, r, "save-unverified") + + require.Len(t, f.calls, 1) + assert.Nil(t, f.calls[0].AssertedAuthor, "an unverified assertion never reaches the worker") + assert.Empty(t, f.calls[0].Author, "and it does not silently fall back to the submitter either") + + got := fetchCommitRequest(t, c, "save-unverified") + requireCondition(t, got, ConditionTypeAuthorAttributed, metav1.ConditionFalse, + crReasonAuthorAssertionUnverified) + // Not a failure: the commit is still made, authored by the committer. + requireCondition(t, got, ConditionTypeReady, metav1.ConditionTrue, crReasonCommitted) + }) + } +} + +// Without the webhook there is no AuthorLookup at all, so an assertion cannot be backed. +func TestCommitRequestReconcile_AssertionWithoutAuthorLookupIsIgnored(t *testing.T) { + cr := assertingCommitRequest("save-no-lookup") + c := newCommitRequestClient(t, nil, cr) + f := &fakeFinalizer{ + result: git.FinalizeResult{Outcome: git.FinalizeCommitted, SHA: "abc123", Branch: "main"}, + resolved: true, + } + r := &CommitRequestReconciler{Client: c, APIReader: c, Finalizer: f} + + reconcileCommitRequest(t, r, "save-no-lookup") + + require.Len(t, f.calls, 1) + assert.Nil(t, f.calls[0].AssertedAuthor) + + got := fetchCommitRequest(t, c, "save-no-lookup") + requireCondition(t, got, ConditionTypeAuthorAttributed, metav1.ConditionFalse, + crReasonAuthorAssertionUnverified) +} + +// A CommitRequest with no spec.author keeps the pre-existing behavior exactly. +func TestCommitRequestReconcile_NoAssertionKeepsAdmissionAttribution(t *testing.T) { + cr := newCommitRequest("save-plain") + c := newCommitRequestClient(t, nil, cr) + f := &fakeFinalizer{ + result: git.FinalizeResult{Outcome: git.FinalizeCommitted, SHA: "abc123", Branch: "main"}, + resolved: true, + } + r := &CommitRequestReconciler{Client: c, APIReader: c, Finalizer: f, AuthorLookup: attributedAlice()} + + reconcileCommitRequest(t, r, "save-plain") + + require.Len(t, f.calls, 1) + assert.Nil(t, f.calls[0].AssertedAuthor) + assert.Equal(t, "alice", f.calls[0].Author) + + got := fetchCommitRequest(t, c, "save-plain") + requireCondition(t, got, ConditionTypeAuthorAttributed, metav1.ConditionTrue, crReasonAttributedFromAdmission) +} + +func TestAssertedUserInfo(t *testing.T) { + t.Parallel() + + full := assertedUserInfo(&configv1alpha3.CommitAuthor{Name: "Ada Lovelace", Email: "ada@example.com"}) + assert.Equal(t, git.UserInfo{Username: "Ada Lovelace", DisplayName: "Ada Lovelace", Email: "ada@example.com"}, full) + + // An omitted email is derived downstream, exactly as for an audit-attributed author + // whose token carried no email claim. + noEmail := assertedUserInfo(&configv1alpha3.CommitAuthor{Name: "Ada Lovelace"}) + assert.Empty(t, noEmail.Email) + assert.Equal(t, "Ada Lovelace", noEmail.Username) +} diff --git a/internal/controller/commitrequest_controller.go b/internal/controller/commitrequest_controller.go index fdd85748..db35984c 100644 --- a/internal/controller/commitrequest_controller.go +++ b/internal/controller/commitrequest_controller.go @@ -107,6 +107,12 @@ type CommitRequestReconciler struct { // +kubebuilder:rbac:groups=configbutler.ai,resources=commitrequests,verbs=get;list;watch // +kubebuilder:rbac:groups=configbutler.ai,resources=commitrequests/status,verbs=get;update;patch +// The validate-operator-types admission webhook issues a SubjectAccessReview to decide +// whether a CommitRequest's submitter may assert a commit author (the assert-author verb +// on the referenced GitTarget). Without this the review cannot run and every assertion is +// denied. See docs/design/multi-tenant/asserted-commit-author.md. +// +kubebuilder:rbac:groups=authorization.k8s.io,resources=subjectaccessreviews,verbs=create + // Reconcile advances one CommitRequest through attribute → attach + poll → // terminal status. With MaxConcurrentReconciles=1 concurrent CommitRequests are // serialized by construction, and the worker keys attaches by request identity so @@ -126,9 +132,9 @@ func (r *CommitRequestReconciler) Reconcile(ctx context.Context, req ctrl.Reques } // 1. ATTRIBUTE: settle the commit author synchronously (present-or-never, §2). - // A hit names the admission submitter; a miss is the configured committer. Either - // way the decision is final — there is no wait and no requeue for the author. - author, attribution := r.attributeAuthor(ctx, commitRequest) + // An authorized spec.author wins; else the admission submitter; else the configured + // committer. Either way the decision is final — no wait, no requeue for the author. + author, assertedAuthor, attribution := r.attributeAuthor(ctx, commitRequest) // First sight: stamp the still-running conditions so the object reports its // progress (kstatus InProgress) and AuthorAttributed is settled immediately. A @@ -149,6 +155,7 @@ func (r *CommitRequestReconciler) Reconcile(ctx context.Context, req ctrl.Reques Name: commitRequest.Name, UID: string(commitRequest.UID), Author: author.Author, + AssertedAuthor: assertedAuthor, GitTargetName: commitRequest.Spec.TargetRef.Name, GitTargetNamespace: commitRequest.Namespace, Message: capCommitRequestMessage(commitRequest.Spec.Message), @@ -186,27 +193,69 @@ func (r *CommitRequestReconciler) Reconcile(ctx context.Context, req ctrl.Reques // the configured committer immediately. The miss case is final — the record is written // before the object is visible, so there is no asynchronous arrival to wait for. // +// spec.author overrides both, but ONLY when the admission record for this object carries +// the authorized verdict. That is the real gate: the webhook is failurePolicy: Ignore, so +// a guard living only there would be bypassable by taking the webhook down. No record — +// off, bypassed, or no Redis — means the assertion is ignored, not honored. +// // The lookup result is logged at Info: it is the counterpart to the admission handler's // "recorded command author" line, so a hit/miss pair makes the whole capture→read path // legible (the first thing to check when a CommitRequest commits as the committer). func (r *CommitRequestReconciler) attributeAuthor( ctx context.Context, commitRequest *configbutleraiv1alpha3.CommitRequest, -) (queue.CommandAuthor, commitRequestAttribution) { +) (queue.CommandAuthor, *git.UserInfo, commitRequestAttribution) { log := logf.FromContext(ctx).WithName("CommitRequestReconciler") + key := client.ObjectKeyFromObject(commitRequest) + if r.AuthorLookup == nil { + if commitRequest.Spec.Author != nil { + log.Info("spec.author ignored: command-author lookup disabled "+ + "(validate-operator-types webhook off or no Redis); committing as committer", + "name", key, "uid", commitRequest.UID) + return queue.CommandAuthor{}, nil, attributionAssertionUnverified + } log.Info("command-author lookup disabled (validate-operator-types webhook off); committing as committer", - "name", client.ObjectKeyFromObject(commitRequest), "uid", commitRequest.UID) - return queue.CommandAuthor{}, attributionCommitter + "name", key, "uid", commitRequest.UID) + return queue.CommandAuthor{}, nil, attributionCommitter + } + + author, found := r.AuthorLookup.LookupCommandAuthor(ctx, commitRequest.UID) + + if commitRequest.Spec.Author != nil { + if !found || !author.AssertAuthorAllowed { + log.Info("spec.author ignored: no authorized admission record backs the assertion; "+ + "committing as committer", "name", key, "uid", commitRequest.UID, "recordFound", found) + return queue.CommandAuthor{}, nil, attributionAssertionUnverified + } + asserted := assertedUserInfo(commitRequest.Spec.Author) + log.Info("commit author asserted by an authorized CommitRequest", + "name", key, "uid", commitRequest.UID, + "submitter", author.Author, "assertedAuthor", asserted.Username) + return author, &asserted, attributionAsserted } - if author, ok := r.AuthorLookup.LookupCommandAuthor(ctx, commitRequest.UID); ok { + + if found { log.Info("command author resolved from admission record", - "name", client.ObjectKeyFromObject(commitRequest), "uid", commitRequest.UID, "author", author.Author) - return author, attributionFromAdmission + "name", key, "uid", commitRequest.UID, "author", author.Author) + return author, nil, attributionFromAdmission } log.Info("no admission command-author record found; committing as committer", - "name", client.ObjectKeyFromObject(commitRequest), "uid", commitRequest.UID) - return queue.CommandAuthor{}, attributionCommitter + "name", key, "uid", commitRequest.UID) + return queue.CommandAuthor{}, nil, attributionCommitter +} + +// assertedUserInfo maps an asserted CommitAuthor onto the git identity that signs the +// commit. Name lands in both Username and DisplayName: DisplayName is what the signature +// header prefers, and Username is the fallback when the name carries a character a +// signature header cannot hold. An empty Email is derived downstream, exactly as it is for +// an audit-attributed author whose token had no email claim. +func assertedUserInfo(author *configbutleraiv1alpha3.CommitAuthor) git.UserInfo { + return git.UserInfo{ + Username: author.Name, + DisplayName: author.Name, + Email: author.Email, + } } // recordCloseDelayWait makes the post-attribution wait — the closeDelaySeconds diff --git a/internal/controller/commitrequest_finalize.go b/internal/controller/commitrequest_finalize.go index 7d197b9e..b4f52661 100644 --- a/internal/controller/commitrequest_finalize.go +++ b/internal/controller/commitrequest_finalize.go @@ -19,16 +19,18 @@ const commitRequestMessageMaxBytes = 1024 // CommitRequest condition reasons (CamelCase tokens surfaced on status.conditions). const ( - crReasonWaitingForCloseDelay = "WaitingForCloseDelay" - crReasonCommitted = "Committed" - crReasonNoWindowInGrace = "NoWindowInGrace" - crReasonWindowMismatch = "WindowMismatch" - crReasonAlreadyPresent = "AlreadyPresent" - crReasonFinalizeFailed = "FinalizeFailed" - crReasonUnexpectedOutcome = "UnexpectedOutcome" - crReasonAttributedFromAdmission = "AttributedFromAdmission" - crReasonCommitterFallback = "CommitterFallback" - crReasonPushed = "Pushed" + crReasonWaitingForCloseDelay = "WaitingForCloseDelay" + crReasonCommitted = "Committed" + crReasonNoWindowInGrace = "NoWindowInGrace" + crReasonWindowMismatch = "WindowMismatch" + crReasonAlreadyPresent = "AlreadyPresent" + crReasonFinalizeFailed = "FinalizeFailed" + crReasonUnexpectedOutcome = "UnexpectedOutcome" + crReasonAttributedFromAdmission = "AttributedFromAdmission" + crReasonAuthorAsserted = "AuthorAsserted" + crReasonAuthorAssertionUnverified = "AuthorAssertionUnverified" + crReasonCommitterFallback = "CommitterFallback" + crReasonPushed = "Pushed" ) // noWindowInGraceMessage is the prose for a NoWindowInGrace outcome: the grace @@ -57,6 +59,14 @@ const ( // validate-operator-types webhook is not configured (or did not record one) — so the // commit is authored by the configured committer. attributionCommitter + // attributionAsserted means spec.author named the commit author and an admission + // record confirms the requester held the assert-author verb on the GitTarget. + attributionAsserted + // attributionAssertionUnverified means spec.author was set but no authorized + // admission record backs it: the webhook is off, was bypassed, or Redis is not + // configured. The assertion is ignored and the commit is authored by the committer — + // fail-closed, and independent of the webhook's failurePolicy. + attributionAssertionUnverified ) // setCommitRequestCondition upserts a condition keyed by type, stamping it with @@ -106,6 +116,16 @@ func markCommitRequestWaitingForCloseDelay(cr *configv1alpha3.CommitRequest, att // affect Ready — it is the honest signal that no admission author record was found. func setCommitRequestAttributed(cr *configv1alpha3.CommitRequest, attribution commitRequestAttribution) { switch attribution { + case attributionAsserted: + setCommitRequestCondition(cr, ConditionTypeAuthorAttributed, metav1.ConditionTrue, + crReasonAuthorAsserted, + "spec.author named the commit author; the requester holds the assert-author verb on the GitTarget") + case attributionAssertionUnverified: + setCommitRequestCondition(cr, ConditionTypeAuthorAttributed, metav1.ConditionFalse, + crReasonAuthorAssertionUnverified, + "spec.author was set but no authorized admission record backs it (the validate-operator-types "+ + "webhook is not configured, was bypassed, or Redis is not set); the assertion was ignored "+ + "and the commit is authored by the configured committer") case attributionFromAdmission: setCommitRequestCondition(cr, ConditionTypeAuthorAttributed, metav1.ConditionTrue, crReasonAttributedFromAdmission, diff --git a/internal/controller/gittarget_controller.go b/internal/controller/gittarget_controller.go index 96a757fd..cc0efa86 100644 --- a/internal/controller/gittarget_controller.go +++ b/internal/controller/gittarget_controller.go @@ -18,6 +18,7 @@ import ( "k8s.io/apimachinery/pkg/runtime" k8stypes "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/tools/record" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" @@ -102,8 +103,14 @@ type GitTargetReconciler struct { Scheme *runtime.Scheme WorkerManager *git.WorkerManager EventRouter *watch.EventRouter + + // Recorder emits the Retargeted event when a destination change completes, naming the + // folder that was abandoned — the one thing a retarget leaves behind and nothing + // deletes. Nil in tests and standalone reconcilers. + Recorder record.EventRecorder } +// +kubebuilder:rbac:groups="",resources=events,verbs=create;patch // +kubebuilder:rbac:groups=configbutler.ai,resources=gittargets,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=configbutler.ai,resources=gittargets/status,verbs=get;update;patch // +kubebuilder:rbac:groups=configbutler.ai,resources=gitproviders,verbs=get;list;watch @@ -124,6 +131,17 @@ func (r *GitTargetReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( target.Status.LastReconcileTime = metav1.Now() gitPathWasRefused := conditionIsFalse(target.Status.Conditions, GitTargetConditionGitPathAccepted) + // A destination change is torn down before anything else runs. The writer reads + // spec.path fresh per write while the branch worker is bound to the branch the stream + // was registered against, so a live event arriving between the spec change and a + // completed retarget would land at the new path on the old branch. Nothing may be + // written until the new destination has passed its own Validated gate. + if destinationMoved(&target) && !retargetAlreadyTornDown(&target) { + if err := r.beginRetarget(ctx, &target, log); err != nil { + return ctrl.Result{}, err + } + } + providerNS := target.Namespace validated, validationMsg, validationResult, validationErr := r.evaluateValidatedGate(ctx, &target, providerNS) if validationErr != nil { @@ -139,6 +157,7 @@ func (r *GitTargetReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( ) r.setBlockedDataPlane(&target) r.setGitPathAcceptedUnknown(&target, "Blocked by Validated=False") + r.markRetargetingUnknown(&target, "Blocked by Validated=False") r.setStalledConditions( &target, GitTargetReadyReasonValidationFailed, @@ -160,6 +179,7 @@ func (r *GitTargetReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( if !encryptionReady { r.setBlockedDataPlane(&target) r.setGitPathAcceptedUnknown(&target, "Blocked by EncryptionConfigured=False") + r.markRetargetingUnknown(&target, "Blocked by EncryptionConfigured=False") r.setStalledConditions( &target, GitTargetReadyReasonEncryptionNotConfigured, @@ -178,6 +198,7 @@ func (r *GitTargetReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( if !wired { r.setBlockedDataPlane(&target) r.setGitPathAcceptedUnknown(&target, "Blocked by worker wiring failure") + r.markRetargetingUnknown(&target, "Blocked by worker wiring failure") r.setStalledConditions( &target, GitTargetReadyReasonWorkerUnavailable, @@ -200,14 +221,25 @@ func (r *GitTargetReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( } if r.EventRouter != nil && r.EventRouter.WatchManager != nil { gitDest := types.NewResourceReference(target.Name, target.Namespace).WithUID(string(target.UID)) - if declareErr := r.EventRouter.WatchManager.DeclareForGitTarget( - ctx, - gitDest, - gitPathWasRefused, - ); declareErr != nil { - log.V(1).Info("stream declaration skipped; surface not observable", - "gitDest", gitDest.String(), "err", declareErr.Error()) + switch { + case !r.sourceClusterRulesCaughtUp(&target, gitDest): + // The rules still name the previous source cluster: a rule recompiles when its + // GitTarget's generation bumps, and this reconcile can win that race. Declaring now + // would open watches against the OLD cluster and write its objects into this + // GitTarget's folder. Wait one requeue instead. + log.Info("waiting for WatchRules to recompile against the new source cluster", + "gitDest", gitDest.String(), "sourceCluster", describeSourceCluster(target.SourceClusterID())) streamsSettling = true + default: + if declareErr := r.EventRouter.WatchManager.DeclareForGitTarget( + ctx, + gitDest, + gitPathWasRefused, + ); declareErr != nil { + log.V(1).Info("stream declaration skipped; surface not observable", + "gitDest", gitDest.String(), "err", declareErr.Error()) + streamsSettling = true + } } streams = r.EventRouter.WatchManager.StreamSummaryForGitTarget(gitDest) gitPath = r.EventRouter.WatchManager.GitPathAcceptanceForGitTarget(gitDest) @@ -221,6 +253,13 @@ func (r *GitTargetReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( r.applyDataPlaneConditions(&target, streams, gitPath) + // The destination is recorded only once a snapshot has actually landed there and the + // path was accepted. That is what keeps the invariant honest: a successful snapshot is + // valid for the destination named by status.observedDestination, and nothing else. + if !streamsSettling { + r.settleDestination(&target, log) + } + if err := r.updateStatusWithRetry(ctx, &target); err != nil { return ctrl.Result{}, err } @@ -265,6 +304,20 @@ func (r *GitTargetReconciler) evaluateValidatedGate( return false, fmt.Sprintf("Validated gate failed: %s", GitTargetReasonInvalidConfig), nil, nil } + // The source-cluster kubeconfig is read from the config plane before any watch opens + // against it, so a typo'd Secret name says so instead of stalling the data plane. + if clusterOK, clusterMsg := r.validateSourceCluster(ctx, target); !clusterOK { + r.setCondition( + target, + GitTargetConditionValidated, + metav1.ConditionFalse, + GitTargetReasonSourceClusterUnreachable, + clusterMsg, + ) + result := ctrl.Result{RequeueAfter: RequeueSteadyInterval} + return false, fmt.Sprintf("Validated gate failed: %s", GitTargetReasonSourceClusterUnreachable), &result, nil + } + r.setCondition( target, GitTargetConditionValidated, diff --git a/internal/controller/gittarget_immutability_test.go b/internal/controller/gittarget_immutability_test.go index 65cf1b35..2a69f7c9 100644 --- a/internal/controller/gittarget_immutability_test.go +++ b/internal/controller/gittarget_immutability_test.go @@ -14,19 +14,19 @@ import ( configbutleraiv1alpha3 "github.com/ConfigButler/gitops-reverser/api/v1alpha3" ) -// A GitTarget's destination — providerRef, branch, path — is immutable: changing where -// it materializes would orphan the old materialization, so the API server rejects the -// change (CEL transition rules) and a relocation is a delete + recreate. This replaces -// the alternative of reconciling a destination move (which would need a generation-aware -// snapshot gate and worker rebinding); making it immutable removes that whole class of -// bug instead of handling it. -var _ = Describe("GitTarget Destination Immutability", func() { +// A GitTarget's repository — providerRef — is immutable: pointing at a different +// repository is not a move, it is a different object, with nothing to migrate and nothing +// to observe. Its branch and path ARE mutable: changing either is a supported retarget, +// which the controller drives (see gittarget_retarget.go and +// docs/design/multi-tenant/gittarget-retarget.md). The snapshot gate is preserved by +// status.observedDestination rather than by freezing the spec. +var _ = Describe("GitTarget Destination Mutability", func() { const ( timeout = time.Second * 10 interval = time.Millisecond * 250 ) - It("rejects changes to providerRef, branch, and path but allows a no-op update", func() { + It("rejects a providerRef change, accepts branch and path changes, and allows a no-op update", func() { ctx := context.Background() key := types.NamespacedName{Name: "immutable-target", Namespace: "default"} @@ -49,29 +49,30 @@ var _ = Describe("GitTarget Destination Immutability", func() { g.Expect(k8sClient.Update(ctx, current)).To(Succeed()) }, timeout, interval).Should(Succeed()) - // Each destination field is immutable. Eventually loops past any optimistic-lock - // conflict from a concurrent status write so the assertion lands on the real - // immutability rejection, not a transient 409. - expectImmutable := func(mutate func(*configbutleraiv1alpha3.GitTarget), wantMsg string) { + // Eventually loops past any optimistic-lock conflict from a concurrent status write + // so each assertion lands on the real API-server verdict, not a transient 409. + expectMutable := func(mutate func(*configbutleraiv1alpha3.GitTarget)) { Eventually(func(g Gomega) { current := &configbutleraiv1alpha3.GitTarget{} g.Expect(k8sClient.Get(ctx, key, current)).To(Succeed()) mutate(current) - err := k8sClient.Update(ctx, current) - g.Expect(err).To(HaveOccurred()) - g.Expect(err.Error()).To(ContainSubstring(wantMsg)) + g.Expect(k8sClient.Update(ctx, current)).To(Succeed()) }, timeout, interval).Should(Succeed()) } - expectImmutable(func(gt *configbutleraiv1alpha3.GitTarget) { - gt.Spec.Path = "moved" - }, "spec.path is immutable") - expectImmutable(func(gt *configbutleraiv1alpha3.GitTarget) { - gt.Spec.Branch = "develop" - }, "spec.branch is immutable") - expectImmutable(func(gt *configbutleraiv1alpha3.GitTarget) { - gt.Spec.ProviderRef.Name = "prov-b" - }, "spec.providerRef is immutable") + expectMutable(func(gt *configbutleraiv1alpha3.GitTarget) { gt.Spec.Path = "moved" }) + expectMutable(func(gt *configbutleraiv1alpha3.GitTarget) { gt.Spec.Branch = "develop" }) + + Eventually(func(g Gomega) { + current := &configbutleraiv1alpha3.GitTarget{} + g.Expect(k8sClient.Get(ctx, key, current)).To(Succeed()) + current.Spec.ProviderRef.Name = "prov-b" + err := k8sClient.Update(ctx, current) + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring("spec.providerRef is immutable")) + // The message must point at the supported move, not just refuse. + g.Expect(err.Error()).To(ContainSubstring("spec.branch and spec.path are mutable")) + }, timeout, interval).Should(Succeed()) }) It("requires a non-empty path: rejects an omitted or empty path but allows an explicit \".\" root", func() { diff --git a/internal/controller/gittarget_path_overlap.go b/internal/controller/gittarget_path_overlap.go index de556847..4a3f74c4 100644 --- a/internal/controller/gittarget_path_overlap.go +++ b/internal/controller/gittarget_path_overlap.go @@ -51,13 +51,28 @@ func gitTargetPathsOverlap(a, b string) bool { } // gitTargetLosesConflict reports whether target should lose an overlap conflict -// against existing. The later-created target loses so the earlier owner keeps its -// folder. When both carry the same creationTimestamp (the API server stamps at -// second precision, so concurrent applies can tie) the loser is chosen -// deterministically by identity — otherwise neither would lose and both could go -// Ready over the same subtree. Both targets share a namespace here, so the -// namespace/name key is unique and stable across every reconcile. +// against existing, so that every materialized folder keeps exactly one owner. +// +// An ESTABLISHED claim beats a PENDING one. A target is established when its +// materialization already lives where its spec says (status.observedDestination +// agrees with spec); it is pending when it is asking to move somewhere, or has never +// materialized at all. This is the rule that survives spec.path becoming mutable: a +// target that retargets ONTO an occupied folder is the newcomer, whatever its age, and +// it must not evict the incumbent that is already writing there. +// +// Creation time only breaks a tie between two claims of the same strength — two fresh +// targets racing to claim a folder, as before. When both carry the same +// creationTimestamp (the API server stamps at second precision, so concurrent applies +// can tie) the loser is chosen deterministically by identity, otherwise neither would +// lose and both could go Ready over the same subtree. Both targets share a namespace +// here, so the namespace/name key is unique and stable across every reconcile. func gitTargetLosesConflict(target, existing *configbutleraiv1alpha3.GitTarget) bool { + targetEstablished := gitTargetIsEstablished(target) + if targetEstablished != gitTargetIsEstablished(existing) { + // Exactly one of them holds the folder. The one that does not, loses. + return !targetEstablished + } + switch { case target.CreationTimestamp.Time.After(existing.CreationTimestamp.Time): return true @@ -68,6 +83,14 @@ func gitTargetLosesConflict(target, existing *configbutleraiv1alpha3.GitTarget) } } +// gitTargetIsEstablished reports whether a GitTarget's materialization already lives at +// the destination its spec names. A target that has never materialized, or one that is +// retargeting, is not established: it is asking for a folder rather than holding one. +func gitTargetIsEstablished(target *configbutleraiv1alpha3.GitTarget) bool { + observed := target.Status.ObservedDestination + return observed != nil && sameDestination(*observed, specDestination(target)) +} + // gitTargetIdentityKey returns a stable, unique ordering key for a GitTarget. func gitTargetIdentityKey(t *configbutleraiv1alpha3.GitTarget) string { return t.Namespace + "/" + t.Name diff --git a/internal/controller/gittarget_path_overlap_test.go b/internal/controller/gittarget_path_overlap_test.go index 092cc5b8..bf11b458 100644 --- a/internal/controller/gittarget_path_overlap_test.go +++ b/internal/controller/gittarget_path_overlap_test.go @@ -154,3 +154,99 @@ func TestGitTargetLosesConflict(t *testing.T) { }) } } + +// established builds a GitTarget whose materialization already lives where its spec says. +func established(name, path string, created time.Time) *configbutleraiv1alpha3.GitTarget { + t := &configbutleraiv1alpha3.GitTarget{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, Namespace: "team-a", CreationTimestamp: metav1.NewTime(created), + }, + Spec: configbutleraiv1alpha3.GitTargetSpec{Branch: "main", Path: path}, + } + t.Status.ObservedDestination = &configbutleraiv1alpha3.GitTargetDestination{Branch: "main", Path: path} + return t +} + +// retargeting builds a GitTarget asking to move to path, still materialized at from. +func retargeting(name, from, to string, created time.Time) *configbutleraiv1alpha3.GitTarget { + t := established(name, from, created) + t.Spec.Path = to + return t +} + +// pending builds a GitTarget that has never materialized anywhere. +func pending(name, path string, created time.Time) *configbutleraiv1alpha3.GitTarget { + t := established(name, path, created) + t.Status.ObservedDestination = nil + return t +} + +// Once spec.path became mutable, creation time stopped being a faithful "who claimed this +// folder first". An OLDER target retargeting onto a YOUNGER incumbent's folder is still the +// newcomer, and must not evict the target that is already writing there. +func TestGitTargetLosesConflict_EstablishedBeatsRetargeting(t *testing.T) { + older := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + younger := older.Add(time.Hour) + + mover := retargeting("mover", "apps", "clusters", older) + incumbent := established("incumbent", "clusters", younger) + + if !gitTargetLosesConflict(mover, incumbent) { + t.Error("the older target retargeting onto an occupied folder must lose: it is the newcomer") + } + if gitTargetLosesConflict(incumbent, mover) { + t.Error("the incumbent, which never moved, must keep its folder") + } +} + +// A target that has never materialized is also a pending claim. +func TestGitTargetLosesConflict_EstablishedBeatsNeverMaterialized(t *testing.T) { + older := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + younger := older.Add(time.Hour) + + fresh := pending("fresh", "clusters", older) + incumbent := established("incumbent", "clusters", younger) + + if !gitTargetLosesConflict(fresh, incumbent) { + t.Error("a target that has never materialized must not evict one that has") + } +} + +// Two claims of the same strength still fall back to creation time, as before. +func TestGitTargetLosesConflict_EqualStrengthFallsBackToCreationTime(t *testing.T) { + older := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + younger := older.Add(time.Hour) + + if !gitTargetLosesConflict(pending("b", "apps", younger), pending("a", "apps", older)) { + t.Error("two fresh targets: the later-created one loses, as before") + } + if !gitTargetLosesConflict(established("b", "apps", younger), established("a", "apps", older)) { + t.Error("two established targets: the later-created one loses") + } + // Both retargeting onto the same folder: neither holds it, so creation time decides. + if !gitTargetLosesConflict( + retargeting("b", "x", "apps", younger), retargeting("a", "y", "apps", older), + ) { + t.Error("two movers: the later-created one loses") + } +} + +func TestGitTargetIsEstablished(t *testing.T) { + created := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + + if !gitTargetIsEstablished(established("a", "apps", created)) { + t.Error("spec and observedDestination agree: established") + } + if gitTargetIsEstablished(retargeting("a", "apps", "clusters", created)) { + t.Error("a target asking to move is not established at its new destination") + } + if gitTargetIsEstablished(pending("a", "apps", created)) { + t.Error("a target that never materialized is not established") + } + // A trailing slash is the same folder, so it is still established. + settled := established("a", "apps", created) + settled.Spec.Path = "apps/" + if !gitTargetIsEstablished(settled) { + t.Error("a trailing slash must not make a settled target look like a mover") + } +} diff --git a/internal/controller/gittarget_retarget.go b/internal/controller/gittarget_retarget.go new file mode 100644 index 00000000..7d76765d --- /dev/null +++ b/internal/controller/gittarget_retarget.go @@ -0,0 +1,254 @@ +// SPDX-License-Identifier: Apache-2.0 + +package controller + +import ( + "context" + "fmt" + "strings" + + "github.com/go-logr/logr" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + configbutleraiv1alpha3 "github.com/ConfigButler/gitops-reverser/api/v1alpha3" + "github.com/ConfigButler/gitops-reverser/internal/types" +) + +const ( + // GitTargetConditionRetargeting is True while spec's destination and + // status.observedDestination disagree: the folder named by observedDestination is + // being abandoned and the one named by spec is being built. + GitTargetConditionRetargeting = "Retargeting" + + // GitTargetReasonDestinationChanged marks the teardown of the old materialization. + GitTargetReasonDestinationChanged = "DestinationChanged" + // GitTargetReasonDestinationSettled marks a completed retarget, or a first + // materialization. + GitTargetReasonDestinationSettled = "DestinationSettled" + + // GitTargetEventRetargeted is the Kubernetes event emitted when a retarget completes. It + // names the abandoned folder, which nothing else deletes. + GitTargetEventRetargeted = "Retargeted" + // GitTargetEventRetargetSuperseded is emitted when a destination change arrives while a + // previous one was still building. It names the intermediate folder that move had begun + // writing, which status.observedDestination never named and nothing deletes. + GitTargetEventRetargetSuperseded = "RetargetSuperseded" +) + +// specDestination is the destination the spec asks for, in the shape status records: the +// path as the user wrote it, minus surrounding whitespace and a trailing slash. It is NOT +// normalizeGitTargetPath's form, which prefixes a "/" for prefix comparison and would +// leak an internal comparison key into a user-facing status field. +func specDestination(target *configbutleraiv1alpha3.GitTarget) configbutleraiv1alpha3.GitTargetDestination { + return configbutleraiv1alpha3.GitTargetDestination{ + Branch: target.Spec.Branch, + Path: displayGitTargetPath(target.Spec.Path), + SourceCluster: target.SourceClusterID(), + } +} + +// displayGitTargetPath trims a path to the form status records. "." — the deliberate +// repository-root choice — survives, because path.Clean keeps it. +func displayGitTargetPath(p string) string { + trimmed := strings.TrimSuffix(strings.TrimSpace(p), "/") + if trimmed == "" { + return "." + } + return trimmed +} + +// sameDestination compares two destinations the way the filesystem does: "apps" and +// "apps/" are the same folder, so a trailing slash is never a move. The source cluster is +// part of the identity — the same folder mirrored from a different cluster holds different +// content — but rotating that cluster's kubeconfig is not a move. +func sameDestination(a, b configbutleraiv1alpha3.GitTargetDestination) bool { + return a.Branch == b.Branch && + a.SourceCluster == b.SourceCluster && + normalizeGitTargetPath(a.Path) == normalizeGitTargetPath(b.Path) +} + +// destinationMoved reports whether the GitTarget has a materialization at a destination +// other than the one its spec now asks for. A target that has never materialized +// (observedDestination absent) has not moved: it has not arrived anywhere yet. +func destinationMoved(target *configbutleraiv1alpha3.GitTarget) bool { + observed := target.Status.ObservedDestination + if observed == nil { + return false + } + return !sameDestination(*observed, specDestination(target)) +} + +// retargetAlreadyTornDown reports whether the old materialization was already torn down +// for the CURRENT generation. It is generation-scoped rather than a plain "is Retargeting +// true?" so that a second destination change, arriving while the first retarget is still +// building, tears down again instead of quietly continuing to build the first one. +func retargetAlreadyTornDown(target *configbutleraiv1alpha3.GitTarget) bool { + c := conditionByType(target.Status.Conditions, GitTargetConditionRetargeting) + return c != nil && c.Status == metav1.ConditionTrue && c.ObservedGeneration == target.Generation +} + +// beginRetarget tears the old materialization down before anything is validated or written +// at the new destination. +// +// Teardown happens FIRST, and that ordering is the whole safety property. The writer reads +// spec.path fresh on every write, while the branch worker is bound to the branch the event +// stream was registered against. So between a spec change and a completed retarget, a live +// event would otherwise be written to the NEW path on the OLD branch. Cancelling the +// watches and unregistering the stream means nothing is written anywhere until the new +// destination has passed its own validation gate. +// +// It is idempotent per generation: a reconcile that finds the teardown already done for +// this generation skips it and lets the rebuild proceed. +func (r *GitTargetReconciler) beginRetarget( + ctx context.Context, + target *configbutleraiv1alpha3.GitTarget, + log logr.Logger, +) error { + abandoned := *target.Status.ObservedDestination + want := specDestination(target) + + log.Info("GitTarget destination changed; tearing down the old materialization", + "from", destinationString(abandoned), "to", destinationString(want)) + + r.noteSupersededDestination(target, want, log) + target.Status.RetargetingTo = &want + + if r.EventRouter != nil { + gitDest := types.NewResourceReference(target.Name, target.Namespace).WithUID(string(target.UID)) + // Release the old branch worker so evaluateWorkerWiringGate binds a fresh stream to + // the new branch's worker. + r.EventRouter.UnregisterGitTargetEventStream(types.NewResourceReference(target.Name, target.Namespace)) + if r.EventRouter.WatchManager != nil { + // Drops the watches AND the durable resume cursors: the new folder must be built + // from a full replay, not resumed into mid-stream. + if err := r.EventRouter.WatchManager.RetargetGitTarget(ctx, gitDest); err != nil { + return fmt.Errorf("retarget %s/%s: %w", target.Namespace, target.Name, err) + } + } + } + + // The push time belonged to the abandoned folder. + target.Status.LastPushTime = nil + r.setCondition(target, GitTargetConditionRetargeting, metav1.ConditionTrue, + GitTargetReasonDestinationChanged, + fmt.Sprintf("moving from %s to %s; the old folder is left in place as unmanaged content "+ + "and must be removed by hand if it is no longer wanted", + destinationString(abandoned), destinationString(want))) + return nil +} + +// settleDestination records that the current materialization now belongs to the spec's +// destination. It is called only once the new folder actually holds a snapshot the +// acceptance gate approved — that is what keeps the invariant honest: a successful snapshot +// is valid for the destination recorded in status.observedDestination. +// +// A completed retarget's message names the folder it abandoned, and that message must +// SURVIVE: it is the only place in status where an operator learns which folder is now +// unmanaged Git content they may want to remove. So once the condition is settled at the +// current destination this returns early rather than overwriting the message with a +// steady-state one on the next periodic reconcile. A Kubernetes event carries the same fact +// for anyone watching the object rather than reading its status. +func (r *GitTargetReconciler) settleDestination(target *configbutleraiv1alpha3.GitTarget, log logr.Logger) { + want := specDestination(target) + observed := target.Status.ObservedDestination + + if observed != nil && sameDestination(*observed, want) && destinationAlreadySettled(target) { + return + } + // The move is over, whichever branch below records it. A destination reverted mid-move + // lands here without ever passing through beginRetarget, so this is the second place an + // abandoned intermediate folder can be discovered. + r.noteSupersededDestination(target, want, log) + target.Status.RetargetingTo = nil + + if observed == nil || sameDestination(*observed, want) { + r.setCondition(target, GitTargetConditionRetargeting, metav1.ConditionFalse, + GitTargetReasonDestinationSettled, + "materialized at "+destinationString(want)) + target.Status.ObservedDestination = &want + return + } + + abandoned := *observed + log.Info("GitTarget retarget complete; the old folder is now unmanaged Git content", + "abandoned", destinationString(abandoned), "current", destinationString(want)) + r.eventf(target, GitTargetEventRetargeted, + "materialized at %s; %s was abandoned and is now unmanaged Git content", + destinationString(want), destinationString(abandoned)) + r.setCondition(target, GitTargetConditionRetargeting, metav1.ConditionFalse, + GitTargetReasonDestinationSettled, + fmt.Sprintf("materialized at %s; %s was abandoned and is now unmanaged Git content — "+ + "remove it by hand if it is no longer wanted", + destinationString(want), destinationString(abandoned))) + target.Status.ObservedDestination = &want +} + +// noteSupersededDestination names the folder an unfinished retarget had already begun +// building, when the destination changes again before it settles — including a revert back to +// where the content already is, which never passes through beginRetarget. +// +// status.observedDestination keeps naming the ORIGINAL folder until a move settles, so nothing +// else would ever name the intermediate one. It holds partially-written, now-unmanaged Git +// content, and — like every folder a retarget leaves — nothing deletes it. +func (r *GitTargetReconciler) noteSupersededDestination( + target *configbutleraiv1alpha3.GitTarget, + want configbutleraiv1alpha3.GitTargetDestination, + log logr.Logger, +) { + superseded := target.Status.RetargetingTo + if superseded == nil || sameDestination(*superseded, want) { + return + } + if observed := target.Status.ObservedDestination; observed != nil && sameDestination(*superseded, *observed) { + // The settled path already names it. + return + } + + log.Info("a retarget was superseded before it settled; its partially-built folder is now unmanaged", + "superseded", destinationString(*superseded), "to", destinationString(want)) + r.eventf(target, GitTargetEventRetargetSuperseded, + "a retarget to %s was superseded by %s before it settled; %s may hold partially-written, "+ + "now unmanaged Git content", + destinationString(*superseded), destinationString(want), destinationString(*superseded)) +} + +// destinationAlreadySettled reports whether the Retargeting condition already records a +// completed move, or a first materialization, at the current destination. +func destinationAlreadySettled(target *configbutleraiv1alpha3.GitTarget) bool { + c := conditionByType(target.Status.Conditions, GitTargetConditionRetargeting) + return c != nil && c.Status == metav1.ConditionFalse && c.Reason == GitTargetReasonDestinationSettled +} + +// eventf records a Kubernetes event on the GitTarget when a recorder is wired. Tests and +// standalone reconcilers leave it nil. +func (r *GitTargetReconciler) eventf( + target *configbutleraiv1alpha3.GitTarget, + reason, messageFmt string, + args ...any, +) { + if r.Recorder == nil { + return + } + r.Recorder.Eventf(target, corev1.EventTypeNormal, reason, messageFmt, args...) +} + +// markRetargetingUnknown records that the destination could not be evaluated because a +// control-plane gate blocked the reconcile before the data plane ran. +func (r *GitTargetReconciler) markRetargetingUnknown(target *configbutleraiv1alpha3.GitTarget, message string) { + // A retarget already under way keeps its True: the old materialization really is torn + // down, and a blocked gate does not put it back. + if retargetAlreadyTornDown(target) { + return + } + r.setCondition(target, GitTargetConditionRetargeting, metav1.ConditionUnknown, + GitTargetReasonNotChecked, message) +} + +func destinationString(d configbutleraiv1alpha3.GitTargetDestination) string { + out := d.Branch + ":" + d.Path + if d.SourceCluster != "" { + out += " (from " + d.SourceCluster + ")" + } + return out +} diff --git a/internal/controller/gittarget_retarget_test.go b/internal/controller/gittarget_retarget_test.go new file mode 100644 index 00000000..e318649a --- /dev/null +++ b/internal/controller/gittarget_retarget_test.go @@ -0,0 +1,371 @@ +// SPDX-License-Identifier: Apache-2.0 + +package controller + +import ( + "fmt" + "testing" + + "github.com/go-logr/logr" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + + configbutleraiv1alpha3 "github.com/ConfigButler/gitops-reverser/api/v1alpha3" +) + +func targetAt(branch, path string) *configbutleraiv1alpha3.GitTarget { + return &configbutleraiv1alpha3.GitTarget{ + ObjectMeta: metav1.ObjectMeta{Name: "acme", Namespace: "team-a", Generation: 1}, + Spec: configbutleraiv1alpha3.GitTargetSpec{ + ProviderRef: configbutleraiv1alpha3.GitProviderReference{Name: "prov"}, + Branch: branch, + Path: path, + }, + } +} + +// materializedAtMainApps stamps the destination the GitTarget's content currently lives at. +// Every fixture here materialized at main:apps first; the spec is what moves. +func materializedAtMainApps(target *configbutleraiv1alpha3.GitTarget) *configbutleraiv1alpha3.GitTarget { + target.Status.ObservedDestination = &configbutleraiv1alpha3.GitTargetDestination{Branch: "main", Path: "apps"} + return target +} + +func TestDestinationMoved(t *testing.T) { + t.Parallel() + + tests := map[string]struct { + target *configbutleraiv1alpha3.GitTarget + want bool + }{ + "never materialized": { + // Absent observedDestination is not a move: the target has not arrived anywhere. + target: targetAt("main", "apps"), + want: false, + }, + "settled": { + target: materializedAtMainApps(targetAt("main", "apps")), + want: false, + }, + "path changed": { + target: materializedAtMainApps(targetAt("main", "clusters/acme")), + want: true, + }, + "branch changed": { + target: materializedAtMainApps(targetAt("live", "apps")), + want: true, + }, + "trailing slash is not a move": { + // spec.path normalizes a trailing slash away, so "apps/" and "apps" name one folder. + target: materializedAtMainApps(targetAt("main", "apps/")), + want: false, + }, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + t.Parallel() + assert.Equal(t, tc.want, destinationMoved(tc.target)) + }) + } +} + +// The teardown is generation-scoped, not a plain "is Retargeting true?", so that a second +// destination change arriving mid-retarget tears down again instead of quietly continuing +// to build the first one. +func TestRetargetAlreadyTornDown(t *testing.T) { + t.Parallel() + + target := materializedAtMainApps(targetAt("live", "apps")) + assert.False(t, retargetAlreadyTornDown(target), "no Retargeting condition yet") + + r := &GitTargetReconciler{} + r.setCondition(target, GitTargetConditionRetargeting, metav1.ConditionTrue, + GitTargetReasonDestinationChanged, "moving") + assert.True(t, retargetAlreadyTornDown(target)) + + // A second spec change bumps the generation, which makes the recorded teardown stale. + target.Generation = 2 + assert.False(t, retargetAlreadyTornDown(target), + "a destination change arriving mid-retarget must tear down again") + + // A False condition is never a completed teardown, whatever its generation. + r.setCondition(target, GitTargetConditionRetargeting, metav1.ConditionFalse, + GitTargetReasonDestinationSettled, "settled") + assert.False(t, retargetAlreadyTornDown(target)) +} + +func TestBeginRetarget_MarksRetargetingAndClearsPushTime(t *testing.T) { + t.Parallel() + + target := materializedAtMainApps(targetAt("main", "clusters/acme")) + now := metav1.Now() + target.Status.LastPushTime = &now + + r := &GitTargetReconciler{} + require.NoError(t, r.beginRetarget(t.Context(), target, logr.Discard())) + + c := conditionByType(target.Status.Conditions, GitTargetConditionRetargeting) + require.NotNil(t, c) + assert.Equal(t, metav1.ConditionTrue, c.Status) + assert.Equal(t, GitTargetReasonDestinationChanged, c.Reason) + assert.Contains(t, c.Message, "main:apps", "the abandoned folder is named") + assert.Contains(t, c.Message, "main:clusters/acme", "and so is the new one") + assert.Contains(t, c.Message, "left in place", + "the operator must be told the old folder is not deleted") + + assert.Nil(t, target.Status.LastPushTime, "the push time belonged to the abandoned folder") + assert.Equal(t, "apps", target.Status.ObservedDestination.Path, + "observedDestination still names the folder being abandoned until the move settles") +} + +func TestSettleDestination_FirstMaterialization(t *testing.T) { + t.Parallel() + + target := targetAt("main", "apps") + r := &GitTargetReconciler{} + r.settleDestination(target, logr.Discard()) + + require.NotNil(t, target.Status.ObservedDestination) + assert.Equal(t, "main", target.Status.ObservedDestination.Branch) + assert.Equal(t, "apps", target.Status.ObservedDestination.Path) + + c := conditionByType(target.Status.Conditions, GitTargetConditionRetargeting) + require.NotNil(t, c) + assert.Equal(t, metav1.ConditionFalse, c.Status) + assert.Equal(t, GitTargetReasonDestinationSettled, c.Reason) + assert.NotContains(t, c.Message, "abandoned", "a first materialization abandons nothing") +} + +func TestSettleDestination_CompletedRetargetNamesTheAbandonedFolder(t *testing.T) { + t.Parallel() + + target := materializedAtMainApps(targetAt("live", "clusters/acme")) + r := &GitTargetReconciler{} + r.settleDestination(target, logr.Discard()) + + assert.Equal(t, "live", target.Status.ObservedDestination.Branch) + assert.Equal(t, "clusters/acme", target.Status.ObservedDestination.Path) + + c := conditionByType(target.Status.Conditions, GitTargetConditionRetargeting) + require.NotNil(t, c) + assert.Equal(t, metav1.ConditionFalse, c.Status) + assert.Contains(t, c.Message, "main:apps was abandoned") + assert.Contains(t, c.Message, "remove it by hand") +} + +// The path is normalized on the way into status, so a settled target never looks moved to +// the next reconcile just because the user wrote a trailing slash. +func TestSettleDestination_NormalizesThePath(t *testing.T) { + t.Parallel() + + target := targetAt("main", "apps/") + r := &GitTargetReconciler{} + r.settleDestination(target, logr.Discard()) + + assert.Equal(t, "apps", target.Status.ObservedDestination.Path) + assert.False(t, destinationMoved(target)) +} + +// A control-plane gate that blocks the reconcile has not put the old materialization back, +// so an in-flight retarget keeps reporting True. +func TestMarkRetargetingUnknown_KeepsAnInFlightRetarget(t *testing.T) { + t.Parallel() + + r := &GitTargetReconciler{} + + inFlight := materializedAtMainApps(targetAt("live", "apps")) + require.NoError(t, r.beginRetarget(t.Context(), inFlight, logr.Discard())) + r.markRetargetingUnknown(inFlight, "Blocked by Validated=False") + assert.Equal(t, metav1.ConditionTrue, + conditionByType(inFlight.Status.Conditions, GitTargetConditionRetargeting).Status) + + settled := materializedAtMainApps(targetAt("main", "apps")) + r.markRetargetingUnknown(settled, "Blocked by Validated=False") + c := conditionByType(settled.Status.Conditions, GitTargetConditionRetargeting) + require.NotNil(t, c) + assert.Equal(t, metav1.ConditionUnknown, c.Status) + assert.Equal(t, GitTargetReasonNotChecked, c.Reason) +} + +func TestDestinationString(t *testing.T) { + t.Parallel() + assert.Equal(t, "main:apps", + destinationString(configbutleraiv1alpha3.GitTargetDestination{Branch: "main", Path: "apps"})) +} + +// The abandoned-folder message is the only place in status where an operator learns which +// folder is now unmanaged Git content they may want to remove. A later reconcile must not +// overwrite it with a steady-state message. +func TestSettleDestination_AbandonedMessageSurvivesLaterReconciles(t *testing.T) { + t.Parallel() + + target := materializedAtMainApps(targetAt("main", "clusters/acme")) + r := &GitTargetReconciler{} + + r.settleDestination(target, logr.Discard()) + first := conditionByType(target.Status.Conditions, GitTargetConditionRetargeting).Message + require.Contains(t, first, "main:apps was abandoned") + + // Two more steady-state reconciles, exactly as the 5-minute requeue produces. + r.settleDestination(target, logr.Discard()) + r.settleDestination(target, logr.Discard()) + + after := conditionByType(target.Status.Conditions, GitTargetConditionRetargeting).Message + assert.Equal(t, first, after, + "a periodic reconcile must not erase the name of the folder the retarget abandoned") + assert.Equal(t, "clusters/acme", target.Status.ObservedDestination.Path) +} + +func TestSettleDestination_ReSettlesAfterANewRetarget(t *testing.T) { + t.Parallel() + + target := materializedAtMainApps(targetAt("main", "clusters/acme")) + r := &GitTargetReconciler{} + r.settleDestination(target, logr.Discard()) + + // A second move: teardown, then settle. The message must name the freshly abandoned + // folder, not the one from the first move. + target.Generation = 2 + target.Spec.Path = "clusters/beta" + require.NoError(t, r.beginRetarget(t.Context(), target, logr.Discard())) + r.settleDestination(target, logr.Discard()) + + msg := conditionByType(target.Status.Conditions, GitTargetConditionRetargeting).Message + assert.Contains(t, msg, "main:clusters/acme was abandoned") + assert.NotContains(t, msg, "main:apps") + assert.Equal(t, "clusters/beta", target.Status.ObservedDestination.Path) +} + +func TestDestinationAlreadySettled(t *testing.T) { + t.Parallel() + + target := targetAt("main", "apps") + assert.False(t, destinationAlreadySettled(target)) + + r := &GitTargetReconciler{} + r.setCondition(target, GitTargetConditionRetargeting, metav1.ConditionTrue, + GitTargetReasonDestinationChanged, "moving") + assert.False(t, destinationAlreadySettled(target)) + + r.setCondition(target, GitTargetConditionRetargeting, metav1.ConditionFalse, + GitTargetReasonDestinationSettled, "settled") + assert.True(t, destinationAlreadySettled(target)) +} + +// recordedEvent is one Kubernetes event the reconciler emitted. +type recordedEvent struct{ reason, message string } + +type fakeRecorder struct{ events []recordedEvent } + +func (f *fakeRecorder) Event(_ runtime.Object, _, reason, message string) { + f.events = append(f.events, recordedEvent{reason, message}) +} + +func (f *fakeRecorder) Eventf(_ runtime.Object, _, reason, messageFmt string, args ...any) { + f.events = append(f.events, recordedEvent{reason, fmt.Sprintf(messageFmt, args...)}) +} + +func (f *fakeRecorder) AnnotatedEventf( + _ runtime.Object, _ map[string]string, _, reason, messageFmt string, args ...any, +) { + f.events = append(f.events, recordedEvent{reason, fmt.Sprintf(messageFmt, args...)}) +} + +func (f *fakeRecorder) reasons() []string { + out := make([]string, 0, len(f.events)) + for _, e := range f.events { + out = append(out, e.reason) + } + return out +} + +// status.retargetingTo makes a move self-describing, and is cleared once it settles. +func TestRetargetingTo_TracksTheMoveAndClearsOnSettle(t *testing.T) { + t.Parallel() + + target := materializedAtMainApps(targetAt("main", "clusters/acme")) + r := &GitTargetReconciler{} + + require.NoError(t, r.beginRetarget(t.Context(), target, logr.Discard())) + require.NotNil(t, target.Status.RetargetingTo) + assert.Equal(t, "clusters/acme", target.Status.RetargetingTo.Path) + + r.settleDestination(target, logr.Discard()) + assert.Nil(t, target.Status.RetargetingTo, "the move is over") +} + +// A destination change arriving mid-move leaves the first move's partially-built folder +// behind. observedDestination still names the ORIGINAL folder, so without this event nothing +// would ever name the intermediate one. +func TestBeginRetarget_NamesTheFolderASupersededMoveAbandoned(t *testing.T) { + t.Parallel() + + recorder := &fakeRecorder{} + r := &GitTargetReconciler{Recorder: recorder} + + // main:apps -> main:intermediate, still building. + target := materializedAtMainApps(targetAt("main", "intermediate")) + require.NoError(t, r.beginRetarget(t.Context(), target, logr.Discard())) + assert.Empty(t, recorder.events, "the first move abandons only observedDestination, named at settle") + + // A second change before the first settles. + target.Generation = 2 + target.Spec.Path = "final" + require.NoError(t, r.beginRetarget(t.Context(), target, logr.Discard())) + + require.Equal(t, []string{GitTargetEventRetargetSuperseded}, recorder.reasons()) + assert.Contains(t, recorder.events[0].message, "main:intermediate") + assert.Contains(t, recorder.events[0].message, "main:final") + assert.Equal(t, "final", target.Status.RetargetingTo.Path) + + // The original folder is still the one observedDestination names, and settle reports it. + assert.Equal(t, "apps", target.Status.ObservedDestination.Path) + r.settleDestination(target, logr.Discard()) + assert.Contains(t, + conditionByType(target.Status.Conditions, GitTargetConditionRetargeting).Message, + "main:apps was abandoned") + require.Equal(t, + []string{GitTargetEventRetargetSuperseded, GitTargetEventRetargeted}, recorder.reasons()) +} + +// Reverting a destination mid-move never passes through beginRetarget — spec agrees with +// observedDestination again — but the folder the abandoned move began building is still there. +func TestSettleDestination_NamesTheFolderARevertedMoveAbandoned(t *testing.T) { + t.Parallel() + + recorder := &fakeRecorder{} + r := &GitTargetReconciler{Recorder: recorder} + + target := materializedAtMainApps(targetAt("main", "elsewhere")) + require.NoError(t, r.beginRetarget(t.Context(), target, logr.Discard())) + require.Empty(t, recorder.events) + + // The operator changes their mind and puts the path back. + target.Generation = 2 + target.Spec.Path = "apps" + require.False(t, destinationMoved(target), "spec agrees with observedDestination again") + + r.settleDestination(target, logr.Discard()) + + require.Equal(t, []string{GitTargetEventRetargetSuperseded}, recorder.reasons()) + assert.Contains(t, recorder.events[0].message, "main:elsewhere") + assert.Nil(t, target.Status.RetargetingTo) + assert.Equal(t, "apps", target.Status.ObservedDestination.Path) +} + +// A move that settles where it said it was going supersedes nothing. +func TestSettleDestination_CompletedMoveEmitsOnlyRetargeted(t *testing.T) { + t.Parallel() + + recorder := &fakeRecorder{} + r := &GitTargetReconciler{Recorder: recorder} + + target := materializedAtMainApps(targetAt("main", "clusters/acme")) + require.NoError(t, r.beginRetarget(t.Context(), target, logr.Discard())) + r.settleDestination(target, logr.Discard()) + + assert.Equal(t, []string{GitTargetEventRetargeted}, recorder.reasons()) +} diff --git a/internal/controller/gittarget_source_cluster.go b/internal/controller/gittarget_source_cluster.go new file mode 100644 index 00000000..585b09a6 --- /dev/null +++ b/internal/controller/gittarget_source_cluster.go @@ -0,0 +1,106 @@ +// SPDX-License-Identifier: Apache-2.0 + +package controller + +import ( + "context" + "fmt" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + k8stypes "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/tools/clientcmd" + + configbutleraiv1alpha3 "github.com/ConfigButler/gitops-reverser/api/v1alpha3" + "github.com/ConfigButler/gitops-reverser/internal/types" +) + +// GitTargetReasonSourceClusterUnreachable is the Validated=False reason for a GitTarget +// whose spec.sourceCluster names a Secret that is missing, empty at its key, or does not +// hold a parseable kubeconfig. It is a control-plane fault, not a data-plane one: nothing +// has been watched or written, and the human fix is on the Secret. +const GitTargetReasonSourceClusterUnreachable = "SourceClusterUnreachable" + +// sourceClusterRulesCaughtUp reports whether the compiled WatchRule/ClusterWatchRule set +// already names the GitTarget's CURRENT source cluster. +// +// The watch data plane learns a GitTarget's source cluster from the rules that point at it, +// because rules resolve their GitTarget when they compile. A rule recompiles when its +// GitTarget's generation bumps — but this reconcile can win that race, and declaring in that +// window would open watches against the previous cluster and write its objects into this +// GitTarget's folder. Waiting one requeue is free; mirroring the wrong cluster is not. +// +// A GitTarget with no rules yet has nothing to watch and nothing to disagree about. +func (r *GitTargetReconciler) sourceClusterRulesCaughtUp( + target *configbutleraiv1alpha3.GitTarget, + gitDest types.ResourceReference, +) bool { + compiled := r.EventRouter.WatchManager.CompiledSourceClusters(gitDest) + switch len(compiled) { + case 0: + // Nothing points at this GitTarget, so there is nothing to watch and nothing to + // disagree about. + return true + case 1: + return compiled[0] == target.SourceClusterID() + default: + // The rules disagree with each other: some have recompiled and some have not. + return false + } +} + +// describeSourceCluster renders a source-cluster id for logs and messages. +func describeSourceCluster(id string) string { + if id == "" { + return "local" + } + return id +} + +// validateSourceCluster checks that a GitTarget's source-cluster kubeconfig can be read and +// parsed from the config plane, before any watch is opened against it. +// +// This is a *legibility* gate, not a security one. Without it, a typo'd Secret name would +// surface only as a stalled data plane and a repeating log line; with it, the GitTarget says +// exactly which Secret it could not read. It deliberately does NOT dial the cluster: an +// unreachable-right-now cluster is a transient the data plane retries, and a controller that +// blocked on a network round trip would stall every other GitTarget behind it. +// +// The kubeconfig bytes are parsed and dropped. Nothing is retained. +func (r *GitTargetReconciler) validateSourceCluster( + ctx context.Context, + target *configbutleraiv1alpha3.GitTarget, +) (bool, string) { + if target.Spec.SourceCluster == nil { + return true, "" + } + + ref := target.Spec.SourceCluster.KubeConfigSecretRef + key := ref.Key + if key == "" { + key = configbutleraiv1alpha3.DefaultKubeConfigSecretKey + } + secretKey := k8stypes.NamespacedName{Namespace: target.Namespace, Name: ref.Name} + + var secret corev1.Secret + if err := r.Get(ctx, secretKey, &secret); err != nil { + if apierrors.IsNotFound(err) { + return false, fmt.Sprintf( + "spec.sourceCluster names Secret %s, which does not exist in this namespace", + secretKey.String()) + } + return false, fmt.Sprintf("cannot read source-cluster Secret %s: %v", secretKey.String(), err) + } + + raw, present := secret.Data[key] + if !present || len(raw) == 0 { + return false, fmt.Sprintf( + "source-cluster Secret %s has no data under key %q (set spec.sourceCluster.kubeConfigSecretRef.key)", + secretKey.String(), key) + } + if _, err := clientcmd.RESTConfigFromKubeConfig(raw); err != nil { + return false, fmt.Sprintf("source-cluster Secret %s key %q is not a usable kubeconfig: %v", + secretKey.String(), key, err) + } + return true, "" +} diff --git a/internal/controller/watchrule_controller.go b/internal/controller/watchrule_controller.go index b22730f8..56ff6c95 100644 --- a/internal/controller/watchrule_controller.go +++ b/internal/controller/watchrule_controller.go @@ -221,14 +221,8 @@ func (r *WatchRuleReconciler) reconcileWatchRuleViaTarget( // I added GitProviderStatus with Conditions. // TODO: Check GitProvider readiness. For now assume ready if found. - // Add rule to store with GitTarget reference and resolved values - r.RuleStore.AddOrUpdateWatchRule( - *watchRule, - target.Name, targetNS, // GitTarget reference (replaces GitDestination) - provider.Name, providerNS, // GitProvider reference (replaces GitRepoConfig) - target.Spec.Branch, - target.Spec.Path, - ) + // Add rule to store with its resolved GitTarget binding (destination + source cluster). + r.RuleStore.AddOrUpdateWatchRule(*watchRule, rulestore.NewTargetBinding(target, provider)) // Trigger WatchManager reconciliation for new/updated rule if r.WatchManager != nil { diff --git a/internal/git/asserted_author_test.go b/internal/git/asserted_author_test.go new file mode 100644 index 00000000..d9e906b2 --- /dev/null +++ b/internal/git/asserted_author_test.go @@ -0,0 +1,125 @@ +// SPDX-License-Identifier: Apache-2.0 + +package git + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestAuthorUserInfo_AssertedAuthorWinsOverEventAuthor(t *testing.T) { + t.Parallel() + + asserted := UserInfo{Username: "Ada Lovelace", DisplayName: "Ada Lovelace", Email: "ada@example.com"} + pendingWrite := PendingWrite{ + Kind: PendingWriteCommit, + Events: []Event{{UserInfo: UserInfo{Username: "alice"}}}, + AssertedAuthor: &asserted, + } + + // The assertion is the more specific, more recent statement, made by a caller who had + // to hold an RBAC verb to make it. + assert.Equal(t, asserted, pendingWrite.AuthorUserInfo()) + assert.Equal(t, "Ada Lovelace", pendingWrite.Author()) +} + +// An atomic write has no per-event author at all. An assertion still names one. +func TestAuthorUserInfo_AssertedAuthorAppliesToAtomicWrite(t *testing.T) { + t.Parallel() + + asserted := UserInfo{Username: "Ada Lovelace"} + pendingWrite := PendingWrite{Kind: PendingWriteAtomic, AssertedAuthor: &asserted} + assert.Equal(t, asserted, pendingWrite.AuthorUserInfo()) +} + +func TestAuthorUserInfo_NoAssertionKeepsEventAuthor(t *testing.T) { + t.Parallel() + + pendingWrite := PendingWrite{ + Kind: PendingWriteCommit, + Events: []Event{{UserInfo: UserInfo{Username: "alice"}}}, + } + assert.Equal(t, "alice", pendingWrite.AuthorUserInfo().Username) +} + +func TestCommitOptionsFor_AssertedAuthorSignsAsAuthorNotCommitter(t *testing.T) { + t.Parallel() + + config := ResolveCommitConfig(nil) + asserted := UserInfo{Username: "Ada Lovelace", DisplayName: "Ada Lovelace", Email: "ada@example.com"} + pendingWrite := PendingWrite{ + Kind: PendingWriteCommit, + Events: []Event{{UserInfo: UserInfo{Username: "alice"}}}, + AssertedAuthor: &asserted, + } + + options := commitOptionsFor(pendingWrite, config, nil, time.Now()) + + require.NotNil(t, options.Author) + assert.Equal(t, "Ada Lovelace", options.Author.Name) + assert.Equal(t, "ada@example.com", options.Author.Email) + + // A reader can always tell the reverser made the commit on someone's behalf. + require.NotNil(t, options.Committer) + assert.Equal(t, DefaultCommitterName, options.Committer.Name) + assert.Equal(t, DefaultCommitterEmail, options.Committer.Email) +} + +// An omitted email is derived from the name, exactly as it is for an audit-attributed +// author whose token carried no email claim. +func TestCommitOptionsFor_AssertedAuthorWithoutEmailGetsSafeAddress(t *testing.T) { + t.Parallel() + + asserted := UserInfo{Username: "Ada Lovelace", DisplayName: "Ada Lovelace"} + pendingWrite := PendingWrite{Kind: PendingWriteCommit, AssertedAuthor: &asserted} + + options := commitOptionsFor(pendingWrite, ResolveCommitConfig(nil), nil, time.Now()) + + require.NotNil(t, options.Author) + assert.Equal(t, "Ada Lovelace", options.Author.Name) + assert.NotEmpty(t, options.Author.Email) + assert.NotEqual(t, DefaultCommitterEmail, options.Author.Email) +} + +func window(author, target, namespace string) *openWindow { + return &openWindow{Author: author, GitTarget: target, GitTargetNamespace: namespace} +} + +// Without an assertion the request finalizes "the requesting author's open window", +// never someone else's. +func TestPendingCommitRequest_MatchesWindow_AuditAuthorBound(t *testing.T) { + t.Parallel() + + pcr := &pendingCommitRequest{author: "alice", gitTargetName: "tenants", gitTargetNamespace: "team-a"} + + assert.True(t, pcr.matchesWindow(window("alice", "tenants", "team-a"))) + assert.False(t, pcr.matchesWindow(window("bob", "tenants", "team-a")), "a foreign window stays open") + assert.False(t, pcr.matchesWindow(window("alice", "other", "team-a"))) + assert.False(t, pcr.matchesWindow(window("alice", "tenants", "team-b"))) + assert.False(t, pcr.matchesWindow(nil)) +} + +// An assertion binds to any open window for the GitTarget: it is a statement about the +// commit being made, not a claim to be whichever actor the audit stream recorded. +func TestPendingCommitRequest_MatchesWindow_AssertedAuthorBindsAnyWindow(t *testing.T) { + t.Parallel() + + asserted := UserInfo{Username: "Ada Lovelace"} + pcr := &pendingCommitRequest{ + author: "gitops-api", + assertedAuthor: &asserted, + gitTargetName: "tenants", + gitTargetNamespace: "team-a", + } + + assert.True(t, pcr.matchesWindow(window("someone-else", "tenants", "team-a"))) + assert.True(t, pcr.matchesWindow(window("", "tenants", "team-a")), "an unattributed window matches too") + + // The GitTarget scope still holds: assert-author is granted per GitTarget, so a + // request can never reach another target's window. + assert.False(t, pcr.matchesWindow(window("someone-else", "other", "team-a"))) + assert.False(t, pcr.matchesWindow(window("someone-else", "tenants", "team-b"))) +} diff --git a/internal/git/branch_worker.go b/internal/git/branch_worker.go index 7604b7a2..97b099d4 100644 --- a/internal/git/branch_worker.go +++ b/internal/git/branch_worker.go @@ -864,6 +864,7 @@ func (l *branchWorkerEventLoop) finalizeOpenWindowWithMessage(reason windowFinal targetName, targetNamespace := l.openWindow.GitTarget, l.openWindow.GitTargetNamespace windowTarget := targetNamespace + "/" + targetName pendingCR := l.openWindow.pendingCR + assertedAuthor := l.openWindow.pendingAuthor // Message precedence (§6.4.2): explicit override, else the attached // CommitRequest message, else the generated grouped-commit message (empty). effectiveMessage := message @@ -892,6 +893,10 @@ func (l *branchWorkerEventLoop) finalizeOpenWindowWithMessage(reason windowFinal return false } pendingWrite.CommitMessage = effectiveMessage + // An authorized CommitRequest's asserted author signs this commit instead of the + // events' audit-derived author. Carried on the write, not applied here, so it + // survives the push cooldown and the conflict rebase-replay with the data. + pendingWrite.AssertedAuthor = assertedAuthor // Carry the claiming CommitRequest onto the write so its result follows the // data: it is resolved Committed once this write is pushed (§6.5). pendingWrite.CommitRequest = pendingCR diff --git a/internal/git/commit_request_attach.go b/internal/git/commit_request_attach.go index c84b9a1e..8e9cf675 100644 --- a/internal/git/commit_request_attach.go +++ b/internal/git/commit_request_attach.go @@ -61,6 +61,13 @@ type AttachCommitRequest struct { // matches is attached; this binds "the open window" to "the requesting // author's open window". Author string + // AssertedAuthor, when set, is the identity a privileged client stated this commit + // is for (CommitRequest spec.author, honored only against an authorized admission + // record). It changes the attach in two ways: the request binds to ANY open window + // for the GitTarget rather than only to one whose audit-derived author matches, and + // the identity becomes the commit's author signature instead of merely selecting a + // window. The committer is unaffected. + AssertedAuthor *UserInfo // GitTargetName / GitTargetNamespace scope the finalize to one GitTarget. GitTargetName string GitTargetNamespace string @@ -93,6 +100,7 @@ func (a AttachCommitRequest) id() commitRequestID { type pendingCommitRequest struct { id commitRequestID author string + assertedAuthor *UserInfo gitTargetName string gitTargetNamespace string message string @@ -103,15 +111,25 @@ type pendingCommitRequest struct { attached bool } -// matchesWindow reports whether the request identifies the given open window by -// author and GitTarget. +// matchesWindow reports whether the request identifies the given open window. +// +// Without an asserted author the match is by audit-derived author AND GitTarget: the +// request finalizes "the requesting author's open window", never someone else's. +// +// With an asserted author the match is by GitTarget alone. The assertion is a statement +// about the commit being made, by a caller who had to hold an RBAC verb to make it — not +// a claim to be whichever actor the audit stream happened to record for the pending edits. func (p *pendingCommitRequest) matchesWindow(w *openWindow) bool { if p == nil || w == nil { return false } - return p.author == w.Author && - p.gitTargetName == w.GitTarget && - p.gitTargetNamespace == w.GitTargetNamespace + if p.gitTargetName != w.GitTarget || p.gitTargetNamespace != w.GitTargetNamespace { + return false + } + if p.assertedAuthor != nil { + return true + } + return p.author == w.Author } // commitRequestOutcomeEntry is a resolved outcome retained for the controller to diff --git a/internal/git/commit_request_attach_loop.go b/internal/git/commit_request_attach_loop.go index 5b70bdc4..3cfe7d36 100644 --- a/internal/git/commit_request_attach_loop.go +++ b/internal/git/commit_request_attach_loop.go @@ -83,14 +83,20 @@ func (l *branchWorkerEventLoop) handleAttachCommitRequest(req *AttachCommitReque l.pendingCRs[id] = &pendingCommitRequest{ id: id, author: req.Author, + assertedAuthor: req.AssertedAuthor, gitTargetName: req.GitTargetName, gitTargetNamespace: req.GitTargetNamespace, message: req.Message, finalizeAt: time.Now().Add(time.Duration(req.CloseDelaySeconds) * time.Second), } + assertedAuthor := "" + if req.AssertedAuthor != nil { + assertedAuthor = req.AssertedAuthor.Username + } l.w.Log.Info("CommitRequest registered with worker", "request", id.Namespace+"/"+id.Name, "author", req.Author, + "assertedAuthor", assertedAuthor, "target", req.GitTargetNamespace+"/"+req.GitTargetName, "closeDelaySeconds", req.CloseDelaySeconds) } @@ -129,15 +135,23 @@ func (l *branchWorkerEventLoop) attachWaitingCommitRequests() { } } -// attachToOpenWindow binds a request's message to the currently-open window. +// attachToOpenWindow binds a request's message — and, when it carries one, its asserted +// author — to the currently-open window. func (l *branchWorkerEventLoop) attachToOpenWindow(pcr *pendingCommitRequest) { l.openWindow.pendingMessage = pcr.message + l.openWindow.pendingAuthor = pcr.assertedAuthor id := pcr.id l.openWindow.pendingCR = &id pcr.attached = true + assertedAuthor := "" + if pcr.assertedAuthor != nil { + assertedAuthor = pcr.assertedAuthor.Username + } l.w.Log.Info("CommitRequest attached to open window", "request", id.Namespace+"/"+id.Name, "author", pcr.author, + "assertedAuthor", assertedAuthor, + "windowAuthor", l.openWindow.Author, "target", pcr.gitTargetNamespace+"/"+pcr.gitTargetName) } diff --git a/internal/git/open_window.go b/internal/git/open_window.go index 517cd75a..7dbfdbee 100644 --- a/internal/git/open_window.go +++ b/internal/git/open_window.go @@ -33,6 +33,11 @@ type openWindow struct { // Once set, whichever path finalizes the window uses it instead of the // generated grouped-commit message, so an early cut-off still carries intent. pendingMessage string + // pendingAuthor is the author a privileged CommitRequest asserted for this window + // (spec.author). When set it overrides the events' own audit-derived author in the + // commit's author signature. Nil is the ordinary case: the window's events name the + // author, or nobody does and the committer signs. + pendingAuthor *UserInfo // pendingCR identifies the CommitRequest claiming this window; at most one. On // finalize its outcome is resolved (Committed once the carrying write pushes). pendingCR *commitRequestID diff --git a/internal/git/pending_writes.go b/internal/git/pending_writes.go index 0d481d52..288ca6f9 100644 --- a/internal/git/pending_writes.go +++ b/internal/git/pending_writes.go @@ -227,7 +227,15 @@ func (p PendingWrite) Author() string { // AuthorUserInfo returns the full author identity for commit-shaped pending // writes, including any OIDC display name and email. Atomic and empty writes // have no per-user author and return the zero value. +// +// An author asserted by an authorized CommitRequest (spec.author) wins over the events' +// own audit-derived identity: it is the more specific, more recent statement, made by a +// caller who had to hold an RBAC verb to make it. It applies even to an atomic write, +// which otherwise has no author at all. func (p PendingWrite) AuthorUserInfo() UserInfo { + if p.AssertedAuthor != nil { + return *p.AssertedAuthor + } if p.Kind == PendingWriteAtomic || len(p.Events) == 0 { return UserInfo{} } diff --git a/internal/git/types.go b/internal/git/types.go index ba6c1c03..918ad9a8 100644 --- a/internal/git/types.go +++ b/internal/git/types.go @@ -193,6 +193,12 @@ type PendingWrite struct { // snapshot's push past its window. Committed *bool + // AssertedAuthor, when set, is the identity an authorized CommitRequest stated this + // commit is for (spec.author). It overrides the events' audit-derived author in the + // commit's author signature; the committer is always the operator. See + // docs/design/multi-tenant/asserted-commit-author.md. + AssertedAuthor *UserInfo + // CommitRequest, when set, is the CommitRequest claiming this write: it is // resolved Committed (with CommitSHA) once this write is pushed (§6.5 of // docs/design/stream/commitrequest-design.md). It rides the write through the diff --git a/internal/manifestanalyzer/render.go b/internal/manifestanalyzer/render.go index 913cca89..81853bd9 100644 --- a/internal/manifestanalyzer/render.go +++ b/internal/manifestanalyzer/render.go @@ -10,7 +10,6 @@ import ( "strings" "github.com/ConfigButler/gitops-reverser/internal/git/manifestedit" - "github.com/ConfigButler/gitops-reverser/internal/types" ) // RenderJSON writes the report as indented JSON. It is the machine-readable form @@ -197,102 +196,3 @@ func planActionTarget(a PlanAction) string { } return fmt.Sprintf("%s#%d", a.Ref.FilePath, a.Ref.DocumentIndex) } - -// RenderScanJSON writes the scan as indented JSON: the machine-readable form shared -// by the CLI and the GitTarget status path. It omits the live desired objects, which -// are unbounded; it carries only the decided plan. -func RenderScanJSON(w io.Writer, result ScanResult) error { - enc := json.NewEncoder(w) - enc.SetIndent("", " ") - return enc.Encode(scanToJSON(result)) -} - -// scanJSON is the compact, bounded JSON projection of a ScanResult. -type scanJSON struct { - Accepted bool `json:"accepted"` - Issues []AcceptanceIssue `json:"issues"` - Retained []retainedJSON `json:"retained,omitempty"` - Plan planJSON `json:"plan"` -} - -type retainedJSON struct { - Path string `json:"path"` - DocumentIndex int `json:"documentIndex"` - Identity manifestedit.Identity `json:"identity"` -} - -type planJSON struct { - Counts map[string]int `json:"counts"` - Actions []planActionJSON `json:"actions"` - Diagnostics []manifestedit.Diagnostic `json:"diagnostics,omitempty"` -} - -type planActionJSON struct { - Kind string `json:"kind"` - Path string `json:"path,omitempty"` - // DocumentIndex is a pointer so that index 0 — a real, common target for a patch - // or drop on a file's first document — is preserved, while a create (which has no - // existing document location) omits it. A plain int with omitempty would drop the - // meaningful 0 and weaken the machine-readable contract. - DocumentIndex *int `json:"documentIndex,omitempty"` - Identity manifestedit.Identity `json:"identity"` - Resource string `json:"resource,omitempty"` - Reason string `json:"reason,omitempty"` -} - -// scanToJSON builds the compact JSON projection. -func scanToJSON(result ScanResult) scanJSON { - out := scanJSON{ - Accepted: result.Acceptance.Accepted, - Issues: result.Acceptance.Issues, - Plan: planJSON{ - Counts: planCounts(result.Plan), - Diagnostics: result.Plan.Diagnostics, - }, - } - for _, rd := range result.Acceptance.Retained { - out.Retained = append(out.Retained, retainedJSON{ - Path: rd.Location.Path, DocumentIndex: rd.Location.DocumentIndex, Identity: rd.Identity, - }) - } - for _, a := range result.Plan.Actions { - out.Plan.Actions = append(out.Plan.Actions, planActionJSON{ - Kind: string(a.Kind), - Path: planActionTarget(a), - DocumentIndex: documentIndexJSON(a), - Identity: a.Identity, - Resource: resourceString(a.Resource), - Reason: a.Reason, - }) - } - return out -} - -// documentIndexJSON returns the action's document index as a pointer: nil for a -// create (no existing document location, so the field is omitted), otherwise the -// real index — including a meaningful 0. -func documentIndexJSON(a PlanAction) *int { - if a.Kind == PlanCreate { - return nil - } - i := a.Ref.DocumentIndex - return &i -} - -// planCounts converts the plan's per-kind counts to string keys for JSON. -func planCounts(plan Plan) map[string]int { - out := map[string]int{} - for kind, n := range plan.Counts() { - out[string(kind)] = n - } - return out -} - -// resourceString renders a resolved resource identity, or "" when it is the zero -// value (a document a structure-only store never resolved). -func resourceString(r types.ResourceIdentifier) string { - if r == (types.ResourceIdentifier{}) { - return "" - } - return r.Key() -} diff --git a/internal/manifestanalyzer/render_test.go b/internal/manifestanalyzer/render_test.go index 7b8f1ce0..ce691830 100644 --- a/internal/manifestanalyzer/render_test.go +++ b/internal/manifestanalyzer/render_test.go @@ -146,86 +146,3 @@ func TestRenderScanText_Accepted(t *testing.T) { t.Errorf("expected no-changes plan line: %s", out) } } - -func TestRenderScanJSON(t *testing.T) { - var buf bytes.Buffer - if err := RenderScanJSON(&buf, scanWithRefusalAndPlan(t)); err != nil { - t.Fatalf("RenderScanJSON: %v", err) - } - - var parsed struct { - Accepted bool `json:"accepted"` - Issues []struct { - Kind string `json:"kind"` - } `json:"issues"` - Retained []struct { - Path string `json:"path"` - } `json:"retained"` - Plan struct { - Counts map[string]int `json:"counts"` - Actions []struct { - Kind string `json:"kind"` - Path string `json:"path"` - Resource string `json:"resource"` - } `json:"actions"` - } `json:"plan"` - } - if err := json.Unmarshal(buf.Bytes(), &parsed); err != nil { - t.Fatalf("scan JSON is invalid: %v\n%s", err, buf.String()) - } - if parsed.Accepted { - t.Errorf("scan JSON should report refusal") - } - if len(parsed.Issues) == 0 || len(parsed.Retained) != 1 { - t.Errorf("scan JSON issues=%d retained=%d", len(parsed.Issues), len(parsed.Retained)) - } - if parsed.Plan.Counts["create"] != 1 || parsed.Plan.Counts["patch"] != 1 { - t.Errorf("scan JSON plan counts = %+v", parsed.Plan.Counts) - } - if len(parsed.Plan.Actions) != 2 { - t.Errorf("scan JSON should carry both actions, got %+v", parsed.Plan.Actions) - } -} - -// TestRenderScanJSON_PreservesDocumentIndexZero guards the machine-readable contract: -// a patch on a file's first document must serialize documentIndex 0 (not omit it), -// while a create — which has no existing document location — omits it. -func TestRenderScanJSON_PreservesDocumentIndexZero(t *testing.T) { - fsys := fstest.MapFS{"deploy.yaml": {Data: []byte(deployYAML)}} - desired := []DesiredResource{desiredDeployWeb(3), desiredConfigMap("new")} // patch deploy#0 + create - result := Scan(context.Background(), fsys, snapMapper(), desired, ScanPolicy{}) - - var buf bytes.Buffer - if err := RenderScanJSON(&buf, result); err != nil { - t.Fatalf("RenderScanJSON: %v", err) - } - if !strings.Contains(buf.String(), `"documentIndex": 0`) { - t.Errorf("a patch on the first document must keep documentIndex 0:\n%s", buf.String()) - } - - var parsed struct { - Plan struct { - Actions []struct { - Kind string `json:"kind"` - DocumentIndex *int `json:"documentIndex"` - } `json:"actions"` - } `json:"plan"` - } - if err := json.Unmarshal(buf.Bytes(), &parsed); err != nil { - t.Fatalf("invalid JSON: %v", err) - } - for _, a := range parsed.Plan.Actions { - switch a.Kind { - case "patch": - if a.DocumentIndex == nil || *a.DocumentIndex != 0 { - t.Errorf("patch action should carry documentIndex 0, got %+v", a.DocumentIndex) - } - case "create": - if a.DocumentIndex != nil { - t.Errorf("create action should omit documentIndex, got %v", *a.DocumentIndex) - } - default: - t.Errorf("unexpected action kind %q", a.Kind) - } - } -} diff --git a/internal/manifestanalyzer/repowalk.go b/internal/manifestanalyzer/repowalk.go index 99cb3a55..8bae6af6 100644 --- a/internal/manifestanalyzer/repowalk.go +++ b/internal/manifestanalyzer/repowalk.go @@ -4,7 +4,6 @@ package manifestanalyzer import ( "context" - "encoding/json" "fmt" "io" "io/fs" @@ -711,14 +710,6 @@ func sortedKeysOf(m map[string]struct{}) []string { return out } -// RenderRepoJSON writes the repo report as indented JSON — the product's interface, -// matching the existing --format json convention. -func RenderRepoJSON(w io.Writer, rep RepoReport) error { - enc := json.NewEncoder(w) - enc.SetIndent("", " ") - return enc.Encode(rep) -} - // RenderRepoText writes a compact human summary of the repo report: one line per // candidate, then the roll-up. It is a convenience view; JSON is the contract. func RenderRepoText(w io.Writer, rep RepoReport) { diff --git a/internal/manifestanalyzer/repowalk_test.go b/internal/manifestanalyzer/repowalk_test.go index 3ee87410..926c1716 100644 --- a/internal/manifestanalyzer/repowalk_test.go +++ b/internal/manifestanalyzer/repowalk_test.go @@ -5,6 +5,7 @@ package manifestanalyzer import ( "bytes" "context" + "encoding/json" "os" "path/filepath" "testing" @@ -61,8 +62,13 @@ func checkGoldenFixture(t *testing.T, fixture string) { } rep.Root = "" + // The engine's own report shape, not the published one. pkg/manifestanalyzer owns the + // machine-readable contract and pins it separately; this corpus pins the classification + // facts (layout, refusal codes, overlaps, namespace inference, rendered/editable split). var buf bytes.Buffer - if err := RenderRepoJSON(&buf, rep); err != nil { + enc := json.NewEncoder(&buf) + enc.SetIndent("", " ") + if err := enc.Encode(rep); err != nil { t.Fatalf("render json: %v", err) } diff --git a/internal/queue/attribution_index.go b/internal/queue/attribution_index.go index 187421d6..9f9063a9 100644 --- a/internal/queue/attribution_index.go +++ b/internal/queue/attribution_index.go @@ -28,8 +28,10 @@ import ( // from one that never arrived. Configurable via --author-attribution-ttl. const DefaultAttributionFactTTL = 15 * time.Minute -// keyPrefix is the fixed root namespace for every Redis key (cursors and facts alike). -const keyPrefix = "gitops-reverser" +// DefaultKeyPrefix is the root namespace every Redis key (cursors, facts, and command +// author records alike) carries when --redis-key-prefix is not set. It is also the value +// every release before the flag existed used, so the default is a no-op upgrade. +const DefaultKeyPrefix = "gitops-reverser" const ( // attributionKeySuffix namespaces audit-sourced resource author facts under the @@ -105,6 +107,10 @@ type AuthorResolution struct { type AttributionIndex struct { client *redis.Client factTTL time.Duration + // keyPrefix is the root namespace this index writes and reads under, shared with the + // RedisStore that built it. Empty only in tests constructed by hand; every key builder + // resolves it so an empty value still lands on DefaultKeyPrefix. + keyPrefix string } // RecordFact stores the attribution fact for one accepted, mutating audit event. A @@ -406,7 +412,7 @@ func attributionResultForFact(fact AuthorFact, weak bool) AttributionResult { // factKeyBase is the per-type prefix shared by every fact key, e.g. // "gitops-reverser:author:v1:audit:apps/deployments". func (a *AttributionIndex) factKeyBase(gr string) string { - return keyPrefix + attributionKeySuffix + gr + return resolveKeyPrefix(a.keyPrefix) + attributionKeySuffix + gr } // factKeyExact is the immutable per-write fact key, e.g. @@ -447,7 +453,7 @@ func (a *AttributionIndex) recordFactIndexSize(ctx context.Context) { } var cursor uint64 var count int64 - pattern := keyPrefix + attributionKeySuffix + "*" + pattern := resolveKeyPrefix(a.keyPrefix) + attributionKeySuffix + "*" for { keys, next, err := a.client.Scan(ctx, cursor, pattern, attributionFactScanBatchSize).Result() if err != nil { diff --git a/internal/queue/command_author_store.go b/internal/queue/command_author_store.go index 747f2374..e886af73 100644 --- a/internal/queue/command_author_store.go +++ b/internal/queue/command_author_store.go @@ -34,6 +34,19 @@ type CommandAuthor struct { DisplayName string `json:"displayName,omitempty"` Email string `json:"email,omitempty"` RequestedAt string `json:"requestedAt,omitempty"` // RFC3339Nano, for lag metrics/debug + + // AssertAuthorAllowed records that the admission webhook ran a SubjectAccessReview + // for this submitter against the `assert-author` verb on the referenced GitTarget, + // and it said yes. It is the ONLY thing that lets the controller honor a + // CommitRequest's spec.author. + // + // The verdict lives on the record rather than being re-derived by the controller + // because the controller cannot know who submitted the object — only admission saw + // the requester. And it must be recorded rather than trusted from the webhook's + // allow/deny alone, because that webhook is failurePolicy: Ignore by design: a + // bypassed webhook writes no record, so the assertion is ignored rather than + // silently honored. Fail-closed, independent of failurePolicy. + AssertAuthorAllowed bool `json:"assertAuthorAllowed,omitempty"` } // CommandAuthorStore records and reads command authorship. It shares RedisStore's @@ -42,6 +55,8 @@ type CommandAuthor struct { type CommandAuthorStore struct { client *redis.Client ttl time.Duration + // keyPrefix is the root namespace shared with the RedisStore that built this store. + keyPrefix string } // RecordCommandAuthor is the admission-side write: capture the authenticated submitter @@ -78,5 +93,5 @@ func (s *CommandAuthorStore) LookupCommandAuthor( // key identifies the command by UID alone — globally unique, like the watch cursor key, // so namespace/name/kind would be redundant (kept out for a tight key). func (s *CommandAuthorStore) key(uid types.UID) string { - return keyPrefix + commandAuthorKeySuffix + escapeKeyField(string(uid)) + return resolveKeyPrefix(s.keyPrefix) + commandAuthorKeySuffix + escapeKeyField(string(uid)) } diff --git a/internal/queue/key_prefix.go b/internal/queue/key_prefix.go new file mode 100644 index 00000000..d2e5bd11 --- /dev/null +++ b/internal/queue/key_prefix.go @@ -0,0 +1,66 @@ +// SPDX-License-Identifier: Apache-2.0 + +package queue + +import ( + "fmt" + "strings" +) + +// maxKeyPrefixLength bounds the root namespace. Redis keys are capped at 512MB, so this +// is not a protocol limit — it is a legibility limit, and a guard against a prefix that +// dwarfs the key it namespaces. +const maxKeyPrefixLength = 128 + +// ValidateKeyPrefix checks a --redis-key-prefix value and returns its normalized form. +// +// Two independent constraints shape the allowed character set: +// +// - The attribution telemetry gauge SCANs ":author:v1:audit:*". Redis glob +// metacharacters (*, ?, [, ], \) in the prefix would silently make that pattern match +// the wrong keyspace, so they are rejected rather than escaped — a prefix is an +// operator-chosen identifier, not user data. +// - Key fields (uid, resourceVersion, namespace) are ':'-delimited and %-escaped by +// escapeKeyField. '%' in the prefix would make an escaped key ambiguous with an +// unescaped one, so it is rejected too. +// +// ':' is allowed, because a prefix like "cell-a:tenant-7" is a natural nesting and every +// suffix constant already begins with ':'. A trailing ':' is normalized away so +// "tenant-7:" and "tenant-7" name the same keyspace rather than two that differ by an +// empty segment. +// +// An empty prefix is rejected: an unprefixed keyspace collides with Redis's own key +// namespace conventions and, more importantly, silently un-namespaces an install that +// meant to set the flag and passed the empty string. Use DefaultKeyPrefix to opt out. +func ValidateKeyPrefix(prefix string) (string, error) { + normalized := strings.TrimRight(strings.TrimSpace(prefix), ":") + if normalized == "" { + return "", fmt.Errorf("redis-key-prefix must be a non-empty identifier (default %q)", DefaultKeyPrefix) + } + if len(normalized) > maxKeyPrefixLength { + return "", fmt.Errorf("redis-key-prefix must be at most %d characters, got %d", + maxKeyPrefixLength, len(normalized)) + } + for _, r := range normalized { + switch { + case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9': + case r == '-', r == '_', r == '.', r == ':': + default: + return "", fmt.Errorf( + "redis-key-prefix %q contains %q; allowed characters are [A-Za-z0-9], '-', '_', '.' and ':'", + prefix, string(r)) + } + } + return normalized, nil +} + +// resolveKeyPrefix normalizes a prefix for internal use, falling back to the default when +// empty. Construction paths that did not go through ValidateKeyPrefix (tests, and the +// zero-value RedisStoreConfig) land here rather than building keys with an empty root. +func resolveKeyPrefix(prefix string) string { + normalized := strings.TrimRight(strings.TrimSpace(prefix), ":") + if normalized == "" { + return DefaultKeyPrefix + } + return normalized +} diff --git a/internal/queue/key_prefix_test.go b/internal/queue/key_prefix_test.go new file mode 100644 index 00000000..8339ea3e --- /dev/null +++ b/internal/queue/key_prefix_test.go @@ -0,0 +1,173 @@ +// SPDX-License-Identifier: Apache-2.0 + +package queue + +import ( + "context" + "strings" + "testing" + + "github.com/alicebob/miniredis/v2" + "github.com/stretchr/testify/require" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +func TestValidateKeyPrefix(t *testing.T) { + t.Parallel() + + tests := map[string]struct { + in string + want string + wantErr string + }{ + "default": {in: DefaultKeyPrefix, want: "gitops-reverser"}, + "nested with colon": {in: "cell-a:tenant-7", want: "cell-a:tenant-7"}, + "trailing colon dropped": {in: "tenant-7:", want: "tenant-7"}, + "surrounding space": {in: " tenant-7 ", want: "tenant-7"}, + "dots and underscores": {in: "rev_1.example.com", want: "rev_1.example.com"}, + + "empty": {in: "", wantErr: "non-empty"}, + "only colons": {in: ":::", wantErr: "non-empty"}, + "glob star": {in: "tenant-*", wantErr: `contains "*"`}, + "glob question": {in: "tenant-?", wantErr: `contains "?"`}, + "glob bracket": {in: "tenant-[a]", wantErr: `contains "["`}, + "escape char": {in: `tenant\a`, wantErr: `contains "\\"`}, + "percent": {in: "tenant%3A", wantErr: `contains "%"`}, + "slash": {in: "tenant/a", wantErr: `contains "/"`}, + "space inside": {in: "tenant a", wantErr: `contains " "`}, + "too long": {in: strings.Repeat("a", maxKeyPrefixLength+1), wantErr: "at most 128 characters"}, + "exactly max long": { + in: strings.Repeat("a", maxKeyPrefixLength), + want: strings.Repeat("a", maxKeyPrefixLength), + }, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + t.Parallel() + got, err := ValidateKeyPrefix(tc.in) + if tc.wantErr != "" { + require.Error(t, err) + require.Contains(t, err.Error(), tc.wantErr) + return + } + require.NoError(t, err) + require.Equal(t, tc.want, got) + }) + } +} + +func TestResolveKeyPrefix_FallsBackToDefault(t *testing.T) { + t.Parallel() + // A hand-built store (tests, zero-value config) must never write to an unprefixed + // keyspace — that would be indistinguishable from another tool's keys. + require.Equal(t, DefaultKeyPrefix, resolveKeyPrefix("")) + require.Equal(t, DefaultKeyPrefix, resolveKeyPrefix(" ")) + require.Equal(t, "tenant-7", resolveKeyPrefix("tenant-7:")) +} + +// newPrefixedRedisStore builds a store on a fresh miniredis under an explicit prefix. +func newPrefixedRedisStore(t *testing.T, prefix string) *RedisStore { + t.Helper() + mr := miniredis.RunT(t) + store, err := NewRedisStore(RedisStoreConfig{Addr: mr.Addr(), KeyPrefix: prefix}) + require.NoError(t, err) + return store +} + +func TestRedisStore_KeyPrefixReachesEveryKeyFamily(t *testing.T) { + t.Parallel() + + store := newPrefixedRedisStore(t, "cell-a:tenant-7") + require.Equal(t, "cell-a:tenant-7", store.KeyPrefix()) + + gvr := schema.GroupVersionResource{Group: "apps", Version: "v1", Resource: "deployments"} + require.Equal(t, + "cell-a:tenant-7:watch:v1:target:gtuid-3:apps/deployments:namespace:team-a:last-rv", + store.watchCursorKey("gtuid-3", gvr, "team-a")) + + idx := store.AttributionIndex(0) + require.Equal(t, "cell-a:tenant-7:author:v1:audit:apps/deployments:object:uid-1:101", + idx.factKeyExact("apps/deployments", "uid-1", "101")) + require.Equal(t, "cell-a:tenant-7:author:v1:audit:apps/deployments:object:uid-1:last", + idx.factKeyLast("apps/deployments", "uid-1")) + require.Equal(t, "cell-a:tenant-7:author:v1:audit:apps/deployments:rv:101", + idx.factKeyRV("apps/deployments", "101")) + + require.Equal(t, "cell-a:tenant-7:author:v1:command:cr-uid", store.CommandAuthorStore().key("cr-uid")) +} + +func TestRedisStore_DefaultPrefixIsUnchanged(t *testing.T) { + t.Parallel() + + // The flag defaults to DefaultKeyPrefix, so an upgrade must not orphan the keys a + // previous release wrote. + store := newPrefixedRedisStore(t, DefaultKeyPrefix) + require.Equal(t, "gitops-reverser:watch:v1:target:gtuid-3:configmaps:cluster:last-rv", + store.watchCursorKey("gtuid-3", coreConfigmapsGVR(), "")) + require.Equal(t, "gitops-reverser:author:v1:command:cr-uid", store.CommandAuthorStore().key("cr-uid")) +} + +// Two reversers sharing one Redis/Valkey and one logical database must not read each +// other's cursors. Sixteen databases is the only separator Redis offers; a prefix is not +// bounded that way. +func TestRedisStore_DistinctPrefixesIsolateCursors(t *testing.T) { + t.Parallel() + + mr := miniredis.RunT(t) + ctx := context.Background() + gvr := coreConfigmapsGVR() + + tenantA, err := NewRedisStore(RedisStoreConfig{Addr: mr.Addr(), KeyPrefix: "tenant-a"}) + require.NoError(t, err) + tenantB, err := NewRedisStore(RedisStoreConfig{Addr: mr.Addr(), KeyPrefix: "tenant-b"}) + require.NoError(t, err) + + require.NoError(t, tenantA.RecordWatchCursor(ctx, "same-uid", gvr, "", "111")) + + rv, ok := tenantA.LookupWatchCursor(ctx, "same-uid", gvr, "") + require.True(t, ok) + require.Equal(t, "111", rv) + + _, ok = tenantB.LookupWatchCursor(ctx, "same-uid", gvr, "") + require.False(t, ok, "tenant-b must not see tenant-a's cursor on the same UID and database") + + require.NoError(t, tenantB.RecordWatchCursor(ctx, "same-uid", gvr, "", "222")) + rv, ok = tenantA.LookupWatchCursor(ctx, "same-uid", gvr, "") + require.True(t, ok) + require.Equal(t, "111", rv, "tenant-b's write must not clobber tenant-a's cursor") +} + +// The assert-author verdict is the only thing that lets the controller honor a +// CommitRequest's spec.author, so it has to survive the Redis round trip. +func TestCommandAuthorStore_AssertAuthorAllowedRoundTrips(t *testing.T) { + t.Parallel() + + store := newPrefixedRedisStore(t, DefaultKeyPrefix) + commandStore := store.CommandAuthorStore() + ctx := context.Background() + + require.NoError(t, commandStore.RecordCommandAuthor(ctx, "cr-allowed", + CommandAuthor{Author: "gitops-api", AssertAuthorAllowed: true})) + got, ok := commandStore.LookupCommandAuthor(ctx, "cr-allowed") + require.True(t, ok) + require.True(t, got.AssertAuthorAllowed) + + // The zero value is the safe one: a record written before this field existed, or by a + // submitter that asserted nothing, must not authorize an assertion. + require.NoError(t, commandStore.RecordCommandAuthor(ctx, "cr-plain", CommandAuthor{Author: "alice"})) + got, ok = commandStore.LookupCommandAuthor(ctx, "cr-plain") + require.True(t, ok) + require.False(t, got.AssertAuthorAllowed) +} + +// The attribution telemetry gauge SCANs ":author:v1:audit:*". A prefix that +// contained a glob metacharacter would make it count the wrong keyspace; validation +// rejects those, so the pattern is always a literal prefix plus one trailing star. +func TestAttributionIndex_ScanPatternIsPrefixed(t *testing.T) { + t.Parallel() + + store := newPrefixedRedisStore(t, "tenant-a") + idx := store.AttributionIndex(0) + require.Equal(t, "tenant-a:author:v1:audit:", idx.factKeyBase("")) +} diff --git a/internal/queue/redis_store.go b/internal/queue/redis_store.go index 81e4b264..7d7ebca6 100644 --- a/internal/queue/redis_store.go +++ b/internal/queue/redis_store.go @@ -34,6 +34,11 @@ type RedisStoreConfig struct { AuthValue string DB int TLSEnabled bool + // KeyPrefix is the root namespace for every key this connection's stores own. Empty + // means DefaultKeyPrefix. Several reversers can then share one Redis/Valkey without + // sharing a logical database — Redis offers only 16 of those, and --redis-db was the + // only separator this operator provided. + KeyPrefix string } // RedisStore is the required Redis/Valkey-backed store. It owns the connection and @@ -42,7 +47,8 @@ type RedisStoreConfig struct { // and knows nothing about attribution: author attribution is an optional layer built // on the same connection via AttributionIndex. type RedisStore struct { - client *redis.Client + client *redis.Client + keyPrefix string } // NewRedisStore opens the Redis/Valkey connection that backs the resume cursors. @@ -61,7 +67,13 @@ func NewRedisStore(cfg RedisStoreConfig) (*RedisStore, error) { options.TLSConfig = &tls.Config{MinVersion: tls.VersionTLS12} } - return &RedisStore{client: redis.NewClient(options)}, nil + return &RedisStore{client: redis.NewClient(options), keyPrefix: resolveKeyPrefix(cfg.KeyPrefix)}, nil +} + +// KeyPrefix returns the root namespace every key this store's family owns is written +// under. Reported at startup so an operator can confirm which keyspace a pod claims. +func (s *RedisStore) KeyPrefix() string { + return s.keyPrefix } // AttributionIndex builds the optional author-attribution fact index on this store's @@ -71,7 +83,7 @@ func (s *RedisStore) AttributionIndex(factTTL time.Duration) *AttributionIndex { if factTTL <= 0 { factTTL = DefaultAttributionFactTTL } - return &AttributionIndex{client: s.client, factTTL: factTTL} + return &AttributionIndex{client: s.client, factTTL: factTTL, keyPrefix: s.keyPrefix} } // CommandAuthorStore builds the command-authorship store on this connection. Wire it @@ -79,7 +91,7 @@ func (s *RedisStore) AttributionIndex(factTTL time.Duration) *AttributionIndex { // record lives in the same top-level author domain as audit facts but in the separate // command subfamily, with its own fixed cleanup TTL. func (s *RedisStore) CommandAuthorStore() *CommandAuthorStore { - return &CommandAuthorStore{client: s.client, ttl: commandAuthorRecordTTL} + return &CommandAuthorStore{client: s.client, ttl: commandAuthorRecordTTL, keyPrefix: s.keyPrefix} } // Ping checks liveness of the underlying Redis/Valkey connection. The readiness gate @@ -122,6 +134,43 @@ func (s *RedisStore) RecordWatchCursor( return nil } +// watchCursorScanBatchSize bounds one SCAN round when forgetting a GitTarget's cursors. +const watchCursorScanBatchSize = 100 + +// ForgetWatchCursors deletes every resume cursor a GitTarget owns, so its next watch +// session rebuilds from a fresh replay instead of resuming mid-stream. +// +// It exists for the retarget: cursors are keyed by GitTarget UID, and a retarget keeps the +// same object — so without this the new folder would only ever receive the changes that +// happened after the move, never the state that already existed. Deletion is not needed on +// GitTarget deletion (a dead target's cursors expire on their own, and a recreated one has +// a new UID). +// +// A Kubernetes UID is hex and dashes, so the escaped uid can never introduce a glob +// metacharacter into the SCAN pattern; the key prefix is validated for the same reason. +func (s *RedisStore) ForgetWatchCursors(ctx context.Context, gitTargetUID string) error { + if strings.TrimSpace(gitTargetUID) == "" { + return nil + } + pattern := s.keyPrefix + watchCursorKeySuffix + "target:" + escapeKeyField(gitTargetUID) + ":*" + var cursor uint64 + for { + keys, next, err := s.client.Scan(ctx, cursor, pattern, watchCursorScanBatchSize).Result() + if err != nil { + return fmt.Errorf("scan watch cursors: %w", err) + } + if len(keys) > 0 { + if err := s.client.Del(ctx, keys...).Err(); err != nil { + return fmt.Errorf("delete watch cursors: %w", err) + } + } + cursor = next + if cursor == 0 { + return nil + } + } +} + // watchCursorKey builds a readable cursor key naming the store and leaf directly, e.g. // "gitops-reverser:watch:v1:target::apps/deployments:namespace:team-a:last-rv" or // "…:configmaps:cluster:last-rv". The GitTarget UID is globally unique, so its @@ -138,6 +187,6 @@ func (s *RedisStore) watchCursorKey( if namespace != "" { scope = "namespace:" + escapeKeyField(namespace) } - return keyPrefix + watchCursorKeySuffix + "target:" + escapeKeyField(gitTargetUID) + ":" + + return s.keyPrefix + watchCursorKeySuffix + "target:" + escapeKeyField(gitTargetUID) + ":" + groupResourceKey(gvr.Group, gvr.Resource) + ":" + scope + ":last-rv" } diff --git a/internal/queue/watch_cursor_forget_test.go b/internal/queue/watch_cursor_forget_test.go new file mode 100644 index 00000000..3198571f --- /dev/null +++ b/internal/queue/watch_cursor_forget_test.go @@ -0,0 +1,96 @@ +// SPDX-License-Identifier: Apache-2.0 + +package queue + +import ( + "context" + "testing" + + "github.com/alicebob/miniredis/v2" + "github.com/stretchr/testify/require" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +func deploymentsGVR() schema.GroupVersionResource { + return schema.GroupVersionResource{Group: "apps", Version: "v1", Resource: "deployments"} +} + +// A retarget keeps the same GitTarget object, so its cursors must be dropped explicitly: +// a resumed watch would deliver only the changes that happen after the move, and the new +// folder would never receive the state that already existed. +func TestForgetWatchCursors_DropsEveryCursorForOneTarget(t *testing.T) { + t.Parallel() + + store := newPrefixedRedisStore(t, DefaultKeyPrefix) + ctx := context.Background() + + require.NoError(t, store.RecordWatchCursor(ctx, "uid-a", coreConfigmapsGVR(), "team-a", "10")) + require.NoError(t, store.RecordWatchCursor(ctx, "uid-a", coreConfigmapsGVR(), "", "11")) + require.NoError(t, store.RecordWatchCursor(ctx, "uid-a", deploymentsGVR(), "team-a", "12")) + require.NoError(t, store.RecordWatchCursor(ctx, "uid-b", coreConfigmapsGVR(), "team-a", "20")) + + require.NoError(t, store.ForgetWatchCursors(ctx, "uid-a")) + + for _, scope := range []struct { + gvr schema.GroupVersionResource + namespace string + }{ + {coreConfigmapsGVR(), "team-a"}, + {coreConfigmapsGVR(), ""}, + {deploymentsGVR(), "team-a"}, + } { + _, ok := store.LookupWatchCursor(ctx, "uid-a", scope.gvr, scope.namespace) + require.False(t, ok, "every cursor for the retargeted GitTarget must be gone") + } + + // Another GitTarget's cursors are untouched: the key prefix is per-UID. + rv, ok := store.LookupWatchCursor(ctx, "uid-b", coreConfigmapsGVR(), "team-a") + require.True(t, ok) + require.Equal(t, "20", rv) +} + +func TestForgetWatchCursors_IsSafeWhenNothingIsStored(t *testing.T) { + t.Parallel() + + store := newPrefixedRedisStore(t, DefaultKeyPrefix) + require.NoError(t, store.ForgetWatchCursors(context.Background(), "never-declared")) +} + +func TestForgetWatchCursors_EmptyUIDIsANoOp(t *testing.T) { + t.Parallel() + + store := newPrefixedRedisStore(t, DefaultKeyPrefix) + ctx := context.Background() + require.NoError(t, store.RecordWatchCursor(ctx, "uid-a", coreConfigmapsGVR(), "", "10")) + + // A GitTarget that never declared has no UID here. Scanning "target::*" would match + // nothing, but returning early makes that impossible to get wrong. + require.NoError(t, store.ForgetWatchCursors(ctx, " ")) + + _, ok := store.LookupWatchCursor(ctx, "uid-a", coreConfigmapsGVR(), "") + require.True(t, ok) +} + +// Two reversers sharing a Redis must not delete each other's cursors. +func TestForgetWatchCursors_ScopedToTheKeyPrefix(t *testing.T) { + t.Parallel() + + mr := miniredis.RunT(t) + ctx := context.Background() + + tenantA, err := NewRedisStore(RedisStoreConfig{Addr: mr.Addr(), KeyPrefix: "tenant-a"}) + require.NoError(t, err) + tenantB, err := NewRedisStore(RedisStoreConfig{Addr: mr.Addr(), KeyPrefix: "tenant-b"}) + require.NoError(t, err) + + require.NoError(t, tenantA.RecordWatchCursor(ctx, "same-uid", coreConfigmapsGVR(), "", "10")) + require.NoError(t, tenantB.RecordWatchCursor(ctx, "same-uid", coreConfigmapsGVR(), "", "20")) + + require.NoError(t, tenantA.ForgetWatchCursors(ctx, "same-uid")) + + _, ok := tenantA.LookupWatchCursor(ctx, "same-uid", coreConfigmapsGVR(), "") + require.False(t, ok) + rv, ok := tenantB.LookupWatchCursor(ctx, "same-uid", coreConfigmapsGVR(), "") + require.True(t, ok) + require.Equal(t, "20", rv, "tenant-b's cursor survives tenant-a's retarget") +} diff --git a/internal/rulestore/store.go b/internal/rulestore/store.go index ead71758..b0267803 100644 --- a/internal/rulestore/store.go +++ b/internal/rulestore/store.go @@ -30,6 +30,11 @@ type CompiledRule struct { GitProviderNamespace string Branch string Path string + // SourceCluster is the cluster this rule's GitTarget mirrors FROM, resolved from + // GitTarget.spec.sourceCluster. Empty is the cluster the operator runs in. It is + // resolved here rather than looked up per event because the watch data plane keys its + // per-cluster clients, catalogs, and registries by it. + SourceCluster string // IsClusterScoped indicates if this rule watches cluster-scoped resources. // Always false for WatchRule (namespace-scoped). @@ -48,6 +53,11 @@ type CompiledResourceRule struct { APIVersions []string // Resources specifies which resource types this rule matches. Resources []string + // ExcludeFieldManagers names field managers whose writes this rule declines to mirror. + // It is a veto within this rule, not a global filter (rules are a logical OR). + ExcludeFieldManagers []string + // ExcludeUsers names attributed identities whose writes this rule declines to mirror. + ExcludeUsers []string } // CompiledClusterRule represents a fully processed ClusterWatchRule, ready for quick lookups. @@ -64,6 +74,9 @@ type CompiledClusterRule struct { GitProviderNamespace string Branch string Path string + // SourceCluster is the cluster this rule's GitTarget mirrors FROM. Empty is the + // cluster the operator runs in. + SourceCluster string // Rules contains the compiled cluster resource rules with per-rule scope. Rules []CompiledClusterResourceRule @@ -79,6 +92,10 @@ type CompiledClusterResourceRule struct { APIVersions []string // Resources specifies which resource types this rule matches. Resources []string + // ExcludeFieldManagers names field managers whose writes this rule declines to mirror. + ExcludeFieldManagers []string + // ExcludeUsers names attributed identities whose writes this rule declines to mirror. + ExcludeUsers []string // Scope indicates whether this rule watches Cluster or Namespaced resources. Scope configv1alpha3.ResourceScope } @@ -100,25 +117,48 @@ func NewStore() *RuleStore { } } -// AddOrUpdateWatchRule adds or updates a WatchRule with a resolved target from GitTarget. -// The chain is: WatchRule -> GitTarget -> GitProvider -// Parameters: -// - rule: the WatchRule to add or update -// - gitTargetName: the name of the GitTarget -// - gitTargetNamespace: the namespace containing the GitTarget -// - gitProviderName: the name of the resolved GitProvider (from GitTarget.Spec.Provider) -// - gitProviderNamespace: the namespace containing the resolved GitProvider -// - branch: the Git branch to write to (from GitTarget.Spec.Branch) -// - path: POSIX-like relative path prefix for writes (from GitTarget.Spec.Path, sanitized upstream) -func (s *RuleStore) AddOrUpdateWatchRule( - rule configv1alpha3.WatchRule, - gitTargetName string, - gitTargetNamespace string, - gitProviderName string, - gitProviderNamespace string, - branch string, - path string, -) { +// TargetBinding is the GitTarget-derived context a rule compiles against: where its +// events are written (provider, branch, path) and which cluster they are watched on. It +// replaces a positional argument list that had grown to six strings, where a transposed +// pair would have silently mirrored the wrong folder. +type TargetBinding struct { + // GitTargetName / GitTargetNamespace identify the GitTarget the rule points at. + GitTargetName string + GitTargetNamespace string + // GitProviderName / GitProviderNamespace are resolved from GitTarget.spec.providerRef. + GitProviderName string + GitProviderNamespace string + // Branch and Path are the GitTarget's destination. Path is POSIX-like and relative, + // sanitized upstream. + Branch string + Path string + // SourceCluster is the cluster the GitTarget mirrors FROM. Empty is the cluster the + // operator runs in. + SourceCluster string +} + +// NewTargetBinding derives a rule's binding from the GitTarget it points at and the +// GitProvider that GitTarget resolves to. Every caller (the two controllers and the +// manager's bootstrap) goes through it, so the six fields can never be assembled two +// different ways. +func NewTargetBinding( + target configv1alpha3.GitTarget, + provider configv1alpha3.GitProvider, +) TargetBinding { + return TargetBinding{ + GitTargetName: target.Name, + GitTargetNamespace: target.Namespace, + GitProviderName: provider.Name, + GitProviderNamespace: provider.Namespace, + Branch: target.Spec.Branch, + Path: target.Spec.Path, + SourceCluster: target.SourceClusterID(), + } +} + +// AddOrUpdateWatchRule adds or updates a WatchRule with its resolved GitTarget binding. +// The chain is: WatchRule -> GitTarget -> GitProvider. +func (s *RuleStore) AddOrUpdateWatchRule(rule configv1alpha3.WatchRule, binding TargetBinding) { s.mu.Lock() defer s.mu.Unlock() @@ -129,47 +169,34 @@ func (s *RuleStore) AddOrUpdateWatchRule( compiled := CompiledRule{ Source: key, - GitTargetRef: gitTargetName, - GitTargetNamespace: gitTargetNamespace, - GitProviderRef: gitProviderName, - GitProviderNamespace: gitProviderNamespace, - Branch: branch, - Path: path, + GitTargetRef: binding.GitTargetName, + GitTargetNamespace: binding.GitTargetNamespace, + GitProviderRef: binding.GitProviderName, + GitProviderNamespace: binding.GitProviderNamespace, + Branch: binding.Branch, + Path: binding.Path, + SourceCluster: binding.SourceCluster, IsClusterScoped: false, ResourceRules: make([]CompiledResourceRule, 0, len(rule.Spec.Rules)), } for _, r := range rule.Spec.Rules { compiled.ResourceRules = append(compiled.ResourceRules, CompiledResourceRule{ - Operations: r.Operations, - APIGroups: r.APIGroups, - APIVersions: r.APIVersions, - Resources: r.Resources, + Operations: r.Operations, + APIGroups: r.APIGroups, + APIVersions: r.APIVersions, + Resources: r.Resources, + ExcludeFieldManagers: r.ExcludeFieldManagers, + ExcludeUsers: r.ExcludeUsers, }) } s.rules[key] = compiled } -// AddOrUpdateClusterWatchRule adds or updates a ClusterWatchRule with a resolved target from GitTarget. -// The chain is: ClusterWatchRule -> GitTarget -> GitProvider -// Parameters: -// - rule: the ClusterWatchRule to add or update -// - gitTargetName: the name of the GitTarget -// - gitTargetNamespace: the namespace containing the GitTarget -// - gitProviderName: the name of the resolved GitProvider (from GitTarget.Spec.Provider) -// - gitProviderNamespace: the namespace containing the resolved GitProvider -// - branch: the Git branch to write to (from GitTarget.Spec.Branch) -// - path: POSIX-like relative path prefix for writes (from GitTarget.Spec.Path, sanitized upstream) -func (s *RuleStore) AddOrUpdateClusterWatchRule( - rule configv1alpha3.ClusterWatchRule, - gitTargetName string, - gitTargetNamespace string, - gitProviderName string, - gitProviderNamespace string, - branch string, - path string, -) { +// AddOrUpdateClusterWatchRule adds or updates a ClusterWatchRule with its resolved +// GitTarget binding. The chain is: ClusterWatchRule -> GitTarget -> GitProvider. +func (s *RuleStore) AddOrUpdateClusterWatchRule(rule configv1alpha3.ClusterWatchRule, binding TargetBinding) { s.mu.Lock() defer s.mu.Unlock() @@ -180,22 +207,25 @@ func (s *RuleStore) AddOrUpdateClusterWatchRule( compiled := CompiledClusterRule{ Source: key, - GitTargetRef: gitTargetName, - GitTargetNamespace: gitTargetNamespace, - GitProviderRef: gitProviderName, - GitProviderNamespace: gitProviderNamespace, - Branch: branch, - Path: path, + GitTargetRef: binding.GitTargetName, + GitTargetNamespace: binding.GitTargetNamespace, + GitProviderRef: binding.GitProviderName, + GitProviderNamespace: binding.GitProviderNamespace, + Branch: binding.Branch, + Path: binding.Path, + SourceCluster: binding.SourceCluster, Rules: make([]CompiledClusterResourceRule, 0, len(rule.Spec.Rules)), } for _, r := range rule.Spec.Rules { compiled.Rules = append(compiled.Rules, CompiledClusterResourceRule{ - Operations: r.Operations, - APIGroups: r.APIGroups, - APIVersions: r.APIVersions, - Resources: r.Resources, - Scope: r.Scope, + Operations: r.Operations, + APIGroups: r.APIGroups, + APIVersions: r.APIVersions, + Resources: r.Resources, + ExcludeFieldManagers: r.ExcludeFieldManagers, + ExcludeUsers: r.ExcludeUsers, + Scope: r.Scope, }) } diff --git a/internal/rulestore/store_test.go b/internal/rulestore/store_test.go index 8edad8fc..0e663420 100644 --- a/internal/rulestore/store_test.go +++ b/internal/rulestore/store_test.go @@ -47,15 +47,14 @@ func TestAddOrUpdateWatchRule(t *testing.T) { rule.Namespace = "default" // Add rule - store.AddOrUpdateWatchRule( - rule, - "test-target", - "default", - "test-provider", - "gitops-system", - "main", - "clusters/prod", - ) + store.AddOrUpdateWatchRule(rule, TargetBinding{ + GitTargetName: "test-target", + GitTargetNamespace: "default", + GitProviderName: "test-provider", + GitProviderNamespace: "gitops-system", + Branch: "main", + Path: "clusters/prod", + }) // Verify it was added key := types.NamespacedName{Name: "test-rule", Namespace: "default"} @@ -88,15 +87,14 @@ func TestAddOrUpdateWatchRule(t *testing.T) { } // Update rule with different values - store.AddOrUpdateWatchRule( - rule, - "test-target", - "default", - "updated-provider", - "gitops-system", - "develop", - "clusters/staging", - ) + store.AddOrUpdateWatchRule(rule, TargetBinding{ + GitTargetName: "test-target", + GitTargetNamespace: "default", + GitProviderName: "updated-provider", + GitProviderNamespace: "gitops-system", + Branch: "develop", + Path: "clusters/staging", + }) compiled, exists = store.rules[key] if !exists { @@ -133,15 +131,14 @@ func TestAddOrUpdateClusterWatchRule(t *testing.T) { rule.Name = "test-cluster-rule" // Add rule - store.AddOrUpdateClusterWatchRule( - rule, - "test-cluster-target", - "gitops-system", - "cluster-provider", - "gitops-system", - "main", - "cluster/audit", - ) + store.AddOrUpdateClusterWatchRule(rule, TargetBinding{ + GitTargetName: "test-cluster-target", + GitTargetNamespace: "gitops-system", + GitProviderName: "cluster-provider", + GitProviderNamespace: "gitops-system", + Branch: "main", + Path: "cluster/audit", + }) // Verify it was added key := types.NamespacedName{Name: "test-cluster-rule", Namespace: ""} @@ -188,7 +185,14 @@ func TestDelete(t *testing.T) { key := types.NamespacedName{Name: "delete-test", Namespace: "default"} // Add rule - store.AddOrUpdateWatchRule(rule, "test-dest", "default", "test-repo", "gitops-system", "main", "test") + store.AddOrUpdateWatchRule(rule, TargetBinding{ + GitTargetName: "test-dest", + GitTargetNamespace: "default", + GitProviderName: "test-repo", + GitProviderNamespace: "gitops-system", + Branch: "main", + Path: "test", + }) // Verify it exists if _, exists := store.rules[key]; !exists { @@ -223,7 +227,14 @@ func TestDeleteClusterWatchRule(t *testing.T) { key := types.NamespacedName{Name: "delete-cluster-test", Namespace: ""} // Add rule - store.AddOrUpdateClusterWatchRule(rule, "test-dest", "gitops-system", "test-repo", "gitops-system", "main", "test") + store.AddOrUpdateClusterWatchRule(rule, TargetBinding{ + GitTargetName: "test-dest", + GitTargetNamespace: "gitops-system", + GitProviderName: "test-repo", + GitProviderNamespace: "gitops-system", + Branch: "main", + Path: "test", + }) // Verify it exists if _, exists := store.clusterRules[key]; !exists { @@ -274,8 +285,22 @@ func TestGetMatchingRules(t *testing.T) { rule2.Name = "deployment-rule" rule2.Namespace = "default" - store.AddOrUpdateWatchRule(rule1, "dest1", "default", "repo1", "gitops-system", "main", "test1") - store.AddOrUpdateWatchRule(rule2, "dest2", "default", "repo2", "gitops-system", "main", "test2") + store.AddOrUpdateWatchRule(rule1, TargetBinding{ + GitTargetName: "dest1", + GitTargetNamespace: "default", + GitProviderName: "repo1", + GitProviderNamespace: "gitops-system", + Branch: "main", + Path: "test1", + }) + store.AddOrUpdateWatchRule(rule2, TargetBinding{ + GitTargetName: "dest2", + GitTargetNamespace: "default", + GitProviderName: "repo2", + GitProviderNamespace: "gitops-system", + Branch: "main", + Path: "test2", + }) tests := []struct { name string @@ -409,15 +434,30 @@ func TestGetMatchingRules_OverlappingRulesUnionOperations(t *testing.T) { // Two rules covering the same resource with complementary operations, plus a // second CREATE rule to prove matches are additive rather than first-wins. - store.AddOrUpdateWatchRule( - podsRule("pod-create", configv1alpha3.OperationCreate), - "dest-a", "default", "repo", "gitops-system", "main", "a") - store.AddOrUpdateWatchRule( - podsRule("pod-update", configv1alpha3.OperationUpdate), - "dest-a", "default", "repo", "gitops-system", "main", "a") - store.AddOrUpdateWatchRule( - podsRule("pod-create-2", configv1alpha3.OperationCreate), - "dest-b", "default", "repo", "gitops-system", "main", "b") + store.AddOrUpdateWatchRule(podsRule("pod-create", configv1alpha3.OperationCreate), TargetBinding{ + GitTargetName: "dest-a", + GitTargetNamespace: "default", + GitProviderName: "repo", + GitProviderNamespace: "gitops-system", + Branch: "main", + Path: "a", + }) + store.AddOrUpdateWatchRule(podsRule("pod-update", configv1alpha3.OperationUpdate), TargetBinding{ + GitTargetName: "dest-a", + GitTargetNamespace: "default", + GitProviderName: "repo", + GitProviderNamespace: "gitops-system", + Branch: "main", + Path: "a", + }) + store.AddOrUpdateWatchRule(podsRule("pod-create-2", configv1alpha3.OperationCreate), TargetBinding{ + GitTargetName: "dest-b", + GitTargetNamespace: "default", + GitProviderName: "repo", + GitProviderNamespace: "gitops-system", + Branch: "main", + Path: "b", + }) tests := []struct { name string @@ -522,24 +562,22 @@ func TestGetMatchingClusterRules(t *testing.T) { } namespacedRule.Name = "pod-cluster-rule" - store.AddOrUpdateClusterWatchRule( - clusterRule, - "dest1", - "gitops-system", - "repo1", - "gitops-system", - "main", - "cluster", - ) - store.AddOrUpdateClusterWatchRule( - namespacedRule, - "dest2", - "gitops-system", - "repo2", - "gitops-system", - "main", - "namespaced", - ) + store.AddOrUpdateClusterWatchRule(clusterRule, TargetBinding{ + GitTargetName: "dest1", + GitTargetNamespace: "gitops-system", + GitProviderName: "repo1", + GitProviderNamespace: "gitops-system", + Branch: "main", + Path: "cluster", + }) + store.AddOrUpdateClusterWatchRule(namespacedRule, TargetBinding{ + GitTargetName: "dest2", + GitTargetNamespace: "gitops-system", + GitProviderName: "repo2", + GitProviderNamespace: "gitops-system", + Branch: "main", + Path: "namespaced", + }) tests := []struct { name string @@ -789,8 +827,22 @@ func TestSnapshotWatchRules(t *testing.T) { rule2.Name = "rule2" rule2.Namespace = "default" - store.AddOrUpdateWatchRule(rule1, "dest1", "default", "repo1", "gitops-system", "main", "test1") - store.AddOrUpdateWatchRule(rule2, "dest2", "default", "repo2", "gitops-system", "main", "test2") + store.AddOrUpdateWatchRule(rule1, TargetBinding{ + GitTargetName: "dest1", + GitTargetNamespace: "default", + GitProviderName: "repo1", + GitProviderNamespace: "gitops-system", + Branch: "main", + Path: "test1", + }) + store.AddOrUpdateWatchRule(rule2, TargetBinding{ + GitTargetName: "dest2", + GitTargetNamespace: "default", + GitProviderName: "repo2", + GitProviderNamespace: "gitops-system", + Branch: "main", + Path: "test2", + }) // Get snapshot snapshot := store.SnapshotWatchRules() @@ -827,7 +879,14 @@ func TestSnapshotClusterWatchRules(t *testing.T) { } rule1.Name = "cluster-rule1" - store.AddOrUpdateClusterWatchRule(rule1, "dest1", "gitops-system", "repo1", "gitops-system", "main", "cluster") + store.AddOrUpdateClusterWatchRule(rule1, TargetBinding{ + GitTargetName: "dest1", + GitTargetNamespace: "gitops-system", + GitProviderName: "repo1", + GitProviderNamespace: "gitops-system", + Branch: "main", + Path: "cluster", + }) snapshot := store.SnapshotClusterWatchRules() @@ -867,7 +926,14 @@ func TestConcurrentAccess(t *testing.T) { rule.Name = "concurrent-rule" rule.Namespace = "default" - store.AddOrUpdateWatchRule(rule, "dest", "default", "repo", "gitops-system", "main", "test") + store.AddOrUpdateWatchRule(rule, TargetBinding{ + GitTargetName: "dest", + GitTargetNamespace: "default", + GitProviderName: "repo", + GitProviderNamespace: "gitops-system", + Branch: "main", + Path: "test", + }) } }(i) } @@ -914,7 +980,14 @@ func TestMultipleResourceRules(t *testing.T) { rule.Name = "multi-rule" rule.Namespace = "default" - store.AddOrUpdateWatchRule(rule, "dest", "default", "repo", "gitops-system", "main", "test") + store.AddOrUpdateWatchRule(rule, TargetBinding{ + GitTargetName: "dest", + GitTargetNamespace: "default", + GitProviderName: "repo", + GitProviderNamespace: "gitops-system", + Branch: "main", + Path: "test", + }) // Should match pod CREATE matches := store.GetMatchingRules(nil, "pods", configv1alpha3.OperationCreate, "", "v1", false) @@ -955,7 +1028,14 @@ func TestGetMatchingRules_MustFilterByNamespaceForNamespacedWatchRule(t *testing } rule.Name = "playground-wr" rule.Namespace = "tilt-playground" - store.AddOrUpdateWatchRule(rule, "target", "tilt-playground", "provider", "tilt-playground", "main", "live") + store.AddOrUpdateWatchRule(rule, TargetBinding{ + GitTargetName: "target", + GitTargetNamespace: "tilt-playground", + GitProviderName: "provider", + GitProviderNamespace: "tilt-playground", + Branch: "main", + Path: "live", + }) sameNSObject := &unstructured.Unstructured{} sameNSObject.SetNamespace("tilt-playground") @@ -992,24 +1072,35 @@ func TestGetMatchingRules_NamespacedWatchRule_NamespaceContract(t *testing.T) { } rule.Name = "ns-wr" rule.Namespace = "tilt-playground" - store.AddOrUpdateWatchRule(rule, "tgt", "tilt-playground", "prov", "tilt-playground", "main", "live") - - store.AddOrUpdateClusterWatchRule( - configv1alpha3.ClusterWatchRule{ - ObjectMeta: metav1.ObjectMeta{Name: "cluster-services"}, - Spec: configv1alpha3.ClusterWatchRuleSpec{ - TargetRef: configv1alpha3.NamespacedTargetReference{Name: "cluster-target", Namespace: "ops"}, - Rules: []configv1alpha3.ClusterResourceRule{{ - Operations: []configv1alpha3.OperationType{configv1alpha3.OperationAll}, - APIGroups: []string{""}, - APIVersions: []string{"v1"}, - Resources: []string{"services"}, - Scope: configv1alpha3.ResourceScopeNamespaced, - }}, - }, + store.AddOrUpdateWatchRule(rule, TargetBinding{ + GitTargetName: "tgt", + GitTargetNamespace: "tilt-playground", + GitProviderName: "prov", + GitProviderNamespace: "tilt-playground", + Branch: "main", + Path: "live", + }) + + store.AddOrUpdateClusterWatchRule(configv1alpha3.ClusterWatchRule{ + ObjectMeta: metav1.ObjectMeta{Name: "cluster-services"}, + Spec: configv1alpha3.ClusterWatchRuleSpec{ + TargetRef: configv1alpha3.NamespacedTargetReference{Name: "cluster-target", Namespace: "ops"}, + Rules: []configv1alpha3.ClusterResourceRule{{ + Operations: []configv1alpha3.OperationType{configv1alpha3.OperationAll}, + APIGroups: []string{""}, + APIVersions: []string{"v1"}, + Resources: []string{"services"}, + Scope: configv1alpha3.ResourceScopeNamespaced, + }}, }, - "cluster-target", "ops", "cluster-provider", "ops", "main", "cluster", - ) + }, TargetBinding{ + GitTargetName: "cluster-target", + GitTargetNamespace: "ops", + GitProviderName: "cluster-provider", + GitProviderNamespace: "ops", + Branch: "main", + Path: "cluster", + }) t.Run("same namespace matches", func(t *testing.T) { obj := &unstructured.Unstructured{} diff --git a/internal/telemetry/exporter.go b/internal/telemetry/exporter.go index d13a8212..bbf8a85c 100644 --- a/internal/telemetry/exporter.go +++ b/internal/telemetry/exporter.go @@ -66,6 +66,17 @@ var ( // spec's drain wait; treat the name/labels as a public observability contract. BranchWorkerQueueDepth metric.Int64Gauge + // WatchEventsExcludedTotal counts live watch events a rule's write exclusion dropped + // before they reached a branch worker. Labelled by {gittarget_namespace, + // gittarget_name, group, resource, reason} where reason is `field_manager` or `user`. + // A steady non-zero rate is the healthy state for a GitTarget paired with a GitOps + // forward leg: it is the forward leg's own applies being declined instead of + // committed back. A rate of zero on such a GitTarget means the exclusion is not + // matching — check the field manager name against the object's managedFields. + // The gittarget_ label prefix avoids the reserved Prometheus pod-scrape labels, as + // for TargetReconcileCompletedTotal. + WatchEventsExcludedTotal metric.Int64Counter + // ResyncBackgroundFailuresTotal counts rule-change resyncs whose apply failed or // timed out at the worker AFTER being enqueued. Delivery is marked on enqueue (the // resync is fire-and-forget to avoid an unbounded re-gather loop — see @@ -210,6 +221,7 @@ func registerCounters() error { {"gitopsreverser_commits_total", &CommitsTotal}, {"gitopsreverser_resync_sweep_deletes_total", &ResyncSweepDeletesTotal}, {"gitopsreverser_target_reconcile_completed_total", &TargetReconcileCompletedTotal}, + {"gitopsreverser_watch_events_excluded_total", &WatchEventsExcludedTotal}, {"gitopsreverser_resync_background_failures_total", &ResyncBackgroundFailuresTotal}, {"gitopsreverser_audit_events_total", &AuditEventsTotal}, {"gitopsreverser_audit_eventlists_total", &AuditEventListsTotal}, diff --git a/internal/watch/api_resource_catalog.go b/internal/watch/api_resource_catalog.go index 1b100061..12c76deb 100644 --- a/internal/watch/api_resource_catalog.go +++ b/internal/watch/api_resource_catalog.go @@ -101,6 +101,44 @@ func (c *APIResourceCatalog) Scan(sensitive types.SensitiveResourcePolicy) (type return scan, true } +// ServesWatchable reports whether the latest scan saw this exact group/version/resource +// served with both list and watch verbs. It is the precondition for opening an informer +// on a resource the operator does not require: an API server that does not aggregate does +// not serve apiregistration.k8s.io at all, and a blind informer on it retries forever. +// It deliberately ignores the allow/deny watch policy, which governs what the operator +// mirrors, not what it may inspect. Reports false before the first trusted scan. +func (c *APIResourceCatalog) ServesWatchable(gvr schema.GroupVersionResource) bool { + c.mu.RLock() + defer c.mu.RUnlock() + if !c.ready { + return false + } + for _, entry := range c.lastScan.Entries { + if entry.GVR != gvr { + continue + } + return hasVerbs(entry.Verbs, "list", "watch") + } + return false +} + +// hasVerbs reports whether every wanted verb is present in the sorted verb list. +func hasVerbs(verbs []string, wanted ...string) bool { + for _, want := range wanted { + found := false + for _, verb := range verbs { + if verb == want { + found = true + break + } + } + if !found { + return false + } + } + return true +} + // DegradedGroupVersions returns the group/versions the latest scan reported as // failed, sorted for stable logging. func (c *APIResourceCatalog) DegradedGroupVersions() []schema.GroupVersion { diff --git a/internal/watch/api_resource_catalog_test.go b/internal/watch/api_resource_catalog_test.go index ebf364f2..7560141d 100644 --- a/internal/watch/api_resource_catalog_test.go +++ b/internal/watch/api_resource_catalog_test.go @@ -141,7 +141,14 @@ func TestNotServedResourceProducesNoGVR(t *testing.T) { Resources: []string{"customresources"}, }}, }, - }, "target", "default", "provider", "default", "main", "live") + }, rulestore.TargetBinding{ + GitTargetName: "target", + GitTargetNamespace: "default", + GitProviderName: "provider", + GitProviderNamespace: "default", + Branch: "main", + Path: "live", + }) manager := &Manager{RuleStore: store, resourceCatalog: newCommonTestCatalog(t)} assert.Empty(t, manager.ComputeRequestedGVRs()) diff --git a/internal/watch/api_surface_triggers_test.go b/internal/watch/api_surface_triggers_test.go new file mode 100644 index 00000000..f10a9c06 --- /dev/null +++ b/internal/watch/api_surface_triggers_test.go @@ -0,0 +1,161 @@ +// SPDX-License-Identifier: Apache-2.0 + +package watch + +import ( + "testing" + + "github.com/go-logr/logr" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +// aggregatingDiscovery is the common test surface plus a served apiregistration.k8s.io/v1, +// i.e. an ordinary kube-apiserver rather than a control plane without an aggregation layer. +func aggregatingDiscovery() staticCatalogDiscovery { + disco := newCommonTestDiscovery() + disco.groups = append(disco.groups, testAPIGroup("apiregistration.k8s.io", "v1")) + disco.resources = append(disco.resources, &metav1.APIResourceList{ + GroupVersion: "apiregistration.k8s.io/v1", + APIResources: []metav1.APIResource{ + {Name: "apiservices", Kind: "APIService", Verbs: metav1.Verbs{"get", "list", "watch"}}, + }, + }) + return disco +} + +func TestAPIResourceCatalog_ServesWatchable(t *testing.T) { + t.Parallel() + + tests := map[string]struct { + discovery staticCatalogDiscovery + gvr schema.GroupVersionResource + want bool + }{ + "crd served on a stock apiserver": { + discovery: newCommonTestDiscovery(), + gvr: crdTriggerGVR(), + want: true, + }, + "apiservices absent without an aggregation layer": { + discovery: newCommonTestDiscovery(), + gvr: apiServiceTriggerGVR(), + want: false, + }, + "apiservices served when the group is aggregated": { + discovery: aggregatingDiscovery(), + gvr: apiServiceTriggerGVR(), + want: true, + }, + "served but not watchable": { + discovery: staticCatalogDiscovery{ + groups: []*metav1.APIGroup{testAPIGroup("apiregistration.k8s.io", "v1")}, + resources: []*metav1.APIResourceList{{ + GroupVersion: "apiregistration.k8s.io/v1", + APIResources: []metav1.APIResource{ + {Name: "apiservices", Kind: "APIService", Verbs: metav1.Verbs{"get", "list"}}, + }, + }}, + }, + gvr: apiServiceTriggerGVR(), + want: false, + }, + "unknown resource": { + discovery: newCommonTestDiscovery(), + gvr: schema.GroupVersionResource{Group: "nope.example.com", Version: "v1", Resource: "widgets"}, + want: false, + }, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + t.Parallel() + catalog := NewAPIResourceCatalog() + _, err := catalog.Refresh(tc.discovery) + require.NoError(t, err) + require.Equal(t, tc.want, catalog.ServesWatchable(tc.gvr)) + }) + } +} + +func TestAPIResourceCatalog_ServesWatchable_BeforeFirstScan(t *testing.T) { + t.Parallel() + // A catalog that has never accepted a trusted scan must not claim anything is + // watchable, or the manager would open an informer on an API surface it has not seen. + require.False(t, NewAPIResourceCatalog().ServesWatchable(crdTriggerGVR())) +} + +func TestSelectAPISurfaceTriggers_SkipsUnservedAPIServices(t *testing.T) { + t.Parallel() + + catalog := newCommonTestCatalog(t) + start, unserved := selectAPISurfaceTriggers(catalog, map[schema.GroupVersionResource]struct{}{}) + + require.Equal(t, []schema.GroupVersionResource{crdTriggerGVR()}, start, + "only the served CRD trigger should start") + require.Equal(t, []schema.GroupVersionResource{apiServiceTriggerGVR()}, unserved, + "apiservices must be reported unserved, not started blindly") +} + +func TestSelectAPISurfaceTriggers_StartsBothWhenAggregationIsServed(t *testing.T) { + t.Parallel() + + catalog := NewAPIResourceCatalog() + _, err := catalog.Refresh(aggregatingDiscovery()) + require.NoError(t, err) + + start, unserved := selectAPISurfaceTriggers(catalog, map[schema.GroupVersionResource]struct{}{}) + + require.ElementsMatch(t, apiSurfaceTriggerGVRs(), start) + require.Empty(t, unserved) +} + +func TestSelectAPISurfaceTriggers_NeverRestartsARunningInformer(t *testing.T) { + t.Parallel() + + catalog := NewAPIResourceCatalog() + _, err := catalog.Refresh(aggregatingDiscovery()) + require.NoError(t, err) + + started := map[schema.GroupVersionResource]struct{}{crdTriggerGVR(): {}} + start, unserved := selectAPISurfaceTriggers(catalog, started) + + require.Equal(t, []schema.GroupVersionResource{apiServiceTriggerGVR()}, start) + require.Empty(t, unserved) +} + +// An aggregation layer installed after startup must be picked up on the next catalog +// refresh, without restarting the operator. +func TestSelectAPISurfaceTriggers_PicksUpAggregationInstalledLater(t *testing.T) { + t.Parallel() + + catalog := newCommonTestCatalog(t) + started := map[schema.GroupVersionResource]struct{}{} + + start, unserved := selectAPISurfaceTriggers(catalog, started) + require.Equal(t, []schema.GroupVersionResource{crdTriggerGVR()}, start) + require.Equal(t, []schema.GroupVersionResource{apiServiceTriggerGVR()}, unserved) + for _, gvr := range start { + started[gvr] = struct{}{} + } + + _, err := catalog.Refresh(aggregatingDiscovery()) + require.NoError(t, err) + + start, unserved = selectAPISurfaceTriggers(catalog, started) + require.Equal(t, []schema.GroupVersionResource{apiServiceTriggerGVR()}, start) + require.Empty(t, unserved) +} + +func TestEnsureAPISurfaceTriggerInformers_NoOpBeforeStart(t *testing.T) { + t.Parallel() + + // triggerCtx is nil until Start runs. Calling in through a controller-driven catalog + // refresh before then must not create a factory bound to a dead context. + m := &Manager{Log: logr.Discard()} + m.ensureAPISurfaceTriggerInformers(t.Context(), m.localCluster(), m.Log) + + require.Nil(t, m.localCluster().triggerFactory) + require.Empty(t, m.localCluster().triggersStarted) +} diff --git a/internal/watch/author_resolver.go b/internal/watch/author_resolver.go index 26100c31..e5e61242 100644 --- a/internal/watch/author_resolver.go +++ b/internal/watch/author_resolver.go @@ -46,8 +46,10 @@ type AttributionLookup interface { // CursorStore persists the last processed resourceVersion for each (GitTarget UID, // GVR, scope) watch shard, bounded by a TTL. The GitTarget is identified by its UID // alone — globally unique, so namespace/name would be redundant. Cursors are refreshed -// on write and never deleted: a live watch keeps its cursor fresh, a dead one's cursor -// expires. Nil means every new watch session rebuilds from a fresh replay. +// on write and expire on their own: a live watch keeps its cursor fresh, a dead one's +// cursor ages out. Nil means every new watch session rebuilds from a fresh replay. +// +// The one case that deletes a cursor is a retarget, through the optional CursorForgetter. type CursorStore interface { LookupWatchCursor( ctx context.Context, @@ -63,6 +65,14 @@ type CursorStore interface { ) error } +// CursorForgetter is the optional half of CursorStore a retarget needs: dropping every +// cursor a GitTarget owns, so its next watch session rebuilds the new folder from a full +// replay rather than resuming mid-stream into it. *queue.RedisStore satisfies it; a store +// that does not is simply never asked (without Redis, watches cold-replay regardless). +type CursorForgetter interface { + ForgetWatchCursors(ctx context.Context, gitTargetUID string) error +} + // AuthorResolver names the commit author for a live watch event from audit facts. type AuthorResolver interface { // ResolveAuthor returns the author UserInfo for a watch event, or ok=false to diff --git a/internal/watch/bootstrap.go b/internal/watch/bootstrap.go index 366b82d5..c7d0b149 100644 --- a/internal/watch/bootstrap.go +++ b/internal/watch/bootstrap.go @@ -10,6 +10,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" configv1alpha3 "github.com/ConfigButler/gitops-reverser/api/v1alpha3" + "github.com/ConfigButler/gitops-reverser/internal/rulestore" ) // bootstrapRuleStore loads existing WatchRule and ClusterWatchRule objects into the in-memory RuleStore @@ -57,13 +58,7 @@ func (m *Manager) bootstrapWatchRule(ctx context.Context, rule configv1alpha3.Wa return err } - m.RuleStore.AddOrUpdateWatchRule( - rule, - target.Name, target.Namespace, - provider.Name, provider.Namespace, - target.Spec.Branch, - target.Spec.Path, - ) + m.RuleStore.AddOrUpdateWatchRule(rule, rulestore.NewTargetBinding(target, provider)) return nil } @@ -79,13 +74,7 @@ func (m *Manager) bootstrapClusterWatchRule(ctx context.Context, rule configv1al return err } - m.RuleStore.AddOrUpdateClusterWatchRule( - rule, - target.Name, target.Namespace, - provider.Name, provider.Namespace, - target.Spec.Branch, - target.Spec.Path, - ) + m.RuleStore.AddOrUpdateClusterWatchRule(rule, rulestore.NewTargetBinding(target, provider)) return nil } diff --git a/internal/watch/cluster_context.go b/internal/watch/cluster_context.go new file mode 100644 index 00000000..5fe81cfb --- /dev/null +++ b/internal/watch/cluster_context.go @@ -0,0 +1,405 @@ +// SPDX-License-Identifier: Apache-2.0 + +package watch + +import ( + "context" + "fmt" + "sort" + "sync" + + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/discovery" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/dynamic/dynamicinformer" + "k8s.io/client-go/rest" + ctrl "sigs.k8s.io/controller-runtime" + + "github.com/ConfigButler/gitops-reverser/internal/types" + "github.com/ConfigButler/gitops-reverser/internal/typeset" +) + +// LocalClusterID identifies the cluster the operator runs in — the config plane, which is +// also the watched cluster in a single-cluster install. It is the zero value on purpose: +// every code path that does not know about source clusters lands on it. +const LocalClusterID = "" + +// SourceClusterResolver turns a GitTarget's source-cluster id into a rest.Config by reading +// the kubeconfig Secret it names from the config plane. It is an interface so the watch +// manager grows no Kubernetes client of its own for this, and so tests can stand up a +// remote cluster without a Secret. +type SourceClusterResolver interface { + // ResolveSourceCluster returns the rest.Config for a cluster id, and an opaque version + // token that changes when the underlying kubeconfig changes (the Secret's + // resourceVersion). An unknown or unreadable id is an error: mirroring the wrong + // cluster into a folder is worse than mirroring none. + ResolveSourceCluster(ctx context.Context, clusterID string) (cfg *rest.Config, version string, err error) +} + +// clusterContext holds everything that used to be a Manager-wide singleton and is in fact a +// property of ONE cluster: its API surface, the followability decisions derived from it, +// the clients that reach it, and the informers that tell us its surface moved. +// +// A single-cluster install has exactly one, keyed by LocalClusterID, and behaves exactly as +// before. A GitTarget that names a source cluster gets its own. +type clusterContext struct { + id string + + // catalog and registry are the "Scan -> Registry" pipeline for this cluster. A CRD + // installed only on the remote is followable only there. + catalog *APIResourceCatalog + registry *typeset.Registry + + // clientsMu guards the client/config fields below. It is PER CLUSTER: resolving a + // remote cluster's kubeconfig reads a Secret from the config plane, and a slow apiserver + // must not block client construction for every other cluster behind one global lock. + clientsMu sync.Mutex + // restConfig is nil until the cluster is first reached. configVersion is the version + // token restConfig was built from; when it changes, the cached clients are dropped so a + // rotated credential takes effect. + restConfig *rest.Config + configVersion string + dynamicClient dynamic.Interface + discovery apiResourceDiscovery + + // Logging state, edge-triggered per cluster: a degraded remote must not silence the + // local cluster's transitions, and vice versa. catalogReadyOnce synchronizes itself; the + // two maps are guarded by Manager.resourceCatalogMu. + catalogReadyOnce sync.Once + catalogDegradedLogged map[schema.GroupVersion]struct{} + typeRefusalsLogged map[string]string + + // API-surface trigger informers, gated on this cluster's own discovery. + // Guarded by Manager.triggersMu. + triggerFactory dynamicinformer.DynamicSharedInformerFactory + triggersStarted map[schema.GroupVersionResource]struct{} + triggersSkipLogged map[schema.GroupVersionResource]struct{} +} + +func newClusterContext(id string) *clusterContext { + return &clusterContext{ + id: id, + catalog: NewAPIResourceCatalog(), + registry: typeset.NewRegistry(), + catalogDegradedLogged: map[schema.GroupVersion]struct{}{}, + typeRefusalsLogged: map[string]string{}, + triggersStarted: map[schema.GroupVersionResource]struct{}{}, + triggersSkipLogged: map[schema.GroupVersionResource]struct{}{}, + } +} + +// isLocal reports whether this context is the cluster the operator runs in. +func (c *clusterContext) isLocal() bool { return c.id == LocalClusterID } + +// describe renders a cluster id for logs. The local cluster has no name of its own. +func describeCluster(id string) string { + if id == LocalClusterID { + return "local" + } + return id +} + +// cluster returns the context for a cluster id, creating it on first use. The local context +// is created lazily too, which is what lets a zero-value Manager work in tests. +func (m *Manager) cluster(id string) *clusterContext { + m.clustersMu.Lock() + defer m.clustersMu.Unlock() + if m.clusters == nil { + m.clusters = map[string]*clusterContext{} + } + cc := m.clusters[id] + if cc == nil { + cc = newClusterContext(id) + if id == LocalClusterID && m.resourceCatalog != nil { + cc.catalog = m.resourceCatalog + } + m.clusters[id] = cc + m.publishClusterOrderLocked() + if id != LocalClusterID { + m.Log.Info("source cluster registered", "clusterID", id) + } + } + return cc +} + +// localCluster is the config plane, and the watched cluster of every GitTarget that does +// not name a source. +func (m *Manager) localCluster() *clusterContext { return m.cluster(LocalClusterID) } + +// activeClusterIDs is every cluster some rule currently points at, plus the local one. The +// local cluster is always active: the operator's own CRs live there, and a rule-less install +// still refreshes its catalog. +func (m *Manager) activeClusterIDs() []string { + seen := map[string]struct{}{LocalClusterID: {}} + if m.RuleStore != nil { + for _, rule := range m.RuleStore.SnapshotWatchRules() { + seen[rule.SourceCluster] = struct{}{} + } + for _, rule := range m.RuleStore.SnapshotClusterWatchRules() { + seen[rule.SourceCluster] = struct{}{} + } + } + out := make([]string, 0, len(seen)) + for id := range seen { + out = append(out, id) + } + sort.Strings(out) + return out +} + +// clusterIDForGitTarget resolves the source cluster of a GitTarget from the rules pointing +// at it. Rules resolve their GitTarget when they compile, so in steady state they agree; a +// GitTarget with no rules yet has nothing to watch and lands on the local cluster. When rules +// disagree — the window in which a spec.sourceCluster change has recompiled some and not +// others — this returns the lexically first, and the table resolver refuses to watch anything +// at all, so the answer is never used to open a watch. +func (m *Manager) clusterIDForGitTarget(gitDest types.ResourceReference) string { + clusters := m.CompiledSourceClusters(gitDest) + if len(clusters) == 0 { + return LocalClusterID + } + return clusters[0] +} + +// CompiledSourceClusters returns every distinct source cluster the currently-COMPILED rules +// name for a GitTarget, sorted. Empty means nothing points at this GitTarget yet. +// +// The GitTarget controller reads it to answer "have my rules caught up with my spec?". A rule +// recompiles when its GitTarget's generation bumps, but the GitTarget's own reconcile may win +// that race — so right after spec.sourceCluster changes, some or all rules still name the OLD +// cluster. Declaring then would open watches against the old cluster and write its objects +// into the new destination's folder. More than one entry means the rules disagree with each +// other, which is the same window seen from the other side. +func (m *Manager) CompiledSourceClusters(gitDest types.ResourceReference) []string { + if m.RuleStore == nil { + return nil + } + key := gitDest.Key() + seen := map[string]struct{}{} + for _, rule := range m.RuleStore.SnapshotWatchRules() { + if rule.GitTargetNamespace+"/"+rule.GitTargetRef == key { + seen[rule.SourceCluster] = struct{}{} + } + } + for _, rule := range m.RuleStore.SnapshotClusterWatchRules() { + if rule.GitTargetNamespace+"/"+rule.GitTargetRef == key { + seen[rule.SourceCluster] = struct{}{} + } + } + if len(seen) == 0 { + return nil + } + out := make([]string, 0, len(seen)) + for id := range seen { + out = append(out, id) + } + sort.Strings(out) + return out +} + +// clusterRESTConfigLocked returns a cluster's rest.Config, resolving it on first use. +// +// It does NOT re-read the kubeconfig Secret when a config is already cached — that read is a +// config-plane API call, and this runs on the watch reconnect path, once per (GitTarget, GVR, +// scope) watch. Rotation is picked up by refreshClusterCredentials on the catalog-refresh +// cadence instead. +// +// The kubeconfig bytes are never retained: the resolver builds a rest.Config, the bytes are +// dropped, and only an opaque version token is remembered. Same reasoning as +// docs/future/secret-value-retention-plan.md — this operator does not hold credential +// material in memory longer than it must, and it starts no Secret informer. +// +// Must be called with cc.clientsMu held. +func (m *Manager) clusterRESTConfigLocked(ctx context.Context, cc *clusterContext) (*rest.Config, error) { + if cc.restConfig != nil { + return cc.restConfig, nil + } + if cc.isLocal() { + cfg, err := ctrl.GetConfig() + if err != nil { + return nil, fmt.Errorf("no REST config for the local cluster: %w", err) + } + cc.restConfig = cfg + return cfg, nil + } + cfg, version, err := m.resolveRemoteConfig(ctx, cc) + if err != nil { + return nil, err + } + cc.restConfig = cfg + cc.configVersion = version + return cfg, nil +} + +// resolveRemoteConfig reads a remote cluster's kubeconfig Secret from the config plane. +func (m *Manager) resolveRemoteConfig(ctx context.Context, cc *clusterContext) (*rest.Config, string, error) { + if m.SourceClusters == nil { + return nil, "", fmt.Errorf("cannot reach source cluster %q: no source-cluster resolver configured", cc.id) + } + cfg, version, err := m.SourceClusters.ResolveSourceCluster(ctx, cc.id) + if err != nil { + return nil, "", fmt.Errorf("resolve source cluster %q: %w", cc.id, err) + } + if cfg == nil { + return nil, "", fmt.Errorf("resolve source cluster %q: nil REST config", cc.id) + } + return cfg, version, nil +} + +// refreshClusterCredentials re-reads a remote cluster's kubeconfig Secret and, when it has +// rotated, drops the cached clients so the next use rebuilds them. It runs on the +// catalog-refresh cadence (every 30s and on every rule change), never on the watch reconnect +// path — one Secret read per cluster per refresh, not one per watch. +// +// A watch already streaming on the old credential keeps working until that credential stops +// being accepted; the reconnect that follows picks up the rebuilt client. +func (m *Manager) refreshClusterCredentials(ctx context.Context, cc *clusterContext) { + if cc.isLocal() { + return + } + cfg, version, err := m.resolveRemoteConfig(ctx, cc) + if err != nil { + // The catalog refresh that follows reports this properly; nothing to drop. + return + } + + cc.clientsMu.Lock() + defer cc.clientsMu.Unlock() + if cc.restConfig != nil && version == cc.configVersion { + return + } + if cc.restConfig != nil { + m.Log.Info("source cluster kubeconfig rotated; rebuilding clients", + "clusterID", cc.id, "version", version) + } + cc.restConfig = cfg + cc.configVersion = version + cc.dynamicClient = nil + cc.discovery = nil +} + +// clusterDynamicClient returns the dynamic client a cluster's watches and lists run on. +func (m *Manager) clusterDynamicClient(ctx context.Context, clusterID string) (dynamic.Interface, error) { + cc := m.cluster(clusterID) + + // Tests inject a fake client for the local cluster without a REST config at all. + if cc.isLocal() && m.dynamicClient != nil { + return m.dynamicClient, nil + } + + cc.clientsMu.Lock() + defer cc.clientsMu.Unlock() + if cc.dynamicClient != nil { + return cc.dynamicClient, nil + } + cfg, err := m.clusterRESTConfigLocked(ctx, cc) + if err != nil { + return nil, err + } + dc, err := dynamic.NewForConfig(cfg) + if err != nil { + return nil, fmt.Errorf("build dynamic client for cluster %q: %w", describeCluster(clusterID), err) + } + cc.dynamicClient = dc + return dc, nil +} + +// clusterDiscovery returns the discovery client backing a cluster's API-resource catalog. +func (m *Manager) clusterDiscovery(ctx context.Context, clusterID string) (apiResourceDiscovery, error) { + cc := m.cluster(clusterID) + + if cc.isLocal() && m.discoveryClient != nil { + return m.discoveryClient() + } + + cc.clientsMu.Lock() + defer cc.clientsMu.Unlock() + if cc.discovery != nil { + return cc.discovery, nil + } + cfg, err := m.clusterRESTConfigLocked(ctx, cc) + if err != nil { + return nil, err + } + disco, err := discovery.NewDiscoveryClientForConfig(cfg) + if err != nil { + return nil, fmt.Errorf("create discovery client for cluster %q: %w", describeCluster(clusterID), err) + } + cc.discovery = disco + return disco, nil +} + +// unionLookup answers GVK->record across every live cluster registry, in a stable order. +// +// The git writer holds ONE typeset.Lookup, used when scanning the manifests already in a Git +// folder to answer "what resource is this document?". Branch workers are keyed by +// (provider, branch) and shared by several GitTargets, so that lookup cannot be per-target +// without threading it through every pending write. A union is safe because GVK->GVR is +// derived from the served resource name: two clusters serving the same GVK agree on the GVR +// in every case short of an outright API-group collision. First answer wins, local first. +type unionLookup struct { + m *Manager +} + +// TypeLookup is the GVK resolver the git writer scans manifests with: a union over every +// live cluster's registry. In a single-cluster install it is exactly the local registry. +func (m *Manager) TypeLookup() typeset.Lookup { + return unionLookup{m: m} +} + +// Ready reports whether ANY cluster has observed its API surface. A writer that can resolve +// some types is more useful than one that resolves none, and a document whose type no live +// registry knows is refused by the acceptance gate anyway. +func (u unionLookup) Ready() bool { + for _, cc := range u.m.orderedClusters() { + if cc.registry.Ready() { + return true + } + } + return false +} + +func (u unionLookup) ByGVK(gvk schema.GroupVersionKind) (typeset.TypeRecord, bool) { + for _, cc := range u.m.orderedClusters() { + if rec, ok := cc.registry.ByGVK(gvk); ok { + return rec, true + } + } + return typeset.TypeRecord{}, false +} + +// orderedClusters returns the live cluster contexts, local first and then remotes sorted by +// id, so the union lookup's "first answer wins" is deterministic across reconciles. +// +// It reads a snapshot published whenever the cluster set changes, rather than taking +// clustersMu and rebuilding a slice: the git writer's GVK lookup calls this once per +// document it scans out of a folder, on the branch-worker goroutine, and that is no place +// for a mutex the reconcile loop also holds. Cluster contexts are created once and never +// mutated in place by this path, so handing out the slice is safe. +func (m *Manager) orderedClusters() []*clusterContext { + if snapshot := m.clusterOrder.Load(); snapshot != nil { + return *snapshot + } + // No cluster has been created yet: force the local one into existence, which publishes + // the first snapshot. + m.localCluster() + if snapshot := m.clusterOrder.Load(); snapshot != nil { + return *snapshot + } + return nil +} + +// publishClusterOrderLocked recomputes the ordered snapshot. Must be called with clustersMu +// held, from the one place that adds a cluster. +func (m *Manager) publishClusterOrderLocked() { + ids := make([]string, 0, len(m.clusters)) + for id := range m.clusters { + ids = append(ids, id) + } + sort.Strings(ids) // LocalClusterID is "" and sorts first. + out := make([]*clusterContext, 0, len(ids)) + for _, id := range ids { + out = append(out, m.clusters[id]) + } + m.clusterOrder.Store(&out) +} diff --git a/internal/watch/cluster_context_test.go b/internal/watch/cluster_context_test.go new file mode 100644 index 00000000..44a959c4 --- /dev/null +++ b/internal/watch/cluster_context_test.go @@ -0,0 +1,373 @@ +// SPDX-License-Identifier: Apache-2.0 + +package watch + +import ( + "context" + "errors" + "testing" + + "github.com/go-logr/logr" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/rest" + + configv1alpha3 "github.com/ConfigButler/gitops-reverser/api/v1alpha3" + "github.com/ConfigButler/gitops-reverser/internal/rulestore" + "github.com/ConfigButler/gitops-reverser/internal/types" +) + +const remoteClusterID = "team-a/acme-kubeconfig/value.yaml" + +// stubSourceClusterResolver stands in for the Secret-backed resolver. +type stubSourceClusterResolver struct { + cfg *rest.Config + version string + err error + calls int +} + +func (s *stubSourceClusterResolver) ResolveSourceCluster( + context.Context, string, +) (*rest.Config, string, error) { + s.calls++ + return s.cfg, s.version, s.err +} + +// storeWithRemoteRule compiles one WatchRule bound to a GitTarget on the remote source +// cluster — the shape every test in this file exercises against the local default. +func storeWithRemoteRule(t *testing.T) *rulestore.RuleStore { + t.Helper() + const sourceCluster = remoteClusterID + store := rulestore.NewStore() + store.AddOrUpdateWatchRule(configv1alpha3.WatchRule{ + ObjectMeta: metav1.ObjectMeta{Name: "rule", Namespace: "team-a"}, + Spec: configv1alpha3.WatchRuleSpec{ + Rules: []configv1alpha3.ResourceRule{{Resources: []string{"configmaps"}}}, + }, + }, rulestore.TargetBinding{ + GitTargetName: "acme", + GitTargetNamespace: "team-a", + GitProviderName: "prov", + GitProviderNamespace: "team-a", + Branch: "main", + Path: "apps", + SourceCluster: sourceCluster, + }) + return store +} + +func TestGitTargetSourceClusterID(t *testing.T) { + t.Parallel() + + local := &configv1alpha3.GitTarget{ObjectMeta: metav1.ObjectMeta{Name: "acme", Namespace: "team-a"}} + assert.Equal(t, LocalClusterID, local.SourceClusterID(), + "a GitTarget without spec.sourceCluster mirrors the cluster the operator runs in") + + remote := &configv1alpha3.GitTarget{ + ObjectMeta: metav1.ObjectMeta{Name: "acme", Namespace: "team-a"}, + Spec: configv1alpha3.GitTargetSpec{ + SourceCluster: &configv1alpha3.SourceClusterSpec{ + KubeConfigSecretRef: configv1alpha3.SecretKeyReference{Name: "acme-kubeconfig"}, + }, + }, + } + assert.Equal(t, remoteClusterID, remote.SourceClusterID(), + "an omitted key defaults to Flux's value.yaml") + + // The key is part of the identity: two GitTargets naming one Secret under different + // keys are pointed at different kubeconfigs, and so at different clusters. + remote.Spec.SourceCluster.KubeConfigSecretRef.Key = "prod.yaml" + assert.Equal(t, "team-a/acme-kubeconfig/prod.yaml", remote.SourceClusterID()) +} + +func TestManager_ClusterIDForGitTarget(t *testing.T) { + t.Parallel() + + gitDest := types.NewResourceReference("acme", "team-a") + + m := &Manager{Log: logr.Discard(), RuleStore: storeWithRemoteRule(t)} + assert.Equal(t, remoteClusterID, m.clusterIDForGitTarget(gitDest)) + + // A GitTarget with no rules yet has nothing to watch, and lands on the local cluster. + assert.Equal(t, LocalClusterID, m.clusterIDForGitTarget(types.NewResourceReference("other", "team-a"))) + + noRules := &Manager{Log: logr.Discard()} + assert.Equal(t, LocalClusterID, noRules.clusterIDForGitTarget(gitDest)) +} + +func TestManager_ActiveClusterIDs(t *testing.T) { + t.Parallel() + + // The local cluster is always active: the operator's own CRs live there. + bare := &Manager{Log: logr.Discard()} + assert.Equal(t, []string{LocalClusterID}, bare.activeClusterIDs()) + + m := &Manager{Log: logr.Discard(), RuleStore: storeWithRemoteRule(t)} + assert.Equal(t, []string{LocalClusterID, remoteClusterID}, m.activeClusterIDs()) +} + +// Each cluster has its own API surface. A CRD installed only on the remote is followable +// only there, and one installed only locally is not followable on the remote. +func TestManager_ClusterRegistriesAreIndependent(t *testing.T) { + t.Parallel() + + m := &Manager{Log: logr.Discard()} + + local := m.cluster(LocalClusterID) + _, err := local.catalog.Refresh(newCommonTestDiscovery()) + require.NoError(t, err) + m.refreshTypeRegistry(local) + + remote := m.cluster(remoteClusterID) + assert.False(t, remote.registry.Ready(), "an unscanned remote registry must not claim readiness") + assert.True(t, local.registry.Ready()) + + iceCream := schema.GroupVersionKind{Group: "shop.example.com", Version: "v1alpha1", Kind: "IceCreamOrder"} + _, okLocal := local.registry.ByGVK(iceCream) + assert.True(t, okLocal) + _, okRemote := remote.registry.ByGVK(iceCream) + assert.False(t, okRemote, "the remote has not observed this type") +} + +// The git writer holds ONE lookup because branch workers are shared across GitTargets that +// may mirror different clusters. It answers from any live registry, local first. +func TestManager_TypeLookupIsAUnion(t *testing.T) { + t.Parallel() + + m := &Manager{Log: logr.Discard()} + lookup := m.TypeLookup() + assert.False(t, lookup.Ready(), "no cluster has observed anything yet") + + remote := m.cluster(remoteClusterID) + _, err := remote.catalog.Refresh(newCommonTestDiscovery()) + require.NoError(t, err) + m.refreshTypeRegistry(remote) + + assert.True(t, lookup.Ready(), "a lookup that can resolve some types is ready") + + configMap := schema.GroupVersionKind{Version: "v1", Kind: "ConfigMap"} + rec, ok := lookup.ByGVK(configMap) + require.True(t, ok, "the remote's types are reachable through the union") + assert.Equal(t, "configmaps", rec.Identity.GVR.Resource) + + _, ok = lookup.ByGVK(schema.GroupVersionKind{Group: "nope.example.com", Version: "v1", Kind: "Widget"}) + assert.False(t, ok) +} + +func TestUnionLookup_LocalWinsTies(t *testing.T) { + t.Parallel() + + m := &Manager{Log: logr.Discard()} + for _, id := range []string{LocalClusterID, remoteClusterID} { + cc := m.cluster(id) + _, err := cc.catalog.Refresh(newCommonTestDiscovery()) + require.NoError(t, err) + m.refreshTypeRegistry(cc) + } + + // orderedClusters sorts by id, and LocalClusterID is "" — so the local answer is first. + ordered := m.orderedClusters() + require.Len(t, ordered, 2) + assert.Equal(t, LocalClusterID, ordered[0].id) + assert.Equal(t, remoteClusterID, ordered[1].id) +} + +// A remote cluster with no resolver configured must fail loudly rather than silently +// falling back to the local cluster — mirroring the wrong cluster into a folder is worse +// than mirroring none. +func TestManager_RemoteClusterWithoutResolverIsAnError(t *testing.T) { + t.Parallel() + + m := &Manager{Log: logr.Discard()} + _, err := m.clusterDynamicClient(context.Background(), remoteClusterID) + + require.Error(t, err) + assert.Contains(t, err.Error(), "no source-cluster resolver configured") +} + +func TestManager_RemoteClusterResolverError(t *testing.T) { + t.Parallel() + + m := &Manager{Log: logr.Discard(), SourceClusters: &stubSourceClusterResolver{err: errors.New("secret not found")}} + _, err := m.clusterDiscovery(context.Background(), remoteClusterID) + + require.Error(t, err) + assert.Contains(t, err.Error(), "secret not found") +} + +// The watch reconnect path must never read the config plane: it runs once per (GitTarget, +// GVR, scope) watch, and a Secret GET per reconnect would couple every reconnect to the +// config-plane apiserver. +func TestManager_CachedClientIsReusedWithoutReadingTheSecret(t *testing.T) { + t.Parallel() + + resolver := &stubSourceClusterResolver{cfg: &rest.Config{Host: "https://one.example"}, version: "1"} + m := &Manager{Log: logr.Discard(), SourceClusters: resolver} + ctx := context.Background() + + first, err := m.clusterDynamicClient(ctx, remoteClusterID) + require.NoError(t, err) + require.Equal(t, 1, resolver.calls) + + second, err := m.clusterDynamicClient(ctx, remoteClusterID) + require.NoError(t, err) + assert.Same(t, first, second) + assert.Equal(t, 1, resolver.calls, "a cached client must not re-read the kubeconfig Secret") +} + +// Rotation is detected on the catalog-refresh cadence, not on the reconnect path. The +// Secret's resourceVersion is the version token, so an unchanged Secret rebuilds nothing. +func TestManager_KubeConfigRotationRebuildsClients(t *testing.T) { + t.Parallel() + + resolver := &stubSourceClusterResolver{cfg: &rest.Config{Host: "https://one.example"}, version: "1"} + m := &Manager{Log: logr.Discard(), SourceClusters: resolver} + ctx := context.Background() + cc := m.cluster(remoteClusterID) + + first, err := m.clusterDynamicClient(ctx, remoteClusterID) + require.NoError(t, err) + + m.refreshClusterCredentials(ctx, cc) + second, err := m.clusterDynamicClient(ctx, remoteClusterID) + require.NoError(t, err) + assert.Same(t, first, second, "an unchanged Secret must not rebuild the client") + + resolver.cfg = &rest.Config{Host: "https://two.example"} + resolver.version = "2" + m.refreshClusterCredentials(ctx, cc) + + third, err := m.clusterDynamicClient(ctx, remoteClusterID) + require.NoError(t, err) + assert.NotSame(t, first, third, "a rotated kubeconfig must rebuild the client") +} + +// The local cluster has no kubeconfig Secret to rotate. +func TestManager_RefreshClusterCredentials_LocalIsANoOp(t *testing.T) { + t.Parallel() + + resolver := &stubSourceClusterResolver{err: errors.New("must not be called")} + m := &Manager{Log: logr.Discard(), SourceClusters: resolver} + m.refreshClusterCredentials(context.Background(), m.localCluster()) + assert.Zero(t, resolver.calls) +} + +func TestManager_LocalClusterIgnoresTheResolver(t *testing.T) { + t.Parallel() + + resolver := &stubSourceClusterResolver{err: errors.New("must not be called")} + m := &Manager{Log: logr.Discard(), SourceClusters: resolver, discoveryClient: commonTestDiscoveryClient()} + + _, err := m.clusterDiscovery(context.Background(), LocalClusterID) + require.NoError(t, err) + assert.Zero(t, resolver.calls, "the local cluster's config comes from controller-runtime") +} + +func TestDescribeCluster(t *testing.T) { + t.Parallel() + assert.Equal(t, "local", describeCluster(LocalClusterID)) + assert.Equal(t, remoteClusterID, describeCluster(remoteClusterID)) +} + +// The GitTarget's source cluster must reach the watch that opens against it: a table +// resolved for a remote target carries its cluster id, and filterFor hands it to the watch. +func TestResolveWatchedTypeTables_CarriesTheSourceCluster(t *testing.T) { + t.Parallel() + + m := &Manager{Log: logr.Discard(), RuleStore: storeWithRemoteRule(t)} + + // The remote's API surface is what the rule's types resolve against. + remote := m.cluster(remoteClusterID) + _, err := remote.catalog.Refresh(newCommonTestDiscovery()) + require.NoError(t, err) + m.refreshTypeRegistry(remote) + + tables := m.resolveWatchedTypeTables(remote.registry.Generation()) + table, ok := tables["team-a/acme"] + require.True(t, ok, "the GitTarget must have a resolved table") + assert.Equal(t, remoteClusterID, table.ClusterID) + require.NotEmpty(t, table.Types, "configmaps resolve against the remote's catalog") + + filter := table.filterFor(targetWatchKey{GVR: table.Types[0].GVR, Namespace: "team-a"}) + assert.Equal(t, remoteClusterID, filter.cluster, + "the watch must open against the GitTarget's source cluster, not the local one") +} + +// A rule whose GitTarget mirrors the remote resolves NOTHING when only the local cluster +// has observed its API surface: the types it names live on a cluster we have not scanned. +func TestResolveWatchedTypeTables_RemoteTypesDoNotComeFromTheLocalCatalog(t *testing.T) { + t.Parallel() + + m := &Manager{Log: logr.Discard(), RuleStore: storeWithRemoteRule(t)} + + local := m.cluster(LocalClusterID) + _, err := local.catalog.Refresh(newCommonTestDiscovery()) + require.NoError(t, err) + m.refreshTypeRegistry(local) + + tables := m.resolveWatchedTypeTables(local.registry.Generation()) + table := tables["team-a/acme"] + assert.Empty(t, table.Types, + "the local cluster's configmaps are not the remote's; a GitTarget must never mirror the wrong cluster") +} + +// A rule recompiles when its GitTarget's generation bumps, but the GitTarget's own reconcile +// can win that race. The controller needs to tell "the rules have not caught up" and "the +// rules disagree with each other" apart from "nothing points at this GitTarget", so it never +// declares against the previous cluster. +func TestManager_CompiledSourceClusters(t *testing.T) { + t.Parallel() + + m := &Manager{Log: logr.Discard(), RuleStore: storeWithRemoteRule(t)} + assert.Equal(t, []string{remoteClusterID}, + m.CompiledSourceClusters(types.NewResourceReference("acme", "team-a"))) + + assert.Empty(t, m.CompiledSourceClusters(types.NewResourceReference("other", "team-a")), + "no rule points at this GitTarget, so there is nothing to disagree about") + + noStore := &Manager{Log: logr.Discard()} + assert.Empty(t, noStore.CompiledSourceClusters(types.NewResourceReference("acme", "team-a"))) +} + +// Mid-recompile, a GitTarget's WatchRule can name the new cluster while its ClusterWatchRule +// still names the old one. The data plane must then watch nothing rather than pick one. +func TestResolveWatchedTypeTables_DisagreeingRulesWatchNothing(t *testing.T) { + t.Parallel() + + store := storeWithRemoteRule(t) + // A second rule for the same GitTarget, still compiled against the local cluster. + store.AddOrUpdateClusterWatchRule(configv1alpha3.ClusterWatchRule{ + ObjectMeta: metav1.ObjectMeta{Name: "stale-rule"}, + Spec: configv1alpha3.ClusterWatchRuleSpec{ + Rules: []configv1alpha3.ClusterResourceRule{{ + Resources: []string{"configmaps"}, + Scope: configv1alpha3.ResourceScopeNamespaced, + }}, + }, + }, rulestore.TargetBinding{ + GitTargetName: "acme", + GitTargetNamespace: "team-a", + Branch: "main", + Path: "apps", + SourceCluster: LocalClusterID, + }) + + m := &Manager{Log: logr.Discard(), RuleStore: store} + for _, id := range []string{LocalClusterID, remoteClusterID} { + cc := m.cluster(id) + _, err := cc.catalog.Refresh(newCommonTestDiscovery()) + require.NoError(t, err) + m.refreshTypeRegistry(cc) + } + + assert.Len(t, m.CompiledSourceClusters(types.NewResourceReference("acme", "team-a")), 2, + "the rules disagree, and both clusters are reported") + + tables := m.resolveWatchedTypeTables(1) + table := tables["team-a/acme"] + assert.Empty(t, table.Types, + "watching one cluster's objects with another cluster's resolved types would mirror the wrong cluster") +} diff --git a/internal/watch/manager.go b/internal/watch/manager.go index 352d4fd0..f22cace2 100644 --- a/internal/watch/manager.go +++ b/internal/watch/manager.go @@ -9,6 +9,7 @@ package watch import ( "context" "sync" + "sync/atomic" "time" "github.com/go-logr/logr" @@ -26,7 +27,6 @@ import ( "github.com/ConfigButler/gitops-reverser/internal/rulestore" "github.com/ConfigButler/gitops-reverser/internal/telemetry" "github.com/ConfigButler/gitops-reverser/internal/types" - "github.com/ConfigButler/gitops-reverser/internal/typeset" ) // RBAC permissions for dynamic watch manager - read-only access to watch all (also future ones!) resource types @@ -89,21 +89,39 @@ type Manager struct { // against what git already holds. See routeLiveTargetWatchEvent. liveContentDedup sync.Map - // resourceCatalog is the shared discovery-backed API surface used by rule planning. + // SourceClusters resolves a GitTarget's source-cluster id into a rest.Config, by + // reading the kubeconfig Secret it names from the config plane. Nil is the + // single-cluster install: every GitTarget mirrors the cluster the operator runs in, + // and no GitTarget may name a source. + SourceClusters SourceClusterResolver + + // clusters holds one clusterContext per distinct source cluster — its API catalog, + // type registry, clients, and trigger informers. LocalClusterID is the cluster the + // operator runs in, and the only one a single-cluster install ever creates. + clustersMu sync.Mutex + clusters map[string]*clusterContext + // clusterOrder is the published, ordered snapshot of clusters (local first). The git + // writer's GVK lookup reads it once per document it scans out of a Git folder, on the + // branch-worker goroutine, so it must not contend on clustersMu with the reconcile loop. + clusterOrder atomic.Pointer[[]*clusterContext] + // resourceCatalogMu guards every clusterContext's catalog/registry logging state. resourceCatalogMu sync.Mutex - resourceCatalog *APIResourceCatalog - // discoveryClient overrides REST-config discovery construction in tests. + // discoveryClient overrides REST-config discovery construction for the LOCAL cluster + // in tests. discoveryClient func() (apiResourceDiscovery, error) + // resourceCatalog seeds the LOCAL cluster's API-resource catalog. Tests set it on a + // zero-value Manager to drive resolution without an API server; production leaves it + // nil and the cluster context builds its own. + resourceCatalog *APIResourceCatalog // catalogRefreshCh coalesces API-surface trigger watch events into manager reconciliation. catalogRefreshCh chan struct{} - // catalogReadyOnce guards the one-time "catalog ready" log line, matching the - // firstMessage/firstGroupReady sync.Once pattern used by the audit consumer. - catalogReadyOnce sync.Once - // catalogDegradedLogged is the degraded group/version set last reflected in - // the log; logCatalogTransitions diffs against it to log appear/clear - // transitions (degradation can recur, so this is not a one-shot). Guarded by - // resourceCatalogMu. - catalogDegradedLogged map[schema.GroupVersion]struct{} + // triggersMu guards every clusterContext's trigger informer set. The informers are + // (re-)evaluated after every catalog refresh — which controllers drive, not just the + // manager's own loop — so this is not a startup-only structure. + triggersMu sync.Mutex + // triggerCtx is the manager's lifetime context, the stop channel every trigger informer + // is started with. Set once by Start; nil before then, which defers informer creation. + triggerCtx context.Context // watchedTypes is the resident, per-GitTarget watched-type table set: the single // source of "what each GitTarget watches", a projection of the type registry's @@ -141,17 +159,10 @@ type Manager struct { gitTargetUIDsMu sync.Mutex gitTargetUIDs map[string]string - // typeRegistry is the followability decision surface (see - // docs/design/manifest/version2/type-followability.md): one typeset.TypeRecord - // per served type, refreshed from the catalog scan on every catalog refresh. It - // is the inventory/status surface ("is this type followable, and if not, why?"); - // typeRegistryInit guards its lazy construction for zero-value Managers in tests. - typeRegistryInit sync.Once - typeRegistry *typeset.Registry - // typeRefusalsLogged is the GVK->summary of every type the registry currently - // refuses, so the central "why is this not followable?" log is edge-triggered: a - // stable refusal is logged once, not on every refresh. Guarded by resourceCatalogMu. - typeRefusalsLogged map[string]string + // excludeUsersWarned records the rules already warned about declaring excludeUsers with + // no author attribution configured, so the warning is edge-triggered rather than + // re-emitted on every 30s rule resolution. + excludeUsersWarned sync.Map // declaredGVRsMu guards declaredGVRs: the type-set each GitTarget last Declared. The watch-first // data plane reads it to drive the per-(GitTarget, type) watch set; re-declaring is idempotent. @@ -180,6 +191,10 @@ func (m *Manager) Start(ctx context.Context) error { defer log.Info("watch ingestion manager stopping") m.initializeManagerState() + // Arm the trigger informers before the first refresh: each successful catalog refresh + // re-evaluates which triggers discovery actually serves, and starts the ones that + // became available. They are stopped by this context, never by a reconcile's. + m.setTriggerContext(ctx) if err := m.bootstrapRuleStore(ctx, log.WithName("bootstrap")); err != nil { log.Error(err, "RuleStore bootstrap failed, continuing with current in-memory state") @@ -189,7 +204,6 @@ func (m *Manager) Start(ctx context.Context) error { if err := m.ReconcileForRuleChange(ctx); err != nil { log.Error(err, "Initial reconciliation failed, will retry periodically") } - m.startAPISurfaceTriggerInformers(ctx, log.WithName("catalog-triggers")) // Periodic reconciliation for CRD detection and missed changes periodicTicker := time.NewTicker(periodicReconcileInterval) @@ -233,26 +247,6 @@ func (m *Manager) NeedLeaderElection() bool { return true } -// dynamicClientFromConfig builds a dynamic client from the controller's REST config. -// If m.dynamicClient is set (e.g. in tests) it is returned directly. It is used by the -// per-type checkpoint fill (mirrorTypeObjects) — the only API touch on a schedule. -func (m *Manager) dynamicClientFromConfig(log logr.Logger) dynamic.Interface { - if m.dynamicClient != nil { - return m.dynamicClient - } - cfg := m.restConfig() - if cfg == nil { - log.Info("skipping seed - no rest config available") - return nil - } - dc, err := dynamic.NewForConfig(cfg) - if err != nil { - log.Error(err, "failed to construct dynamic client for seed") - return nil - } - return dc -} - // ReconcileForRuleChange refreshes the trusted API catalog and the resident watched-type // tables when rules change or a CRD is installed/removed. It no longer starts object // informers or gathers a whole-GitTarget snapshot (R3): the catalog refresh drives the diff --git a/internal/watch/manager_catalog.go b/internal/watch/manager_catalog.go index b4a4323d..08723865 100644 --- a/internal/watch/manager_catalog.go +++ b/internal/watch/manager_catalog.go @@ -4,7 +4,6 @@ package watch import ( "context" - "errors" "fmt" "math" "sort" @@ -14,31 +13,15 @@ import ( "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/metric" "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/client-go/discovery" - "k8s.io/client-go/dynamic" "k8s.io/client-go/dynamic/dynamicinformer" - "k8s.io/client-go/rest" "k8s.io/client-go/tools/cache" - ctrl "sigs.k8s.io/controller-runtime" configv1alpha3 "github.com/ConfigButler/gitops-reverser/api/v1alpha3" "github.com/ConfigButler/gitops-reverser/internal/telemetry" + "github.com/ConfigButler/gitops-reverser/internal/types" "github.com/ConfigButler/gitops-reverser/internal/typeset" ) -// restConfig acquires the controller runtime REST config. -// Returns nil if no config is available (e.g., in unit tests without a cluster). -func (m *Manager) restConfig() *rest.Config { - // ctrl.GetConfig reads KUBECONFIG or in-cluster config. In tests/e2e this is - // set up by the test harness/Kind. In unit tests without a cluster it returns - // an error, which callers handle gracefully. - cfg, err := ctrl.GetConfig() - if err != nil { - return nil - } - return cfg -} - func crdTriggerGVR() schema.GroupVersionResource { return schema.GroupVersionResource{ Group: "apiextensions.k8s.io", @@ -55,23 +38,50 @@ func apiServiceTriggerGVR() schema.GroupVersionResource { } } -// RefreshAPIResourceCatalog refreshes trusted catalog data from Kubernetes discovery. +// RefreshAPIResourceCatalog refreshes every active cluster's catalog from its own +// discovery. It returns the LOCAL cluster's error only: a remote cluster that cannot be +// reached must fail its own GitTargets (which it does, through its unready registry), never +// the local cluster's. Remote failures are logged and left for the affected targets to +// surface. func (m *Manager) RefreshAPIResourceCatalog(ctx context.Context) error { - catalog := m.apiResourceCatalog() - disco, err := m.apiResourceDiscovery() + localErr := m.refreshClusterCatalog(ctx, LocalClusterID) + for _, id := range m.activeClusterIDs() { + if id == LocalClusterID { + continue + } + if err := m.refreshClusterCatalog(ctx, id); err != nil { + m.Log.V(1).Info("source cluster catalog refresh failed", + "clusterID", id, "err", err.Error()) + } + } + return localErr +} + +// refreshClusterCatalog refreshes one cluster's trusted catalog data from its discovery, +// re-derives its followability registry, and re-arms its API-surface trigger informers. +func (m *Manager) refreshClusterCatalog(ctx context.Context, clusterID string) error { + cc := m.cluster(clusterID) + // One Secret read per cluster per refresh. A rotated kubeconfig drops the cached clients + // here, so the watch reconnect path never has to read the config plane itself. + m.refreshClusterCredentials(ctx, cc) + disco, err := m.clusterDiscovery(ctx, clusterID) if err != nil { return err } start := time.Now() - changed, refreshErr := catalog.Refresh(disco) + changed, refreshErr := cc.catalog.Refresh(disco) recordCatalogRefresh(ctx, changed, refreshErr, time.Since(start)) if refreshErr == nil { // Re-derive the followability records from the fresh scan before logging, so // the ready line can report how many served types are followable. - m.refreshTypeRegistry() - stats := catalog.Stats() + m.refreshTypeRegistry(cc) + stats := cc.catalog.Stats() recordCatalogStats(ctx, stats) - m.logCatalogTransitions(catalog, stats) + m.logCatalogTransitions(cc, stats) + // The fresh scan is the only source of truth for which trigger resources this API + // server actually serves, so trigger informers are (re-)armed here rather than once + // at startup. An aggregation layer installed later is picked up on its refresh. + m.ensureAPISurfaceTriggerInformers(ctx, cc, m.Log.WithName("catalog-triggers")) } return refreshErr } @@ -80,18 +90,18 @@ func (m *Manager) RefreshAPIResourceCatalog(ctx context.Context) error { // only: the first successful build, and when the set of group/versions that // discovery cannot serve appears or clears. Steady-state refreshes - which run // on every rule change, periodic tick, and CRD/APIService event - stay silent. -func (m *Manager) logCatalogTransitions(catalog *APIResourceCatalog, stats CatalogStats) { - log := m.Log.WithName("catalog") +func (m *Manager) logCatalogTransitions(cc *clusterContext, stats CatalogStats) { + log := m.Log.WithName("catalog").WithValues("cluster", describeCluster(cc.id)) - if catalog.Ready() { - m.catalogReadyOnce.Do(func() { + if cc.catalog.Ready() { + cc.catalogReadyOnce.Do(func() { log.Info("API resource catalog ready", "allowedResources", stats.AllowedResources, "excludedResources", stats.ExcludedResources, "trustedGroupVersions", stats.TrustedGroupVersions, "degradedGroupVersions", stats.DegradedGroupVersions, - "followableTypes", len(m.FollowableTypeRecords()), - "knownTypes", len(m.TypeRecords()), + "followableTypes", len(cc.registry.Followable()), + "knownTypes", len(cc.registry.All()), "generation", stats.Generation) }) } @@ -101,19 +111,19 @@ func (m *Manager) logCatalogTransitions(catalog *APIResourceCatalog, stats Catal current := make(map[schema.GroupVersion]struct{}) var appeared []schema.GroupVersion - for _, gv := range catalog.DegradedGroupVersions() { + for _, gv := range cc.catalog.DegradedGroupVersions() { current[gv] = struct{}{} - if _, known := m.catalogDegradedLogged[gv]; !known { + if _, known := cc.catalogDegradedLogged[gv]; !known { appeared = append(appeared, gv) } } var cleared []schema.GroupVersion - for gv := range m.catalogDegradedLogged { + for gv := range cc.catalogDegradedLogged { if _, still := current[gv]; !still { cleared = append(cleared, gv) } } - m.catalogDegradedLogged = current + cc.catalogDegradedLogged = current if len(appeared) > 0 { log.Info("API discovery degraded - the cluster cannot serve these group/versions; "+ @@ -189,24 +199,22 @@ func recordCatalogStats(ctx context.Context, stats CatalogStats) { } } +// apiResourceCatalog returns the LOCAL cluster's catalog. Callers that know which cluster +// they mean go through m.cluster(id).catalog instead. func (m *Manager) apiResourceCatalog() *APIResourceCatalog { - m.resourceCatalogMu.Lock() - defer m.resourceCatalogMu.Unlock() - if m.resourceCatalog == nil { - m.resourceCatalog = NewAPIResourceCatalog() - } - return m.resourceCatalog + return m.localCluster().catalog } -// typeRegistryInstance returns the lazily-built followability registry, so a -// zero-value Manager (used widely in tests) needs no explicit setup. +// typeRegistryInstance returns the LOCAL cluster's followability registry, so a zero-value +// Manager (used widely in tests) needs no explicit setup. Target-scoped callers go through +// m.clusterRegistry(clusterID). func (m *Manager) typeRegistryInstance() *typeset.Registry { - m.typeRegistryInit.Do(func() { - if m.typeRegistry == nil { - m.typeRegistry = typeset.NewRegistry() - } - }) - return m.typeRegistry + return m.localCluster().registry +} + +// clusterRegistry is the followability decision surface for one cluster. +func (m *Manager) clusterRegistry(clusterID string) *typeset.Registry { + return m.cluster(clusterID).registry } // refreshTypeRegistry publishes the catalog's latest normalized scan to the typeset @@ -215,18 +223,17 @@ func (m *Manager) typeRegistryInstance() *typeset.Registry { // catalog refresh, so the registry tracks discovery and its grace clocks advance on // the same cadence the catalog scans do. It is the "Scan -> Registry" pipeline of // docs/design/manifest/version2/type-followability.md. -func (m *Manager) refreshTypeRegistry() { +func (m *Manager) refreshTypeRegistry(cc *clusterContext) { // Only publish once the catalog holds trusted data, so the registry's readiness // tracks the catalog's: an unready catalog must leave the registry unready, which // is what makes the live mapper fall closed (CatalogUnavailable) rather than treat // an empty scan as a trusted "nothing is served". - scan, ok := m.apiResourceCatalog().Scan(m.SensitiveResources) + scan, ok := cc.catalog.Scan(m.SensitiveResources) if !ok { return } - reg := m.typeRegistryInstance() - reg.UpdateFromScan(scan) - m.logTypeRefusals(reg) + cc.registry.UpdateFromScan(scan) + m.logTypeRefusals(cc) } // logTypeRefusals is the single central place that explains why a served type is not @@ -235,23 +242,23 @@ func (m *Manager) refreshTypeRegistry() { // once rather than on every refresh. The full machine-readable answer always lives on // the registry record (TypeRecords / FollowableTypeRecords), so callers that need it // read there rather than parse logs. -func (m *Manager) logTypeRefusals(reg *typeset.Registry) { - log := m.Log.WithName("followability") +func (m *Manager) logTypeRefusals(cc *clusterContext) { + log := m.Log.WithName("followability").WithValues("cluster", describeCluster(cc.id)) m.resourceCatalogMu.Lock() defer m.resourceCatalogMu.Unlock() current := map[string]string{} - for _, rec := range reg.All() { + for _, rec := range cc.registry.All() { if rec.Followable() { continue } key := rec.Identity.GVK.String() current[key] = rec.Followability.Summary - if prev, known := m.typeRefusalsLogged[key]; !known || prev != rec.Followability.Summary { + if prev, known := cc.typeRefusalsLogged[key]; !known || prev != rec.Followability.Summary { log.V(1).Info("type is not followable", "gvk", key, "gvr", rec.Identity.GVR.String(), "reason", rec.Followability.Summary) } } - m.typeRefusalsLogged = current + cc.typeRefusalsLogged = current } // TypeRegistry returns the live followability registry, the single decision surface @@ -274,21 +281,6 @@ func (m *Manager) TypeRecords() []typeset.TypeRecord { return m.typeRegistryInstance().All() } -func (m *Manager) apiResourceDiscovery() (apiResourceDiscovery, error) { - if m.discoveryClient != nil { - return m.discoveryClient() - } - cfg := m.restConfig() - if cfg == nil { - return nil, errors.New("no REST config available for API resource discovery") - } - disco, err := discovery.NewDiscoveryClientForConfig(cfg) - if err != nil { - return nil, fmt.Errorf("create API resource discovery client: %w", err) - } - return disco, nil -} - // ruleResourceSelector is one rule's (apiGroups, apiVersions, resources, scope) tuple, // the unit ResolveWatchRuleResources / ResolveClusterWatchRuleResources match against the // followable set. @@ -310,7 +302,10 @@ func (m *Manager) ResolveWatchRuleResources( scope: configv1alpha3.ResourceScopeNamespaced, }) } - return m.resolveRuleResourceStatus(selectors) + // A WatchRule resolves its types against the cluster its GitTarget mirrors, so a CRD + // installed only on the source cluster counts, and one installed only locally does not. + clusterID := m.clusterIDForGitTarget(types.NewResourceReference(rule.Spec.TargetRef.Name, rule.Namespace)) + return m.resolveRuleResourceStatus(clusterID, selectors) } // ResolveClusterWatchRuleResources reports one ClusterWatchRule's resource-resolution @@ -325,7 +320,9 @@ func (m *Manager) ResolveClusterWatchRuleResources( groups: rr.APIGroups, versions: rr.APIVersions, resources: rr.Resources, scope: rr.Scope, }) } - return m.resolveRuleResourceStatus(selectors) + clusterID := m.clusterIDForGitTarget( + types.NewResourceReference(rule.Spec.TargetRef.Name, rule.Spec.TargetRef.Namespace)) + return m.resolveRuleResourceStatus(clusterID, selectors) } // resolveRuleResourceStatus reports a rule's resource-resolution status from the type @@ -334,10 +331,14 @@ func (m *Manager) ResolveClusterWatchRuleResources( // explain why an individual selector matched nothing: absent, refused, and not-yet-served // are all the same to a mirror. Status only reports catalog readiness and how many distinct // followable types the rule currently watches. -func (m *Manager) resolveRuleResourceStatus(selectors []ruleResourceSelector) (bool, string) { - m.refreshTypeRegistry() - reg := m.typeRegistryInstance() +func (m *Manager) resolveRuleResourceStatus(clusterID string, selectors []ruleResourceSelector) (bool, string) { + cc := m.cluster(clusterID) + m.refreshTypeRegistry(cc) + reg := cc.registry if !reg.Ready() { + if clusterID != LocalClusterID { + return false, "the source cluster's API resource catalog is not ready" + } return false, "API resource catalog is not ready" } records := reg.Followable() @@ -360,37 +361,114 @@ func (m *Manager) signalCatalogRefresh() { } } -func (m *Manager) startAPISurfaceTriggerInformers(ctx context.Context, log logr.Logger) { - cfg := m.restConfig() - if cfg == nil { - log.V(1).Info("skipping API surface trigger informers - no REST config available") +// ensureAPISurfaceTriggerInformers starts one cluster's CRD and APIService trigger +// informers, but only for the resources ITS discovery reports as served with list+watch. +// Neither is universal: an API server without an aggregation layer serves no +// apiregistration.k8s.io, and a blind informer on it makes client-go's reflector retry and +// log forever — benign, endlessly repeated noise that is exactly how a real error gets +// missed. +// +// It is idempotent and re-evaluated after every successful catalog refresh, so an +// aggregation layer (or the apiextensions group) installed later is picked up without a +// restart. Informers already started are never restarted, and a skip is logged once per +// resource per cluster, not once per refresh. +func (m *Manager) ensureAPISurfaceTriggerInformers(ctx context.Context, cc *clusterContext, log logr.Logger) { + m.triggersMu.Lock() + defer m.triggersMu.Unlock() + + stopCtx := m.triggerCtx + if stopCtx == nil { + // Start has not run yet; the first refresh happens inside it and re-enters here. return } - dynamicClient, err := dynamic.NewForConfig(cfg) - if err != nil { - log.Error(err, "failed to create API surface trigger client") + if !cc.catalog.Ready() { + log.V(1).Info("deferring API surface trigger informers - catalog not ready", + "cluster", describeCluster(cc.id)) return } + log = log.WithValues("cluster", describeCluster(cc.id)) + if cc.triggerFactory == nil { + dynamicClient, err := m.clusterDynamicClient(ctx, cc.id) + if err != nil { + log.V(1).Info("skipping API surface trigger informers - no client available", "err", err.Error()) + return + } + cc.triggerFactory = dynamicinformer.NewDynamicSharedInformerFactory(dynamicClient, 0) + } - factory := dynamicinformer.NewDynamicSharedInformerFactory(dynamicClient, 0) handler := cache.ResourceEventHandlerFuncs{ AddFunc: func(any) { m.signalCatalogRefresh() }, UpdateFunc: func(any, any) { m.signalCatalogRefresh() }, DeleteFunc: func(any) { m.signalCatalogRefresh() }, } - informers := []cache.SharedIndexInformer{ - factory.ForResource(crdTriggerGVR()).Informer(), - factory.ForResource(apiServiceTriggerGVR()).Informer(), + + start, unserved := selectAPISurfaceTriggers(cc.catalog, cc.triggersStarted) + for _, gvr := range unserved { + if _, logged := cc.triggersSkipLogged[gvr]; logged { + continue + } + cc.triggersSkipLogged[gvr] = struct{}{} + log.Info("API surface trigger not served by this API server; "+ + "the catalog refreshes on its periodic tick instead", "gvr", gvr.String()) } - for _, informer := range informers { + + var fresh []cache.SharedIndexInformer + for _, gvr := range start { + informer := cc.triggerFactory.ForResource(gvr).Informer() if _, addErr := informer.AddEventHandler(handler); addErr != nil { - log.Error(addErr, "failed to add API surface trigger handler") - return + log.Error(addErr, "failed to add API surface trigger handler", "gvr", gvr.String()) + continue } + cc.triggersStarted[gvr] = struct{}{} + delete(cc.triggersSkipLogged, gvr) + fresh = append(fresh, informer) + log.Info("API surface trigger informer started", "gvr", gvr.String()) + } + if len(fresh) == 0 { + return } - factory.Start(ctx.Done()) - go waitForAPISurfaceTriggerSync(ctx, log, informers) + // Start is idempotent per informer: it launches only the ones not yet running. + cc.triggerFactory.Start(stopCtx.Done()) + go waitForAPISurfaceTriggerSync(stopCtx, log, fresh) +} + +// apiSurfaceTriggerGVRs are the resources whose changes mean the API surface moved: a CRD +// (custom types appear/disappear) and an APIService (an aggregated group appears/goes +// unhealthy). Neither is guaranteed to be served. +func apiSurfaceTriggerGVRs() []schema.GroupVersionResource { + return []schema.GroupVersionResource{crdTriggerGVR(), apiServiceTriggerGVR()} +} + +// selectAPISurfaceTriggers splits the trigger resources not yet running into the ones +// discovery says are watchable now (start) and the ones it does not serve (unserved). It +// is the whole decision behind ensureAPISurfaceTriggerInformers, kept pure so the +// "no aggregation layer" case is testable without an API server. +func selectAPISurfaceTriggers( + catalog *APIResourceCatalog, + started map[schema.GroupVersionResource]struct{}, +) ([]schema.GroupVersionResource, []schema.GroupVersionResource) { + var start, unserved []schema.GroupVersionResource + for _, gvr := range apiSurfaceTriggerGVRs() { + if _, running := started[gvr]; running { + continue + } + if catalog.ServesWatchable(gvr) { + start = append(start, gvr) + continue + } + unserved = append(unserved, gvr) + } + return start, unserved +} + +// setTriggerContext records the manager's lifetime context, the stop channel every +// trigger informer is started with. Informers must outlive the reconcile call that +// discovers their resource became available, so they can never use its context. +func (m *Manager) setTriggerContext(ctx context.Context) { + m.triggersMu.Lock() + defer m.triggersMu.Unlock() + m.triggerCtx = ctx } func waitForAPISurfaceTriggerSync(ctx context.Context, log logr.Logger, informers []cache.SharedIndexInformer) { diff --git a/internal/watch/manager_snapshot_test.go b/internal/watch/manager_snapshot_test.go index da2bd29c..b18226f6 100644 --- a/internal/watch/manager_snapshot_test.go +++ b/internal/watch/manager_snapshot_test.go @@ -69,35 +69,43 @@ func gitTargetFixture() *configv1alpha3.GitTarget { // addSecretsWatchRule registers a namespaced WatchRule in ns-a for my-target watching secrets — // the standard single-namespaced-type fixture the splice/scope/audit-tail tests resolve against. func addSecretsWatchRule(store *rulestore.RuleStore) { - store.AddOrUpdateWatchRule( - configv1alpha3.WatchRule{ - ObjectMeta: metav1.ObjectMeta{Name: "wr-secrets", Namespace: "ns-a"}, - Spec: configv1alpha3.WatchRuleSpec{ - TargetRef: configv1alpha3.LocalTargetReference{Name: "my-target"}, - Rules: []configv1alpha3.ResourceRule{{ - APIGroups: []string{""}, APIVersions: []string{"v1"}, Resources: []string{"secrets"}, - }}, - }, + store.AddOrUpdateWatchRule(configv1alpha3.WatchRule{ + ObjectMeta: metav1.ObjectMeta{Name: "wr-secrets", Namespace: "ns-a"}, + Spec: configv1alpha3.WatchRuleSpec{ + TargetRef: configv1alpha3.LocalTargetReference{Name: "my-target"}, + Rules: []configv1alpha3.ResourceRule{{ + APIGroups: []string{""}, APIVersions: []string{"v1"}, Resources: []string{"secrets"}, + }}, }, - "my-target", "gitops-reverser", "provider", "gitops-reverser", "main", "live", - ) + }, rulestore.TargetBinding{ + GitTargetName: "my-target", + GitTargetNamespace: "gitops-reverser", + GitProviderName: "provider", + GitProviderNamespace: "gitops-reverser", + Branch: "main", + Path: "live", + }) } // addClusterWatchRule registers a cluster-scoped ClusterWatchRule for my-target. func addClusterWatchRule(store *rulestore.RuleStore, name, resource string) { - store.AddOrUpdateClusterWatchRule( - configv1alpha3.ClusterWatchRule{ - ObjectMeta: metav1.ObjectMeta{Name: name}, - Spec: configv1alpha3.ClusterWatchRuleSpec{ - TargetRef: configv1alpha3.NamespacedTargetReference{Name: "my-target", Namespace: "gitops-reverser"}, - Rules: []configv1alpha3.ClusterResourceRule{{ - APIGroups: []string{""}, APIVersions: []string{"v1"}, Resources: []string{resource}, - Scope: configv1alpha3.ResourceScopeCluster, - }}, - }, + store.AddOrUpdateClusterWatchRule(configv1alpha3.ClusterWatchRule{ + ObjectMeta: metav1.ObjectMeta{Name: name}, + Spec: configv1alpha3.ClusterWatchRuleSpec{ + TargetRef: configv1alpha3.NamespacedTargetReference{Name: "my-target", Namespace: "gitops-reverser"}, + Rules: []configv1alpha3.ClusterResourceRule{{ + APIGroups: []string{""}, APIVersions: []string{"v1"}, Resources: []string{resource}, + Scope: configv1alpha3.ResourceScopeCluster, + }}, }, - "my-target", "gitops-reverser", "provider", "gitops-reverser", "main", "live", - ) + }, rulestore.TargetBinding{ + GitTargetName: "my-target", + GitTargetNamespace: "gitops-reverser", + GitProviderName: "provider", + GitProviderNamespace: "gitops-reverser", + Branch: "main", + Path: "live", + }) } func myTargetRef() itypes.ResourceReference { @@ -172,18 +180,22 @@ func TestResolveSnapshotGVRs_ScopesNamespacedAndClusterWide(t *testing.T) { // so the snapshot is not silently narrowed. func TestResolveSnapshotGVRs_WildcardResourceExpands(t *testing.T) { store := rulestore.NewStore() - store.AddOrUpdateWatchRule( - configv1alpha3.WatchRule{ - ObjectMeta: metav1.ObjectMeta{Name: "wr-all", Namespace: "ns-a"}, - Spec: configv1alpha3.WatchRuleSpec{ - TargetRef: configv1alpha3.LocalTargetReference{Name: "my-target"}, - Rules: []configv1alpha3.ResourceRule{{ - APIGroups: []string{""}, APIVersions: []string{"v1"}, Resources: []string{"*"}, - }}, - }, + store.AddOrUpdateWatchRule(configv1alpha3.WatchRule{ + ObjectMeta: metav1.ObjectMeta{Name: "wr-all", Namespace: "ns-a"}, + Spec: configv1alpha3.WatchRuleSpec{ + TargetRef: configv1alpha3.LocalTargetReference{Name: "my-target"}, + Rules: []configv1alpha3.ResourceRule{{ + APIGroups: []string{""}, APIVersions: []string{"v1"}, Resources: []string{"*"}, + }}, }, - "my-target", "gitops-reverser", "provider", "gitops-reverser", "main", "live", - ) + }, rulestore.TargetBinding{ + GitTargetName: "my-target", + GitTargetNamespace: "gitops-reverser", + GitProviderName: "provider", + GitProviderNamespace: "gitops-reverser", + Branch: "main", + Path: "live", + }) m := streamingManager(t, gitTargetFixture(), store) gvrs, err := m.resolveSnapshotGVRs(context.Background(), myTargetRef()) diff --git a/internal/watch/materialization.go b/internal/watch/materialization.go index fea3d611..3e32f379 100644 --- a/internal/watch/materialization.go +++ b/internal/watch/materialization.go @@ -4,6 +4,7 @@ package watch import ( "context" + "fmt" "github.com/ConfigButler/gitops-reverser/internal/types" ) @@ -34,3 +35,30 @@ func (m *Manager) ForgetGitTargetDeclaration(gitDest types.ResourceReference) { defer m.declaredGVRsMu.Unlock() delete(m.declaredGVRs, gitDest.String()) } + +// RetargetGitTarget tears the current materialization down so the next Declare rebuilds it +// from a fresh full replay at the GitTarget's new destination. +// +// It differs from ForgetGitTargetDeclaration in one load-bearing way: it also drops the +// durable resume cursors. Those are keyed by GitTarget UID, and a retarget keeps the same +// object, so a resumed watch would deliver only the changes that happen after the move — +// the new folder would never receive the state that already existed. A deletion needs no +// such drop (a dead target's cursors expire, and a recreated one has a new UID). +// +// A cursor store that cannot be reached is reported, not swallowed: silently resuming into +// a new folder would mirror an arbitrary suffix of the cluster's state. +func (m *Manager) RetargetGitTarget(ctx context.Context, gitDest types.ResourceReference) error { + uid := m.resolveGitTargetUID(gitDest) + m.ForgetGitTargetDeclaration(gitDest) + + forgetter, ok := m.WatchCursorStore.(CursorForgetter) + if !ok || uid == "" { + // No Redis (watches cold-replay anyway), or a target that never declared. + return nil + } + if err := forgetter.ForgetWatchCursors(ctx, uid); err != nil { + return fmt.Errorf("forget watch cursors for %s: %w", gitDest.String(), err) + } + m.Log.Info("dropped watch resume cursors for retarget", "gitDest", gitDest.String(), "uid", uid) + return nil +} diff --git a/internal/watch/retarget_test.go b/internal/watch/retarget_test.go new file mode 100644 index 00000000..2ca90f20 --- /dev/null +++ b/internal/watch/retarget_test.go @@ -0,0 +1,107 @@ +// SPDX-License-Identifier: Apache-2.0 + +package watch + +import ( + "context" + "errors" + "testing" + + "github.com/go-logr/logr" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "k8s.io/apimachinery/pkg/runtime/schema" + + "github.com/ConfigButler/gitops-reverser/internal/types" +) + +// forgettingCursorStore satisfies both CursorStore and CursorForgetter. +type forgettingCursorStore struct { + forgotten []string + err error +} + +func (s *forgettingCursorStore) LookupWatchCursor( + context.Context, string, schema.GroupVersionResource, string, +) (string, bool) { + return "", false +} + +func (s *forgettingCursorStore) RecordWatchCursor( + context.Context, string, schema.GroupVersionResource, string, string, +) error { + return nil +} + +func (s *forgettingCursorStore) ForgetWatchCursors(_ context.Context, uid string) error { + s.forgotten = append(s.forgotten, uid) + return s.err +} + +// plainCursorStore is a CursorStore that cannot forget — the shape a future non-Redis +// implementation might have. It must simply not be asked. +type plainCursorStore struct{} + +func (plainCursorStore) LookupWatchCursor( + context.Context, string, schema.GroupVersionResource, string, +) (string, bool) { + return "", false +} + +func (plainCursorStore) RecordWatchCursor( + context.Context, string, schema.GroupVersionResource, string, string, +) error { + return nil +} + +func TestRetargetGitTarget_DropsCursorsForTheTargetsUID(t *testing.T) { + t.Parallel() + + store := &forgettingCursorStore{} + m := &Manager{Log: logr.Discard(), WatchCursorStore: store} + gitDest := types.NewResourceReference("acme", "team-a").WithUID("uid-1") + + require.NoError(t, m.RetargetGitTarget(context.Background(), gitDest)) + + assert.Equal(t, []string{"uid-1"}, store.forgotten, + "a retarget keeps the object's UID, so its cursors must be dropped explicitly") +} + +// Silently resuming into a new folder would mirror an arbitrary suffix of the cluster's +// state, so a cursor store that cannot be reached fails the reconcile. +func TestRetargetGitTarget_ReportsAFailingCursorStore(t *testing.T) { + t.Parallel() + + store := &forgettingCursorStore{err: errors.New("redis down")} + m := &Manager{Log: logr.Discard(), WatchCursorStore: store} + + err := m.RetargetGitTarget(context.Background(), types.NewResourceReference("acme", "team-a").WithUID("uid-1")) + + require.Error(t, err) + assert.Contains(t, err.Error(), "redis down") + assert.Contains(t, err.Error(), "team-a/acme") +} + +func TestRetargetGitTarget_NoCursorStoreIsFine(t *testing.T) { + t.Parallel() + + // Without Redis every watch cold-replays anyway, so there is nothing to forget. + m := &Manager{Log: logr.Discard()} + require.NoError(t, m.RetargetGitTarget(context.Background(), + types.NewResourceReference("acme", "team-a").WithUID("uid-1"))) + + m = &Manager{Log: logr.Discard(), WatchCursorStore: plainCursorStore{}} + require.NoError(t, m.RetargetGitTarget(context.Background(), + types.NewResourceReference("acme", "team-a").WithUID("uid-1"))) +} + +// A GitTarget that never declared has no UID to key cursors by. +func TestRetargetGitTarget_NeverDeclaredTargetForgetsNothing(t *testing.T) { + t.Parallel() + + store := &forgettingCursorStore{} + m := &Manager{Log: logr.Discard(), WatchCursorStore: store} + + require.NoError(t, m.RetargetGitTarget(context.Background(), types.NewResourceReference("acme", "team-a"))) + assert.Empty(t, store.forgotten) +} diff --git a/internal/watch/scope_resolve.go b/internal/watch/scope_resolve.go index 39a947a9..586e08d2 100644 --- a/internal/watch/scope_resolve.go +++ b/internal/watch/scope_resolve.go @@ -57,15 +57,16 @@ func (m *Manager) resolveSnapshotGVRForType( gitDest types.ResourceReference, gvr schema.GroupVersionResource, ) (snapshotGVR, bool, error) { - if err := m.RefreshAPIResourceCatalog(ctx); err != nil { + clusterID := m.clusterIDForGitTarget(gitDest) + if err := m.refreshClusterCatalog(ctx, clusterID); err != nil { return snapshotGVR{}, false, fmt.Errorf("refresh API resource catalog for %s: %w", gitDest.String(), err) } m.refreshWatchedTypeTables() - if !m.typeRegistryInstance().Ready() { + if !m.clusterRegistry(clusterID).Ready() { return snapshotGVR{}, false, fmt.Errorf( - "aborting per-type reconcile for %s: the cluster API surface has not been observed yet", - gitDest.String()) + "aborting per-type reconcile for %s: the %s cluster API surface has not been observed yet", + gitDest.String(), describeCluster(clusterID)) } table := m.residentWatchedTypeTable(gitDest) @@ -80,7 +81,7 @@ func (m *Manager) resolveSnapshotGVRForType( return snapshotGVR{}, false, nil } - if m.typeWobbling(gvr) { + if m.typeWobbling(clusterID, gvr) { return snapshotGVR{}, false, fmt.Errorf( "aborting per-type reconcile for %s: %s within the removal grace (currently unserved); "+ "refusing to reconcile a reduced view", @@ -101,16 +102,17 @@ func (m *Manager) resolveSnapshotGVRs( ctx context.Context, gitDest types.ResourceReference, ) ([]snapshotGVR, error) { - if err := m.RefreshAPIResourceCatalog(ctx); err != nil { + clusterID := m.clusterIDForGitTarget(gitDest) + if err := m.refreshClusterCatalog(ctx, clusterID); err != nil { return nil, fmt.Errorf("refresh API resource catalog for %s: %w", gitDest.String(), err) } m.refreshWatchedTypeTables() - if !m.typeRegistryInstance().Ready() { + if !m.clusterRegistry(clusterID).Ready() { return nil, fmt.Errorf( - "aborting scope resolution for %s: the cluster API surface has not been observed yet; "+ + "aborting scope resolution for %s: the %s cluster API surface has not been observed yet; "+ "refusing to reconcile a partial cluster view", - gitDest.String()) + gitDest.String(), describeCluster(clusterID)) } table := m.residentWatchedTypeTable(gitDest) @@ -133,7 +135,7 @@ func (m *Manager) resolveSnapshotGVRs( func (m *Manager) retainedWatchedTypes(table WatchedTypeTable) []schema.GroupVersionKind { var out []schema.GroupVersionKind for _, wt := range table.Types { - if m.typeWobbling(wt.GVR) { + if m.typeWobbling(table.ClusterID, wt.GVR) { out = append(out, wt.GVK) } } @@ -144,8 +146,8 @@ func (m *Manager) retainedWatchedTypes(table WatchedTypeTable) []schema.GroupVer // the removal grace, but not actually served right now (a discovery wobble). It is the single // "do not reconcile or sweep this type" predicate, shared by the whole-GitTarget scope resolve // and the per-type gate, so both fail closed on exactly the same registry verdict. -func (m *Manager) typeWobbling(gvr schema.GroupVersionResource) bool { - rec, ok := m.typeRegistryInstance().ByGVR(gvr) +func (m *Manager) typeWobbling(clusterID string, gvr schema.GroupVersionResource) bool { + rec, ok := m.clusterRegistry(clusterID).ByGVR(gvr) return ok && rec.Followability.Verdict == typeset.VerdictRetained } diff --git a/internal/watch/source_cluster_resolver.go b/internal/watch/source_cluster_resolver.go new file mode 100644 index 00000000..7b0cc096 --- /dev/null +++ b/internal/watch/source_cluster_resolver.go @@ -0,0 +1,94 @@ +// SPDX-License-Identifier: Apache-2.0 + +package watch + +import ( + "context" + "fmt" + "strings" + + corev1 "k8s.io/api/core/v1" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// secretSourceClusterResolver resolves a source-cluster id — "//", +// as GitTarget.SourceClusterID() renders it — into a rest.Config, by reading that Secret +// from the CONFIG PLANE: the cluster the operator runs in. +// +// This is the whole point of the split. The credential for a cluster never has to live on +// that cluster, and the watched cluster holds nothing but the watched resources — no +// Secret, no configbutler.ai CRDs at all. +type secretSourceClusterResolver struct { + // client reads Secrets from the config plane. Secret reads bypass the controller-runtime + // cache (see newManager), so a rotated kubeconfig is seen without a Secret informer. + client client.Client + + // qps and burst bound the rate at which the operator talks to a source cluster. A + // remote cluster is reached over a network the local one is not, so it gets client-side + // throttling the in-cluster config does not carry by default. + qps float32 + burst int +} + +// NewSecretSourceClusterResolver builds the production source-cluster resolver. +func NewSecretSourceClusterResolver(c client.Client, qps float32, burst int) SourceClusterResolver { + return &secretSourceClusterResolver{client: c, qps: qps, burst: burst} +} + +// sourceClusterRef is a source-cluster id parsed back into the Secret it names. +type sourceClusterRef struct { + Namespace string + Name string + Key string +} + +// sourceClusterIDSegments is the number of "/"-separated parts in a source-cluster id. +const sourceClusterIDSegments = 3 + +// parseSourceClusterID splits the id GitTarget.SourceClusterID() produces. It is a private +// encoding, never a user input: a GitTarget's namespace cannot contain "/" and neither can +// a Secret name, so the first two segments are unambiguous and the rest is the data key. +func parseSourceClusterID(id string) (sourceClusterRef, error) { + parts := strings.SplitN(id, "/", sourceClusterIDSegments) + if len(parts) != sourceClusterIDSegments || parts[0] == "" || parts[1] == "" || parts[2] == "" { + return sourceClusterRef{}, fmt.Errorf("malformed source cluster id %q, want //", id) + } + return sourceClusterRef{Namespace: parts[0], Name: parts[1], Key: parts[2]}, nil +} + +func (r *secretSourceClusterResolver) ResolveSourceCluster( + ctx context.Context, + clusterID string, +) (*rest.Config, string, error) { + ref, err := parseSourceClusterID(clusterID) + if err != nil { + return nil, "", err + } + + var secret corev1.Secret + if err := r.client.Get(ctx, client.ObjectKey{Namespace: ref.Namespace, Name: ref.Name}, &secret); err != nil { + return nil, "", fmt.Errorf("read kubeconfig Secret %s/%s: %w", ref.Namespace, ref.Name, err) + } + raw, ok := secret.Data[ref.Key] + if !ok || len(raw) == 0 { + return nil, "", fmt.Errorf("kubeconfig Secret %s/%s has no data under key %q", + ref.Namespace, ref.Name, ref.Key) + } + + cfg, err := clientcmd.RESTConfigFromKubeConfig(raw) + if err != nil { + return nil, "", fmt.Errorf("parse kubeconfig from Secret %s/%s key %q: %w", + ref.Namespace, ref.Name, ref.Key, err) + } + if r.qps > 0 { + cfg.QPS = r.qps + cfg.Burst = r.burst + } + + // The Secret's resourceVersion is the version token: it changes on every rotation, and + // on nothing else. The kubeconfig bytes themselves are dropped here — only the built + // rest.Config survives the call. + return cfg, secret.ResourceVersion, nil +} diff --git a/internal/watch/source_cluster_resolver_test.go b/internal/watch/source_cluster_resolver_test.go new file mode 100644 index 00000000..d5ae1f07 --- /dev/null +++ b/internal/watch/source_cluster_resolver_test.go @@ -0,0 +1,119 @@ +// SPDX-License-Identifier: Apache-2.0 + +package watch + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +const validKubeConfig = `apiVersion: v1 +kind: Config +clusters: +- name: acme + cluster: + server: https://acme.example:6443 +contexts: +- name: acme + context: + cluster: acme + user: acme +current-context: acme +users: +- name: acme + user: + token: abc123 +` + +func kubeConfigSecret(key, value string) *corev1.Secret { + return &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "acme-kubeconfig", Namespace: "team-a", ResourceVersion: "42"}, + Data: map[string][]byte{key: []byte(value)}, + } +} + +func newResolver(t *testing.T, objects ...runtime.Object) SourceClusterResolver { + t.Helper() + scheme := runtime.NewScheme() + require.NoError(t, corev1.AddToScheme(scheme)) + c := fake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(objects...).Build() + return NewSecretSourceClusterResolver(c, 20, 30) +} + +func TestParseSourceClusterID(t *testing.T) { + t.Parallel() + + ref, err := parseSourceClusterID("team-a/acme-kubeconfig/value.yaml") + require.NoError(t, err) + assert.Equal(t, sourceClusterRef{Namespace: "team-a", Name: "acme-kubeconfig", Key: "value.yaml"}, ref) + + // A data key may itself contain slashes; only the first two segments are structural. + ref, err = parseSourceClusterID("team-a/acme/nested/key.yaml") + require.NoError(t, err) + assert.Equal(t, "nested/key.yaml", ref.Key) + + for _, bad := range []string{"", "team-a", "team-a/acme", "/acme/key", "team-a//key", "team-a/acme/"} { + _, err := parseSourceClusterID(bad) + require.Error(t, err, "id %q must be rejected", bad) + } +} + +func TestSecretSourceClusterResolver_ResolvesAndVersions(t *testing.T) { + t.Parallel() + + r := newResolver(t, kubeConfigSecret("value.yaml", validKubeConfig)) + cfg, version, err := r.ResolveSourceCluster(context.Background(), "team-a/acme-kubeconfig/value.yaml") + + require.NoError(t, err) + assert.Equal(t, "https://acme.example:6443", cfg.Host) + assert.Equal(t, "42", version, "the Secret's resourceVersion is the rotation token") + assert.InDelta(t, 20.0, float64(cfg.QPS), 0.001, "a remote cluster gets client-side throttling") + assert.Equal(t, 30, cfg.Burst) +} + +func TestSecretSourceClusterResolver_MissingSecret(t *testing.T) { + t.Parallel() + + r := newResolver(t) + _, _, err := r.ResolveSourceCluster(context.Background(), "team-a/acme-kubeconfig/value.yaml") + + require.Error(t, err) + assert.Contains(t, err.Error(), "read kubeconfig Secret team-a/acme-kubeconfig") +} + +func TestSecretSourceClusterResolver_MissingKey(t *testing.T) { + t.Parallel() + + r := newResolver(t, kubeConfigSecret("other.yaml", validKubeConfig)) + _, _, err := r.ResolveSourceCluster(context.Background(), "team-a/acme-kubeconfig/value.yaml") + + require.Error(t, err) + assert.Contains(t, err.Error(), `no data under key "value.yaml"`) +} + +func TestSecretSourceClusterResolver_UnparseableKubeConfig(t *testing.T) { + t.Parallel() + + r := newResolver(t, kubeConfigSecret("value.yaml", "this is not a kubeconfig")) + _, _, err := r.ResolveSourceCluster(context.Background(), "team-a/acme-kubeconfig/value.yaml") + + require.Error(t, err) + assert.Contains(t, err.Error(), "parse kubeconfig") +} + +func TestSecretSourceClusterResolver_MalformedID(t *testing.T) { + t.Parallel() + + r := newResolver(t) + _, _, err := r.ResolveSourceCluster(context.Background(), "not-an-id") + + require.Error(t, err) + assert.Contains(t, err.Error(), "malformed source cluster id") +} diff --git a/internal/watch/stream_readiness.go b/internal/watch/stream_readiness.go index c6c3c265..c8d3bef0 100644 --- a/internal/watch/stream_readiness.go +++ b/internal/watch/stream_readiness.go @@ -130,8 +130,10 @@ func (m *Manager) StreamSummaryForGitTarget(gitDest types.ResourceReference) Str // StreamSummaryForWatchRule reports stream readiness for one namespaced WatchRule. func (m *Manager) StreamSummaryForWatchRule(rule configv1alpha3.WatchRule) StreamSummary { - m.refreshTypeRegistry() - records := m.typeRegistryInstance().Followable() + gitDest := types.NewResourceReference(rule.Spec.TargetRef.Name, rule.Namespace) + cc := m.cluster(m.clusterIDForGitTarget(gitDest)) + m.refreshTypeRegistry(cc) + records := cc.registry.Followable() var keys []targetWatchKey names := map[schema.GroupVersionResource]string{} for _, rr := range rule.Spec.Rules { @@ -143,17 +145,15 @@ func (m *Manager) StreamSummaryForWatchRule(rule configv1alpha3.WatchRule) Strea names[rec.Identity.GVR] = streamDisplayName(rec.Identity.GVR) } } - return m.streamSummaryForExpectedKeys( - types.NewResourceReference(rule.Spec.TargetRef.Name, rule.Namespace), - deduplicateTargetWatchKeys(keys), - names, - ) + return m.streamSummaryForExpectedKeys(gitDest, deduplicateTargetWatchKeys(keys), names) } // StreamSummaryForClusterWatchRule reports stream readiness for one ClusterWatchRule. func (m *Manager) StreamSummaryForClusterWatchRule(rule configv1alpha3.ClusterWatchRule) StreamSummary { - m.refreshTypeRegistry() - records := m.typeRegistryInstance().Followable() + cc := m.cluster(m.clusterIDForGitTarget( + types.NewResourceReference(rule.Spec.TargetRef.Name, rule.Spec.TargetRef.Namespace))) + m.refreshTypeRegistry(cc) + records := cc.registry.Followable() var keys []targetWatchKey names := map[schema.GroupVersionResource]string{} for _, rr := range rule.Spec.Rules { diff --git a/internal/watch/target_watch.go b/internal/watch/target_watch.go index 919c5ead..b5b955e6 100644 --- a/internal/watch/target_watch.go +++ b/internal/watch/target_watch.go @@ -60,13 +60,14 @@ func (m *Manager) EnsureGitTargetWatches( if m.EventRouter == nil { return nil } - if err := m.RefreshAPIResourceCatalog(ctx); err != nil { + clusterID := m.clusterIDForGitTarget(gitDest) + if err := m.refreshClusterCatalog(ctx, clusterID); err != nil { return fmt.Errorf("refresh API resource catalog for %s: %w", gitDest.String(), err) } m.refreshWatchedTypeTables() - if !m.typeRegistryInstance().Ready() { - return fmt.Errorf("aborting watch setup for %s: the cluster API surface has not been observed yet", - gitDest.String()) + if !m.clusterRegistry(clusterID).Ready() { + return fmt.Errorf("aborting watch setup for %s: the %s cluster API surface has not been observed yet", + gitDest.String(), describeCluster(clusterID)) } table := m.residentWatchedTypeTable(gitDest) @@ -82,6 +83,22 @@ func (m *Manager) replaceGitTargetWatches( ctx context.Context, table WatchedTypeTable, forceRecheck ...bool, +) error { + return m.replaceGitTargetWatchesIfRunning(ctx, table, false, forceRecheck...) +} + +// replaceGitTargetWatchesIfRunning reconciles a GitTarget's watch set. +// +// requireRunning=true means "only refresh a set that is still running": the periodic refresh +// must never CREATE one. Without that, a GitTarget torn down between the moment the refresh +// snapshotted the running keys and the moment it re-took the lock — a delete, or a retarget — +// would be resurrected: fresh watch goroutines launched against an object that no longer has +// a materialization, writing stream state and cursors after its teardown. +func (m *Manager) replaceGitTargetWatchesIfRunning( + ctx context.Context, + table WatchedTypeTable, + requireRunning bool, + forceRecheck ...bool, ) error { specs := targetWatchSpecs(table) keys := sortedTargetWatchSpecKeys(specs) @@ -93,7 +110,7 @@ func (m *Manager) replaceGitTargetWatches( m.targetWatches = map[string]*targetWatchSet{} } key := table.GitDest.Key() - if m.prepareTargetWatchSetReplacementLocked(key, specs, force) { + if m.skipTargetWatchReplacementLocked(key, specs, requireRunning, force) { m.targetWatchesMu.Unlock() cancel() return nil @@ -127,21 +144,25 @@ func (m *Manager) replaceGitTargetWatches( log := m.Log.WithName("target-watch").WithValues("gitDest", table.GitDest.String()) for _, watchKey := range keys { - ops := table.operationsFor(watchKey) - go m.runTargetWatch(childCtx, log, table.GitDest, watchKey, ops) + go m.runTargetWatch(childCtx, log, table.GitDest, watchKey, table.filterFor(watchKey)) } log.Info("watch-first target watch set reconciled", "watchCount", len(keys)) return nil } -func (m *Manager) prepareTargetWatchSetReplacementLocked( +// skipTargetWatchReplacementLocked reports whether this reconcile has nothing to do, and +// cancels the prior watch set when it does. It returns true for a set whose specs are +// unchanged, and — when requireRunning — for a target that has no running set at all. +func (m *Manager) skipTargetWatchReplacementLocked( key string, specs map[targetWatchKey]string, - force bool, + requireRunning, force bool, ) bool { prior := m.targetWatches[key] if prior == nil { - return false + // requireRunning: the set was torn down (deleted or retargeted) since the caller + // snapshotted it. This refresh must not bring it back. + return requireRunning } if !force && equalTargetWatchSpecs(prior.specs, specs) { return true @@ -164,7 +185,9 @@ func (m *Manager) refreshRunningTargetWatches(ctx context.Context) { if _, ok := running[table.GitDest.Key()]; !ok { continue } - if err := m.replaceGitTargetWatches(ctx, table); err != nil { + // requireRunning: the set may have been torn down (deleted or retargeted) since the + // snapshot above; this refresh must not bring it back. + if err := m.replaceGitTargetWatchesIfRunning(ctx, table, true); err != nil { m.Log.Error(err, "refresh running GitTarget watches failed", "gitDest", table.GitDest.String()) } } @@ -185,18 +208,30 @@ func (m *Manager) forgetGitTargetWatches(gitDest types.ResourceReference) { m.dropTargetGitPathAcceptanceLocked(gitDest) } +// watchFilter is one watch's admission state: the union operation set the stream must +// carry, and the per-rule clauses that decide whether a given live event is mirrored. +// Live events consult the clauses; the replay/snapshot path consults only ops, because an +// exclusion suppresses a write, never the state a mark-and-sweep reconciles against. +type watchFilter struct { + // cluster is the source cluster this watch runs against. LocalClusterID is the cluster + // the operator runs in. + cluster string + ops OperationSet + selections RuleSelections +} + func targetWatchSpecs(table WatchedTypeTable) map[targetWatchKey]string { out := map[targetWatchKey]string{} for _, wt := range table.Types { namespaces := wt.SnapshotNamespaces() if len(namespaces) == 0 { key := targetWatchKey{GVR: wt.GVR} - out[key] = operationSpec(wt.NamespaceOps[""]) + out[key] = watchSpec(table.ClusterID, wt.NamespaceSelections[""]) continue } for _, ns := range namespaces { key := targetWatchKey{GVR: wt.GVR, Namespace: ns} - out[key] = operationSpec(wt.NamespaceOps[ns]) + out[key] = watchSpec(table.ClusterID, wt.NamespaceSelections[ns]) } } return out @@ -216,11 +251,21 @@ func sortedTargetWatchSpecKeys(specs map[targetWatchKey]string) []targetWatchKey return out } -func operationSpec(ops OperationSet) string { - if len(ops) == 0 { - return "*" - } - return fmt.Sprint(ops.Sorted()) +// watchSpec fingerprints one watch's admission state. It covers the source cluster and the +// exclusions as well as the operations, so repointing a GitTarget at another cluster — or +// editing a rule's excludeUsers/excludeFieldManagers — restarts the affected watch instead +// of leaving the running goroutine bound to the old cluster and the old clauses. The watch +// key alone (GVR, namespace) is identical across clusters, so without the cluster here a +// source-cluster change would compare equal and reuse the running watch. +func watchSpec(clusterID string, selections RuleSelections) string { + spec := "*" + if len(selections) > 0 { + spec = selections.Key() + } + if clusterID == LocalClusterID { + return spec + } + return clusterID + "|" + spec } func equalTargetWatchSpecs(a, b map[targetWatchKey]string) bool { @@ -235,16 +280,24 @@ func equalTargetWatchSpecs(a, b map[targetWatchKey]string) bool { return true } -func (t WatchedTypeTable) operationsFor(key targetWatchKey) OperationSet { +// filterFor resolves the admission state for one watch key. A namespaced key falls back +// to the cluster-wide clauses, matching how a ClusterWatchRule's stream covers a +// namespaced type across every namespace. +func (t WatchedTypeTable) filterFor(key targetWatchKey) watchFilter { + selections := t.selectionsFor(key) + return watchFilter{cluster: t.ClusterID, ops: selections.Ops(), selections: selections} +} + +func (t WatchedTypeTable) selectionsFor(key targetWatchKey) RuleSelections { for _, wt := range t.Types { if wt.GVR != key.GVR { continue } - if ops := wt.NamespaceOps[key.Namespace]; ops != nil { - return ops + if selections := wt.NamespaceSelections[key.Namespace]; selections != nil { + return selections } if key.Namespace != "" { - return wt.NamespaceOps[""] + return wt.NamespaceSelections[""] } } return nil @@ -255,10 +308,10 @@ func (m *Manager) runTargetWatch( log logr.Logger, gitDest types.ResourceReference, key targetWatchKey, - ops OperationSet, + filter watchFilter, ) { for ctx.Err() == nil { - err := m.targetWatchReplayAndStream(ctx, log, gitDest, key, ops) + err := m.targetWatchReplayAndStream(ctx, log, gitDest, key, filter) if ctx.Err() != nil { return } @@ -278,11 +331,11 @@ func (m *Manager) targetWatchReplayAndStream( log logr.Logger, gitDest types.ResourceReference, key targetWatchKey, - ops OperationSet, + filter watchFilter, ) error { cursorExpired := false if cursor, ok := m.lookupTargetWatchCursor(ctx, gitDest, key); ok { - err := m.targetWatchResumeAndStream(ctx, log, gitDest, key, ops, cursor) + err := m.targetWatchResumeAndStream(ctx, log, gitDest, key, filter, cursor) if !errors.Is(err, errTargetWatchExpired) { return err } @@ -318,12 +371,12 @@ func (m *Manager) targetWatchReplayAndStream( "target watch replay in progress", ) replaying := true - w, err := m.openTargetWatch(ctx, key.GVR, key.Namespace, opts) + w, err := m.openTargetWatch(ctx, filter.cluster, key.GVR, key.Namespace, opts) if err != nil { if watchListUnsupported(err) { log.Error(err, "WARNING: sendInitialEvents unsupported; falling back to LIST plus buffered WATCH", "gvr", key.GVR.String(), "namespace", key.Namespace, "err", err.Error()) - return m.targetWatchListAndStream(ctx, log, gitDest, key, ops) + return m.targetWatchListAndStream(ctx, log, gitDest, key, filter) } if ctx.Err() != nil { return nil @@ -349,7 +402,7 @@ func (m *Manager) targetWatchReplayAndStream( return errTargetWatchClosed } nextReplaying, err := m.handleTargetWatchSessionEvent( - ctx, log, gitDest, key, ops, ev, replaying, &replay, + ctx, log, gitDest, key, filter, ev, replaying, &replay, ) if err != nil { return err @@ -364,10 +417,10 @@ func (m *Manager) targetWatchResumeAndStream( log logr.Logger, gitDest types.ResourceReference, key targetWatchKey, - ops OperationSet, + filter watchFilter, cursor string, ) error { - w, err := m.openTargetWatch(ctx, key.GVR, key.Namespace, metav1.ListOptions{ + w, err := m.openTargetWatch(ctx, filter.cluster, key.GVR, key.Namespace, metav1.ListOptions{ ResourceVersion: cursor, AllowWatchBookmarks: true, }) @@ -399,7 +452,7 @@ func (m *Manager) targetWatchResumeAndStream( "target watch resumed from durable cursor", ) m.recordTargetReconcileCompleted(gitDest, "cursor_resume") - return m.streamLiveTargetWatchEvents(ctx, log, gitDest, key, ops, w.ResultChan()) + return m.streamLiveTargetWatchEvents(ctx, log, gitDest, key, filter, w.ResultChan()) } func (m *Manager) targetWatchListAndStream( @@ -407,9 +460,9 @@ func (m *Manager) targetWatchListAndStream( log logr.Logger, gitDest types.ResourceReference, key targetWatchKey, - ops OperationSet, + filter watchFilter, ) error { - w, err := m.openTargetWatch(ctx, key.GVR, key.Namespace, metav1.ListOptions{ + w, err := m.openTargetWatch(ctx, filter.cluster, key.GVR, key.Namespace, metav1.ListOptions{ AllowWatchBookmarks: true, }) if err != nil { @@ -430,7 +483,7 @@ func (m *Manager) targetWatchListAndStream( buffered := make(chan watch.Event, targetWatchBufferCapacity) go bufferTargetWatchEvents(ctx, w.ResultChan(), buffered) - list, err := m.openTargetList(ctx, key.GVR, key.Namespace, metav1.ListOptions{}) + list, err := m.openTargetList(ctx, filter.cluster, key.GVR, key.Namespace, metav1.ListOptions{}) if err != nil { if ctx.Err() != nil { return nil @@ -462,7 +515,7 @@ func (m *Manager) targetWatchListAndStream( StreamReasonAllStreamsReady, "target watch list fallback complete", ) - return m.streamLiveTargetWatchEvents(ctx, log, gitDest, key, ops, buffered, revision) + return m.streamLiveTargetWatchEvents(ctx, log, gitDest, key, filter, buffered, revision) } func (m *Manager) handleTargetWatchSessionEvent( @@ -470,13 +523,13 @@ func (m *Manager) handleTargetWatchSessionEvent( log logr.Logger, gitDest types.ResourceReference, key targetWatchKey, - ops OperationSet, + filter watchFilter, ev watch.Event, replaying bool, replay *[]manifestanalyzer.DesiredResource, ) (bool, error) { if !replaying { - rv, err := m.routeLiveTargetWatchEvent(ctx, log, gitDest, key, ops, ev) + rv, err := m.routeLiveTargetWatchEvent(ctx, log, gitDest, key, filter, ev) if err != nil { return false, err } @@ -591,7 +644,7 @@ func (m *Manager) streamLiveTargetWatchEvents( log logr.Logger, gitDest types.ResourceReference, key targetWatchKey, - ops OperationSet, + filter watchFilter, events <-chan watch.Event, floors ...string, ) error { @@ -610,7 +663,7 @@ func (m *Manager) streamLiveTargetWatchEvents( if targetWatchEventAtOrBeforeFloor(ev, floor) { continue } - if err := m.processLiveTargetWatchEvent(ctx, log, gitDest, key, ops, ev); err != nil { + if err := m.processLiveTargetWatchEvent(ctx, log, gitDest, key, filter, ev); err != nil { return err } } @@ -622,7 +675,7 @@ func (m *Manager) processLiveTargetWatchEvent( log logr.Logger, gitDest types.ResourceReference, key targetWatchKey, - ops OperationSet, + filter watchFilter, ev watch.Event, ) error { if targetWatchExpired(ev) { @@ -631,7 +684,7 @@ func (m *Manager) processLiveTargetWatchEvent( // fresh replay (overwriting the stale cursor); no explicit delete needed. return errTargetWatchExpired } - rv, err := m.routeLiveTargetWatchEvent(ctx, log, gitDest, key, ops, ev) + rv, err := m.routeLiveTargetWatchEvent(ctx, log, gitDest, key, filter, ev) if err != nil { return err } @@ -643,7 +696,7 @@ func (m *Manager) routeLiveTargetWatchEvent( log logr.Logger, gitDest types.ResourceReference, key targetWatchKey, - ops OperationSet, + filter watchFilter, ev watch.Event, ) (string, error) { rv := targetWatchEventResourceVersion(ev) @@ -658,10 +711,16 @@ func (m *Manager) routeLiveTargetWatchEvent( return rv, nil } op := operationForLiveTargetWatchEvent(ev.Type, u) - if !ops.Match(op) { + if !filter.ops.Match(op) { return rv, nil } event := targetWatchGitEvent(key.GVR, u, op) + // Identity exclusion runs before the content dedup, so a write this GitTarget + // declines never seeds the dedup cache with content that was not routed to Git. + admission := m.admitLiveTargetWatchEvent(ctx, log, gitDest, key, filter, &event, u, op) + if !admission.Admitted { + return rv, nil + } // Drop a no-op UPDATE before it reaches the worker: a /status-only change // sanitizes to identical git content but ships unattributed (its /status audit // is dropped), so routing it would split an open commit window on the author @@ -672,7 +731,9 @@ func (m *Manager) routeLiveTargetWatchEvent( "resource", event.Identifier.String()) return rv, nil } - m.attachAuthor(ctx, &event, key.GVR, u) + if !admission.AuthorAttached { + m.attachAuthor(ctx, &event, key.GVR, u) + } if err := m.EventRouter.RouteToGitTargetEventStream(event, gitDest); err != nil { log.V(1).Info("target watch route failed", "gitDest", gitDest.String(), "gvr", key.GVR.String(), "err", err.Error()) @@ -686,6 +747,59 @@ func (m *Manager) routeLiveTargetWatchEvent( } } +// liveEventAdmission is the outcome of applying a rule's write exclusions to one live +// event. AuthorAttached lets the caller skip a second attribution lookup: attribution +// costs a bounded grace-window wait, normally deferred until after the content dedup so +// status-only churn never pays for it. Only excludeUsers forces it early, because the +// identity is the thing being matched. +type liveEventAdmission struct { + Admitted bool + AuthorAttached bool +} + +// admitLiveTargetWatchEvent applies the rules' write exclusions to one live event. +// Admitted=false means the event is dropped: a GitOps forward leg's own apply, mirrored +// back into the branch it came from, is the loop this exists to break. +func (m *Manager) admitLiveTargetWatchEvent( + ctx context.Context, + log logr.Logger, + gitDest types.ResourceReference, + key targetWatchKey, + filter watchFilter, + event *git.Event, + u *unstructured.Unstructured, + op string, +) liveEventAdmission { + if !filter.selections.HasExclusions() { + return liveEventAdmission{Admitted: true} + } + + // Empty for a DELETE: managedFields names the last writer, not the deleter. + lastWriters := lastWritersForOperation(op, u) + + username := "" + authorAttached := false + if filter.selections.NeedsAuthor() { + m.attachAuthor(ctx, event, key.GVR, u) + authorAttached = true + // Empty when attribution is off or the grace elapsed with no matching fact, which + // makes every excludeUsers clause fail open rather than lose a human's edit. + username = event.UserInfo.Username + } + + if filter.selections.Admits(op, lastWriters, username) { + return liveEventAdmission{Admitted: true, AuthorAttached: authorAttached} + } + + reason := filter.selections.ExclusionReason(op, lastWriters, username) + log.V(1).Info("target watch event excluded by rule", + "gitDest", gitDest.String(), "gvr", key.GVR.String(), + "resource", event.Identifier.String(), "operation", op, + "reason", reason, "lastWriters", lastWriters, "user", username) + m.recordExcludedWatchEvent(gitDest, key.GVR, reason) + return liveEventAdmission{AuthorAttached: authorAttached} +} + // attachAuthor names the commit author for a live watch event from the optional // attribution index. The live object still carries its UID and resourceVersion // here (sanitize strips them inside targetWatchGitEvent), so the resolver joins on @@ -831,6 +945,7 @@ func (s OperationSet) Match(op string) bool { func (m *Manager) openTargetWatch( ctx context.Context, + clusterID string, gvr schema.GroupVersionResource, namespace string, opts metav1.ListOptions, @@ -838,9 +953,10 @@ func (m *Manager) openTargetWatch( if m.targetWatchOpen != nil { return m.targetWatchOpen(ctx, gvr, namespace, opts) } - dc := m.dynamicClientFromConfig(m.Log) - if dc == nil { - return nil, errors.New("no dynamic client for target watch") + dc, err := m.clusterDynamicClient(ctx, clusterID) + if err != nil { + return nil, fmt.Errorf("no dynamic client for target watch on cluster %s: %w", + describeCluster(clusterID), err) } resource := dc.Resource(gvr) if namespace != "" { @@ -851,6 +967,7 @@ func (m *Manager) openTargetWatch( func (m *Manager) openTargetList( ctx context.Context, + clusterID string, gvr schema.GroupVersionResource, namespace string, opts metav1.ListOptions, @@ -858,9 +975,10 @@ func (m *Manager) openTargetList( if m.targetWatchList != nil { return m.targetWatchList(ctx, gvr, namespace, opts) } - dc := m.dynamicClientFromConfig(m.Log) - if dc == nil { - return nil, errors.New("no dynamic client for target watch list") + dc, err := m.clusterDynamicClient(ctx, clusterID) + if err != nil { + return nil, fmt.Errorf("no dynamic client for target list on cluster %s: %w", + describeCluster(clusterID), err) } resource := dc.Resource(gvr) if namespace != "" { diff --git a/internal/watch/target_watch_test.go b/internal/watch/target_watch_test.go index 65eb389f..13180b02 100644 --- a/internal/watch/target_watch_test.go +++ b/internal/watch/target_watch_test.go @@ -31,10 +31,10 @@ func TestTargetWatchSpecs_UsesOneWatchPerScope(t *testing.T) { Types: []WatchedType{ { GVR: configmapsGVR, - NamespaceOps: map[string]OperationSet{ + NamespaceSelections: nsSelections(map[string]OperationSet{ "apps": {"CREATE": struct{}{}}, "ops": {"UPDATE": struct{}{}}, - }, + }), }, { GVR: schema.GroupVersionResource{ @@ -42,9 +42,9 @@ func TestTargetWatchSpecs_UsesOneWatchPerScope(t *testing.T) { Version: "v1", Resource: "clusterroles", }, - NamespaceOps: map[string]OperationSet{ + NamespaceSelections: nsSelections(map[string]OperationSet{ "": {"*": struct{}{}}, - }, + }), }, }, } @@ -52,13 +52,46 @@ func TestTargetWatchSpecs_UsesOneWatchPerScope(t *testing.T) { specs := targetWatchSpecs(table) require.Len(t, specs, 3) - assert.Equal(t, "[CREATE]", specs[targetWatchKey{GVR: configmapsGVR, Namespace: "apps"}]) - assert.Equal(t, "[UPDATE]", specs[targetWatchKey{GVR: configmapsGVR, Namespace: "ops"}]) - assert.Equal(t, "[*]", specs[targetWatchKey{ + assert.Equal(t, "CREATE", specs[targetWatchKey{GVR: configmapsGVR, Namespace: "apps"}]) + assert.Equal(t, "UPDATE", specs[targetWatchKey{GVR: configmapsGVR, Namespace: "ops"}]) + assert.Equal(t, "*", specs[targetWatchKey{ GVR: schema.GroupVersionResource{Group: "rbac.authorization.k8s.io", Version: "v1", Resource: "clusterroles"}, }]) } +// The watch spec is the fingerprint replaceGitTargetWatches compares to decide whether a +// running watch may be reused. It must therefore cover the exclusions: a rule that starts +// declining Flux's writes has to restart its watch with the new clauses, not keep the +// goroutine that captured the old ones. +func TestTargetWatchSpecs_ExclusionChangeChangesTheSpec(t *testing.T) { + specFor := func(selections RuleSelections) string { + table := WatchedTypeTable{ + GitDest: types.NewResourceReference("target", "default"), + Types: []WatchedType{ + {GVR: configmapsGVR, NamespaceSelections: map[string]RuleSelections{"apps": selections}}, + }, + } + return targetWatchSpecs(table)[targetWatchKey{GVR: configmapsGVR, Namespace: "apps"}] + } + + ops := OperationSet{"UPDATE": struct{}{}} + plain := specFor(RuleSelections{{Ops: ops}}) + excludingFlux := specFor( + RuleSelections{{Ops: ops, Exclusion: newWriteExclusion([]string{"kustomize-controller"}, nil)}}, + ) + excludingArgo := specFor( + RuleSelections{{Ops: ops, Exclusion: newWriteExclusion([]string{"argocd-controller"}, nil)}}, + ) + + assert.NotEqual(t, plain, excludingFlux) + assert.NotEqual(t, excludingFlux, excludingArgo) + + // Declaring the same exclusions in a different order is the same watch. + orderA := specFor(RuleSelections{{Ops: ops, Exclusion: newWriteExclusion([]string{"a", "b"}, []string{"x", "y"})}}) + orderB := specFor(RuleSelections{{Ops: ops, Exclusion: newWriteExclusion([]string{"b", "a"}, []string{"y", "x"})}}) + assert.Equal(t, orderA, orderB) +} + func TestReplaceGitTargetWatches_ReusesUnchangedSetAndRestartsOnSpecChange(t *testing.T) { gitDest := types.NewResourceReference("target", "default") ctx, cancel := context.WithCancel(context.Background()) @@ -82,8 +115,8 @@ func TestReplaceGitTargetWatches_ReusesUnchangedSetAndRestartsOnSpecChange(t *te first := WatchedTypeTable{ GitDest: gitDest, Types: []WatchedType{{ - GVR: configmapsGVR, - NamespaceOps: map[string]OperationSet{"apps": {"CREATE": struct{}{}}}, + GVR: configmapsGVR, + NamespaceSelections: nsSelections(map[string]OperationSet{"apps": {"CREATE": struct{}{}}}), }}, } require.NoError(t, manager.replaceGitTargetWatches(ctx, first)) @@ -99,8 +132,8 @@ func TestReplaceGitTargetWatches_ReusesUnchangedSetAndRestartsOnSpecChange(t *te changed := WatchedTypeTable{ GitDest: gitDest, Types: []WatchedType{{ - GVR: configmapsGVR, - NamespaceOps: map[string]OperationSet{"apps": {"UPDATE": struct{}{}}}, + GVR: configmapsGVR, + NamespaceSelections: nsSelections(map[string]OperationSet{"apps": {"UPDATE": struct{}{}}}), }}, } require.NoError(t, manager.replaceGitTargetWatches(ctx, changed)) @@ -124,7 +157,7 @@ func TestRouteLiveTargetWatchEvent_ForwardsObjectEventsAsCommitter(t *testing.T) logr.Discard(), gitDest, targetWatchKey{GVR: configmapsGVR, Namespace: "apps"}, - OperationSet{"CREATE": struct{}{}}, + opsFilter(OperationSet{"CREATE": struct{}{}}), watch.Event{Type: watch.Added, Object: obj}, ) @@ -154,7 +187,7 @@ func TestRouteLiveTargetWatchEvent_RespectsOperationFilters(t *testing.T) { logr.Discard(), gitDest, targetWatchKey{GVR: configmapsGVR, Namespace: "apps"}, - OperationSet{"DELETE": struct{}{}}, + opsFilter(OperationSet{"DELETE": struct{}{}}), watch.Event{Type: watch.Modified, Object: configMapObject("13")}, ) @@ -189,7 +222,7 @@ func TestRouteLiveTargetWatchEvent_AttributesAuthorFromResolver(t *testing.T) { logr.Discard(), gitDest, targetWatchKey{GVR: configmapsGVR, Namespace: "apps"}, - OperationSet{"CREATE": struct{}{}}, + opsFilter(OperationSet{"CREATE": struct{}{}}), watch.Event{Type: watch.Added, Object: configMapObject("12")}, ) @@ -218,7 +251,7 @@ func TestRouteLiveTargetWatchEvent_DeletionTimestampRendersAsDelete(t *testing.T logr.Discard(), gitDest, targetWatchKey{GVR: configmapsGVR, Namespace: "apps"}, - OperationSet{"DELETE": struct{}{}}, + opsFilter(OperationSet{"DELETE": struct{}{}}), watch.Event{Type: watch.Modified, Object: terminatingConfigMapObject("20")}, ) @@ -247,7 +280,7 @@ func TestRouteLiveTargetWatchEvent_ModifiedWithoutDeletionTimestampRendersAsUpda logr.Discard(), gitDest, targetWatchKey{GVR: configmapsGVR, Namespace: "apps"}, - OperationSet{"UPDATE": struct{}{}}, + opsFilter(OperationSet{"UPDATE": struct{}{}}), watch.Event{Type: watch.Modified, Object: configMapObject("21")}, ) @@ -273,10 +306,10 @@ func TestRouteLiveTargetWatchEvent_TerminatingThenDeletedAreIdenticalRemovals(t key := targetWatchKey{GVR: configmapsGVR, Namespace: "apps"} ops := OperationSet{"DELETE": struct{}{}} - _, err := manager.routeLiveTargetWatchEvent(context.Background(), logr.Discard(), gitDest, key, ops, + _, err := manager.routeLiveTargetWatchEvent(context.Background(), logr.Discard(), gitDest, key, opsFilter(ops), watch.Event{Type: watch.Modified, Object: terminatingConfigMapObject("20")}) require.NoError(t, err) - _, err = manager.routeLiveTargetWatchEvent(context.Background(), logr.Discard(), gitDest, key, ops, + _, err = manager.routeLiveTargetWatchEvent(context.Background(), logr.Discard(), gitDest, key, opsFilter(ops), watch.Event{Type: watch.Deleted, Object: configMapObject("22")}) require.NoError(t, err) @@ -298,7 +331,7 @@ func TestHandleTargetWatchSessionEvent_CompletesReplayWithoutRouter(t *testing.T logr.Discard(), gitDest, key, - nil, + watchFilter{}, watch.Event{Type: watch.Added, Object: configMapObject("10")}, true, &replay, @@ -315,7 +348,7 @@ func TestHandleTargetWatchSessionEvent_CompletesReplayWithoutRouter(t *testing.T logr.Discard(), gitDest, key, - nil, + watchFilter{}, watch.Event{Type: watch.Bookmark, Object: bookmark}, true, &replay, @@ -346,7 +379,7 @@ func TestTargetWatchReplayAndStream_ReturnsWhenContextCancels(t *testing.T) { logr.Discard(), types.NewResourceReference("target", "default"), targetWatchKey{GVR: configmapsGVR, Namespace: "apps"}, - nil, + watchFilter{}, ) }() @@ -399,7 +432,7 @@ func TestTargetWatchReplayAndStream_FallsBackWhenReplayWatchIsForbidden(t *testi logr.Discard(), gitDest, targetWatchKey{GVR: configmapsGVR, Namespace: "apps"}, - nil, + watchFilter{}, ) }() @@ -459,7 +492,7 @@ func TestTargetWatchReplayAndStream_ResumesFromStoredCursor(t *testing.T) { logr.Discard(), gitDest, targetWatchKey{GVR: configmapsGVR, Namespace: "apps"}, - nil, + watchFilter{}, ) }() @@ -494,6 +527,7 @@ func TestOpenTargetWatch_UsesConfiguredHook(t *testing.T) { w, err := manager.openTargetWatch( context.Background(), + LocalClusterID, configmapsGVR, "apps", metav1.ListOptions{ResourceVersion: "42"}, @@ -519,11 +553,11 @@ func TestTargetWatchOperationHelpers(t *testing.T) { })) table := WatchedTypeTable{Types: []WatchedType{{ - GVR: configmapsGVR, - NamespaceOps: map[string]OperationSet{"": {"CREATE": struct{}{}}}, + GVR: configmapsGVR, + NamespaceSelections: nsSelections(map[string]OperationSet{"": {"CREATE": struct{}{}}}), }}} - assert.True(t, table.operationsFor(targetWatchKey{GVR: configmapsGVR, Namespace: "apps"}).Match("CREATE")) - assert.False(t, table.operationsFor(targetWatchKey{GVR: configmapsGVR, Namespace: "apps"}).Match("UPDATE")) + assert.True(t, table.filterFor(targetWatchKey{GVR: configmapsGVR, Namespace: "apps"}).ops.Match("CREATE")) + assert.False(t, table.filterFor(targetWatchKey{GVR: configmapsGVR, Namespace: "apps"}).ops.Match("UPDATE")) assert.True(t, OperationSet(nil).Match("DELETE")) assert.True(t, OperationSet{"*": struct{}{}}.Match("UPDATE")) assert.Equal(t, "DELETE", operationForWatchEvent(watch.Deleted)) @@ -631,7 +665,7 @@ func TestTargetWatchReplayAndStream_ExpiredCursorFallsBackToFreshReplay(t *testi go func() { done <- manager.targetWatchReplayAndStream( ctx, logr.Discard(), gitDest, - targetWatchKey{GVR: configmapsGVR, Namespace: "apps"}, nil, + targetWatchKey{GVR: configmapsGVR, Namespace: "apps"}, watchFilter{}, ) }() @@ -807,3 +841,29 @@ func (f *fakeWatchCursorStore) lastLookedUpUID() string { defer f.mu.Unlock() return f.lookedUpUID } + +// The watch key (GVR, namespace) is identical across clusters, so the source cluster has to +// be part of the spec fingerprint. Without it, repointing a GitTarget at another cluster +// would compare equal and reuse the running watch — still bound to the old cluster. +func TestTargetWatchSpecs_SourceClusterChangeChangesTheSpec(t *testing.T) { + specFor := func(clusterID string) string { + table := WatchedTypeTable{ + GitDest: types.NewResourceReference("target", "default"), + ClusterID: clusterID, + Types: []WatchedType{{ + GVR: configmapsGVR, + NamespaceSelections: nsSelections(map[string]OperationSet{"apps": {"UPDATE": struct{}{}}}), + }}, + } + return targetWatchSpecs(table)[targetWatchKey{GVR: configmapsGVR, Namespace: "apps"}] + } + + local := specFor(LocalClusterID) + remoteA := specFor("team-a/kubeconfig-a/value.yaml") + remoteB := specFor("team-a/kubeconfig-b/value.yaml") + + assert.NotEqual(t, local, remoteA) + assert.NotEqual(t, remoteA, remoteB) + // The local cluster keeps the bare fingerprint, so single-cluster installs see no churn. + assert.Equal(t, "UPDATE", local) +} diff --git a/internal/watch/watched_type_helpers_test.go b/internal/watch/watched_type_helpers_test.go index 30949a9b..976ab6a1 100644 --- a/internal/watch/watched_type_helpers_test.go +++ b/internal/watch/watched_type_helpers_test.go @@ -11,6 +11,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" configv1alpha3 "github.com/ConfigButler/gitops-reverser/api/v1alpha3" + "github.com/ConfigButler/gitops-reverser/internal/rulestore" ) // clusterRuleForResource builds a "test-target" ClusterWatchRule for one core/v1 namespaced @@ -54,10 +55,14 @@ func watchRuleForTarget(name, gitTargetName, namespace string) configv1alpha3.Wa // table set the splice scope resolution and demand Declare read. func TestRefreshWatchedTypeTables_ConcurrentRefreshesConverge(t *testing.T) { manager, store := makeWatchedTypeManager(t) - store.AddOrUpdateClusterWatchRule( - clusterRuleForResource("rule-1", "configmaps"), - "test-target", "test-ns", "test-provider", "test-ns", "main", "test-path", - ) + store.AddOrUpdateClusterWatchRule(clusterRuleForResource("rule-1", "configmaps"), rulestore.TargetBinding{ + GitTargetName: "test-target", + GitTargetNamespace: "test-ns", + GitProviderName: "test-provider", + GitProviderNamespace: "test-ns", + Branch: "main", + Path: "test-path", + }) var wg sync.WaitGroup for range 8 { @@ -71,10 +76,14 @@ func TestRefreshWatchedTypeTables_ConcurrentRefreshesConverge(t *testing.T) { }() } // Concurrently add a second rule mid-flight. - store.AddOrUpdateClusterWatchRule( - clusterRuleForResource("rule-2", "secrets"), - "test-target", "test-ns", "test-provider", "test-ns", "main", "test-path", - ) + store.AddOrUpdateClusterWatchRule(clusterRuleForResource("rule-2", "secrets"), rulestore.TargetBinding{ + GitTargetName: "test-target", + GitTargetNamespace: "test-ns", + GitProviderName: "test-provider", + GitProviderNamespace: "test-ns", + Branch: "main", + Path: "test-path", + }) wg.Wait() // A final settled refresh must reflect both rules. diff --git a/internal/watch/watched_type_metrics_test.go b/internal/watch/watched_type_metrics_test.go index 02b5417c..c12d0c21 100644 --- a/internal/watch/watched_type_metrics_test.go +++ b/internal/watch/watched_type_metrics_test.go @@ -8,6 +8,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/ConfigButler/gitops-reverser/internal/rulestore" "github.com/ConfigButler/gitops-reverser/internal/telemetry" ) @@ -18,10 +19,14 @@ func TestRefreshWatchedTypeTables_RecordsResolvedTypeGauge(t *testing.T) { require.NoError(t, err) manager, store := makeWatchedTypeManager(t) - store.AddOrUpdateClusterWatchRule( - clusterRuleForResource("rule-1", "configmaps"), - "test-target", "test-ns", "test-provider", "test-ns", "main", "test-path", - ) + store.AddOrUpdateClusterWatchRule(clusterRuleForResource("rule-1", "configmaps"), rulestore.TargetBinding{ + GitTargetName: "test-target", + GitTargetNamespace: "test-ns", + GitProviderName: "test-provider", + GitProviderNamespace: "test-ns", + Branch: "main", + Path: "test-path", + }) manager.refreshWatchedTypeTables() diff --git a/internal/watch/watched_type_resolver.go b/internal/watch/watched_type_resolver.go index 3eeaadba..73385b50 100644 --- a/internal/watch/watched_type_resolver.go +++ b/internal/watch/watched_type_resolver.go @@ -56,15 +56,17 @@ func (m *Manager) refreshWatchedTypeTables() { m.watchedTypes.refreshMu.Lock() defer m.watchedTypes.refreshMu.Unlock() - // Lazily populate the registry the first time (unit tests drive this path without - // RefreshAPIResourceCatalog); in production the catalog refresh keeps it current, so - // the heavy scan→registry rebuild stays off this path. - if !m.typeRegistryInstance().Ready() { - m.refreshTypeRegistry() + // Lazily populate each active cluster's registry the first time (unit tests drive this + // path without RefreshAPIResourceCatalog); in production the catalog refresh keeps them + // current, so the heavy scan→registry rebuild stays off this path. + for _, id := range m.activeClusterIDs() { + cc := m.cluster(id) + if !cc.registry.Ready() { + m.refreshTypeRegistry(cc) + } } - reg := m.typeRegistryInstance() - revision := reg.Revision() + revision, generation := m.clusterRegistryRevisions() fingerprint := m.rulesFingerprint() m.watchedTypes.mu.Lock() @@ -76,7 +78,7 @@ func (m *Manager) refreshWatchedTypeTables() { return } - tables := m.resolveWatchedTypeTables(reg.Generation()) + tables := m.resolveWatchedTypeTables(generation) m.watchedTypes.mu.Lock() previous := m.watchedTypes.tables @@ -89,6 +91,18 @@ func (m *Manager) refreshWatchedTypeTables() { recordWatchedTypeMetrics(previous, tables) } +// clusterRegistryRevisions folds every live cluster registry's revision and generation into +// one pair, so a discovery change on ANY watched cluster re-resolves the tables. Summing is +// enough: revisions only ever advance, so any change moves the sum. +func (m *Manager) clusterRegistryRevisions() (uint64, uint64) { + var revision, generation uint64 + for _, cc := range m.orderedClusters() { + revision += cc.registry.Revision() + generation += cc.registry.Generation() + } + return revision, generation +} + // recordWatchedTypeMetrics publishes the per-GitTarget watched-type count gauge after a // re-resolution. A GitTarget present before but gone now is zeroed so its series does // not linger. @@ -172,9 +186,34 @@ func (m *Manager) residentWatchedTypeTable(gitDest types.ResourceReference) Watc // targetSelections accumulates one GitTarget's selected followable records and write // destination while folding that target's rules. type targetSelections struct { - gitDest types.ResourceReference - dest string - selections []watchSelection + gitDest types.ResourceReference + dest string + // clusterID is the source cluster the rules for this GitTarget name. They all resolved + // the same GitTarget when they compiled, so in steady state they agree. + clusterID string + // clusterSet records whether clusterID has been assigned yet, so the first rule wins + // rather than the last. + clusterSet bool + // clusterConflict is true when two rules for this GitTarget disagree about the source + // cluster. That happens only in the window where a spec.sourceCluster change has + // recompiled some of the target's rules and not others. A table is then built with NO + // types: watching one cluster's objects while another cluster's types were resolved + // would mirror the wrong cluster into the folder, which is worse than mirroring none. + clusterConflict bool + selections []watchSelection +} + +// noteCluster folds one rule's source cluster into the target's, flagging a disagreement +// rather than letting the last rule silently win. +func (ts *targetSelections) noteCluster(clusterID string) { + if !ts.clusterSet { + ts.clusterID = clusterID + ts.clusterSet = true + return + } + if ts.clusterID != clusterID { + ts.clusterConflict = true + } } // resolveWatchedTypeTables projects every GitTarget's rules onto the type registry's @@ -185,49 +224,73 @@ func (m *Manager) resolveWatchedTypeTables(generation uint64) map[string]Watched if m.RuleStore == nil { return map[string]WatchedTypeTable{} } - records := m.typeRegistryInstance().Followable() byTarget := map[string]*targetSelections{} - get := func(ref types.ResourceReference, providerNS, provider, branch, path string) *targetSelections { + get := func(ref types.ResourceReference, binding targetBindingInfo) *targetSelections { key := ref.Key() ts := byTarget[key] if ts == nil { ts = &targetSelections{gitDest: ref} byTarget[key] = ts } - ts.dest = watchPlanDest(providerNS, provider, branch, path) + ts.dest = watchPlanDest(binding.providerNS, binding.provider, binding.branch, binding.path) + ts.noteCluster(binding.clusterID) return ts } - m.collectWatchRuleSelections(records, get) - m.collectClusterWatchRuleSelections(records, get) + m.collectWatchRuleSelections(get) + m.collectClusterWatchRuleSelections(get) tables := make(map[string]WatchedTypeTable, len(byTarget)) for key, ts := range byTarget { - table := buildWatchedTypeTable(ts.gitDest, generation, ts.selections) + selections := ts.selections + if ts.clusterConflict { + // A spec.sourceCluster change has recompiled some of this GitTarget's rules and + // not others. Open no watches until they agree: the GitTarget controller is + // already waiting on the same condition, and this keeps the data plane honest if + // it ever declares first. + m.Log.Info("GitTarget rules disagree about the source cluster; watching nothing until they agree", + "gitDest", ts.gitDest.String()) + selections = nil + } + table := buildWatchedTypeTable(ts.gitDest, generation, selections) table.Dest = ts.dest + table.ClusterID = ts.clusterID tables[key] = table } return tables } +// targetBindingInfo is the GitTarget-derived context a rule carries into table resolution. +type targetBindingInfo struct { + providerNS, provider, branch, path string + clusterID string +} + // collectWatchRuleSelections folds every namespaced WatchRule into its GitTarget's // selected records, scoping each record to the rule's own namespace. func (m *Manager) collectWatchRuleSelections( - records []typeset.TypeRecord, - get func(types.ResourceReference, string, string, string, string) *targetSelections, + get func(types.ResourceReference, targetBindingInfo) *targetSelections, ) { for _, rule := range m.RuleStore.SnapshotWatchRules() { ts := get( types.NewResourceReference(rule.GitTargetRef, rule.GitTargetNamespace), - rule.GitProviderNamespace, rule.GitProviderRef, rule.Branch, rule.Path, + targetBindingInfo{ + providerNS: rule.GitProviderNamespace, provider: rule.GitProviderRef, + branch: rule.Branch, path: rule.Path, clusterID: rule.SourceCluster, + }, ) + // Types resolve against the cluster this rule's GitTarget mirrors: a CRD installed + // only on the source cluster is followable, and one installed only locally is not. + records := m.clusterRegistry(rule.SourceCluster).Followable() for _, rr := range rule.ResourceRules { matched := matchFollowableRecords( records, rr.APIGroups, rr.APIVersions, rr.Resources, configv1alpha3.ResourceScopeNamespaced) + exclusion := newWriteExclusion(rr.ExcludeFieldManagers, rr.ExcludeUsers) + m.warnIfExcludeUsersWithoutAttribution(rule.Source.String(), exclusion) for _, rec := range matched { ts.selections = append(ts.selections, watchSelection{ - record: rec, namespace: rule.Source.Namespace, ops: rr.Operations, + record: rec, namespace: rule.Source.Namespace, ops: rr.Operations, exclusion: exclusion, }) } } @@ -237,19 +300,24 @@ func (m *Manager) collectWatchRuleSelections( // collectClusterWatchRuleSelections folds every ClusterWatchRule into its GitTarget's // selected records as cluster-wide streams. func (m *Manager) collectClusterWatchRuleSelections( - records []typeset.TypeRecord, - get func(types.ResourceReference, string, string, string, string) *targetSelections, + get func(types.ResourceReference, targetBindingInfo) *targetSelections, ) { for _, rule := range m.RuleStore.SnapshotClusterWatchRules() { ts := get( types.NewResourceReference(rule.GitTargetRef, rule.GitTargetNamespace), - rule.GitProviderNamespace, rule.GitProviderRef, rule.Branch, rule.Path, + targetBindingInfo{ + providerNS: rule.GitProviderNamespace, provider: rule.GitProviderRef, + branch: rule.Branch, path: rule.Path, clusterID: rule.SourceCluster, + }, ) + records := m.clusterRegistry(rule.SourceCluster).Followable() for _, rr := range rule.Rules { matched := matchFollowableRecords(records, rr.APIGroups, rr.APIVersions, rr.Resources, rr.Scope) + exclusion := newWriteExclusion(rr.ExcludeFieldManagers, rr.ExcludeUsers) + m.warnIfExcludeUsersWithoutAttribution(rule.Source.String(), exclusion) for _, rec := range matched { ts.selections = append(ts.selections, watchSelection{ - record: rec, namespace: "", ops: rr.Operations, + record: rec, namespace: "", ops: rr.Operations, exclusion: exclusion, }) } } diff --git a/internal/watch/watched_type_resolver_test.go b/internal/watch/watched_type_resolver_test.go index eb043635..4b555745 100644 --- a/internal/watch/watched_type_resolver_test.go +++ b/internal/watch/watched_type_resolver_test.go @@ -36,10 +36,14 @@ func gitDestRef(name string) types.ResourceReference { func TestRefreshWatchedTypeTables_ClusterWatchRuleResolvesClusterWideType(t *testing.T) { manager, store := makeWatchedTypeManager(t) - store.AddOrUpdateClusterWatchRule( - clusterRuleForResource("rule-1", "configmaps"), - "test-target", "test-ns", "test-provider", "test-ns", "main", "test-path", - ) + store.AddOrUpdateClusterWatchRule(clusterRuleForResource("rule-1", "configmaps"), rulestore.TargetBinding{ + GitTargetName: "test-target", + GitTargetNamespace: "test-ns", + GitProviderName: "test-provider", + GitProviderNamespace: "test-ns", + Branch: "main", + Path: "test-path", + }) manager.refreshWatchedTypeTables() @@ -55,14 +59,22 @@ func TestRefreshWatchedTypeTables_ClusterWatchRuleResolvesClusterWideType(t *tes func TestRefreshWatchedTypeTables_WatchRuleScopesTypeToItsNamespace(t *testing.T) { manager, store := makeWatchedTypeManager(t) - store.AddOrUpdateWatchRule( - watchRuleForTarget("rule-a", "wt-ns-target", "ns-a"), - "wt-ns-target", "test-ns", "test-provider", "test-ns", "main", "test-path", - ) - store.AddOrUpdateWatchRule( - watchRuleForTarget("rule-b", "wt-ns-target", "ns-b"), - "wt-ns-target", "test-ns", "test-provider", "test-ns", "main", "test-path", - ) + store.AddOrUpdateWatchRule(watchRuleForTarget("rule-a", "wt-ns-target", "ns-a"), rulestore.TargetBinding{ + GitTargetName: "wt-ns-target", + GitTargetNamespace: "test-ns", + GitProviderName: "test-provider", + GitProviderNamespace: "test-ns", + Branch: "main", + Path: "test-path", + }) + store.AddOrUpdateWatchRule(watchRuleForTarget("rule-b", "wt-ns-target", "ns-b"), rulestore.TargetBinding{ + GitTargetName: "wt-ns-target", + GitTargetNamespace: "test-ns", + GitProviderName: "test-provider", + GitProviderNamespace: "test-ns", + Branch: "main", + Path: "test-path", + }) manager.refreshWatchedTypeTables() @@ -75,19 +87,27 @@ func TestRefreshWatchedTypeTables_WatchRuleScopesTypeToItsNamespace(t *testing.T func TestRefreshWatchedTypeTables_RuleChangeReResolves(t *testing.T) { manager, store := makeWatchedTypeManager(t) - store.AddOrUpdateClusterWatchRule( - clusterRuleForResource("rule-1", "configmaps"), - "test-target", "test-ns", "test-provider", "test-ns", "main", "test-path", - ) + store.AddOrUpdateClusterWatchRule(clusterRuleForResource("rule-1", "configmaps"), rulestore.TargetBinding{ + GitTargetName: "test-target", + GitTargetNamespace: "test-ns", + GitProviderName: "test-provider", + GitProviderNamespace: "test-ns", + Branch: "main", + Path: "test-path", + }) manager.refreshWatchedTypeTables() first, _ := manager.watchedTypeTableForGitDest(gitDestRef("test-target")) require.Len(t, first.Types, 1) // A second rule selecting a different resource is reflected on the next refresh. - store.AddOrUpdateClusterWatchRule( - clusterRuleForResource("rule-2", "secrets"), - "test-target", "test-ns", "test-provider", "test-ns", "main", "test-path", - ) + store.AddOrUpdateClusterWatchRule(clusterRuleForResource("rule-2", "secrets"), rulestore.TargetBinding{ + GitTargetName: "test-target", + GitTargetNamespace: "test-ns", + GitProviderName: "test-provider", + GitProviderNamespace: "test-ns", + Branch: "main", + Path: "test-path", + }) manager.refreshWatchedTypeTables() second, _ := manager.watchedTypeTableForGitDest(gitDestRef("test-target")) @@ -102,10 +122,14 @@ func TestResolveWatchedTypeTables_NilRuleStoreIsEmpty(t *testing.T) { func TestRefreshWatchedTypeTables_NoChangeReusesResolvedTables(t *testing.T) { manager, store := makeWatchedTypeManager(t) - store.AddOrUpdateClusterWatchRule( - clusterRuleForResource("rule-1", "configmaps"), - "test-target", "test-ns", "test-provider", "test-ns", "main", "test-path", - ) + store.AddOrUpdateClusterWatchRule(clusterRuleForResource("rule-1", "configmaps"), rulestore.TargetBinding{ + GitTargetName: "test-target", + GitTargetNamespace: "test-ns", + GitProviderName: "test-provider", + GitProviderNamespace: "test-ns", + Branch: "main", + Path: "test-path", + }) manager.refreshWatchedTypeTables() manager.watchedTypes.mu.Lock() firstRev := manager.watchedTypes.revision @@ -122,17 +146,25 @@ func TestRefreshWatchedTypeTables_NoChangeReusesResolvedTables(t *testing.T) { func TestRulesFingerprint_StableUntilRuleChanges(t *testing.T) { manager, store := makeWatchedTypeManager(t) - store.AddOrUpdateClusterWatchRule( - clusterRuleForResource("rule-1", "configmaps"), - "test-target", "test-ns", "test-provider", "test-ns", "main", "test-path", - ) + store.AddOrUpdateClusterWatchRule(clusterRuleForResource("rule-1", "configmaps"), rulestore.TargetBinding{ + GitTargetName: "test-target", + GitTargetNamespace: "test-ns", + GitProviderName: "test-provider", + GitProviderNamespace: "test-ns", + Branch: "main", + Path: "test-path", + }) fp1 := manager.rulesFingerprint() assert.Equal(t, fp1, manager.rulesFingerprint(), "fingerprint must be stable for unchanged rules") - store.AddOrUpdateClusterWatchRule( - clusterRuleForResource("rule-2", "secrets"), - "test-target", "test-ns", "test-provider", "test-ns", "main", "test-path", - ) + store.AddOrUpdateClusterWatchRule(clusterRuleForResource("rule-2", "secrets"), rulestore.TargetBinding{ + GitTargetName: "test-target", + GitTargetNamespace: "test-ns", + GitProviderName: "test-provider", + GitProviderNamespace: "test-ns", + Branch: "main", + Path: "test-path", + }) assert.NotEqual(t, fp1, manager.rulesFingerprint(), "a new rule must move the fingerprint") } @@ -140,10 +172,14 @@ func TestRefreshWatchedTypeTables_KeepsTargetWithUnresolvableRulesAsEmptyTable(t manager, store := makeWatchedTypeManager(t) // "ghosts" is not served by the common catalog: the rule resolves to nothing, // but the GitTarget must still appear as an empty table, not vanish. - store.AddOrUpdateClusterWatchRule( - clusterRuleForResource("rule-1", "ghosts"), - "test-target", "test-ns", "test-provider", "test-ns", "main", "test-path", - ) + store.AddOrUpdateClusterWatchRule(clusterRuleForResource("rule-1", "ghosts"), rulestore.TargetBinding{ + GitTargetName: "test-target", + GitTargetNamespace: "test-ns", + GitProviderName: "test-provider", + GitProviderNamespace: "test-ns", + Branch: "main", + Path: "test-path", + }) manager.refreshWatchedTypeTables() table, ok := manager.watchedTypeTableForGitDest(gitDestRef("test-target")) @@ -157,21 +193,25 @@ func TestRefreshWatchedTypeTables_KeepsTargetWithUnresolvableRulesAsEmptyTable(t func TestRefreshWatchedTypeTables_ExcludesAmbiguousGVK(t *testing.T) { store := rulestore.NewStore() manager := &Manager{Log: logr.Discard(), RuleStore: store, resourceCatalog: newWidgetConflictCatalog(t)} - store.AddOrUpdateClusterWatchRule( - configv1alpha3.ClusterWatchRule{ - ObjectMeta: metav1.ObjectMeta{Name: "rule-widgets"}, - Spec: configv1alpha3.ClusterWatchRuleSpec{ - TargetRef: configv1alpha3.NamespacedTargetReference{Name: "test-target", Namespace: "test-ns"}, - Rules: []configv1alpha3.ClusterResourceRule{{ - APIGroups: []string{"example.com"}, - APIVersions: []string{"v1"}, - Resources: []string{"*"}, - Scope: configv1alpha3.ResourceScopeNamespaced, - }}, - }, + store.AddOrUpdateClusterWatchRule(configv1alpha3.ClusterWatchRule{ + ObjectMeta: metav1.ObjectMeta{Name: "rule-widgets"}, + Spec: configv1alpha3.ClusterWatchRuleSpec{ + TargetRef: configv1alpha3.NamespacedTargetReference{Name: "test-target", Namespace: "test-ns"}, + Rules: []configv1alpha3.ClusterResourceRule{{ + APIGroups: []string{"example.com"}, + APIVersions: []string{"v1"}, + Resources: []string{"*"}, + Scope: configv1alpha3.ResourceScopeNamespaced, + }}, }, - "test-target", "test-ns", "test-provider", "test-ns", "main", "test-path", - ) + }, rulestore.TargetBinding{ + GitTargetName: "test-target", + GitTargetNamespace: "test-ns", + GitProviderName: "test-provider", + GitProviderNamespace: "test-ns", + Branch: "main", + Path: "test-path", + }) manager.refreshWatchedTypeTables() diff --git a/internal/watch/watched_type_table.go b/internal/watch/watched_type_table.go index fc7a89e6..cd6d67d8 100644 --- a/internal/watch/watched_type_table.go +++ b/internal/watch/watched_type_table.go @@ -60,18 +60,28 @@ type WatchedType struct { ServedVersion string Preferred bool - // NamespaceOps maps each watched namespace to the union of operation filters - // for this type in that namespace. The empty-string key is a cluster-wide - // stream: a cluster-scoped resource, or a namespaced resource a ClusterWatchRule - // follows across every namespace. - NamespaceOps map[string]OperationSet + // NamespaceSelections maps each watched namespace to the rule clauses that select + // this type there: an operation set plus that rule's write exclusions. The + // empty-string key is a cluster-wide stream: a cluster-scoped resource, or a + // namespaced resource a ClusterWatchRule follows across every namespace. + // + // Clauses are kept per-rule rather than merged, because an exclusion vetoes only + // within its own rule — a merged view could not express "rule A excludes Flux, rule + // B does not", which must admit Flux (rules are a logical OR). + NamespaceSelections map[string]RuleSelections +} + +// NamespaceOps is the per-namespace union of operation filters — what each watch must +// stream, independent of which rule admits a given event. +func (t WatchedType) NamespaceOps(namespace string) OperationSet { + return t.NamespaceSelections[namespace].Ops() } // ClusterWide reports whether this type is gathered with a single cluster-wide // stream, true for a cluster-scoped resource and for a namespaced resource a // ClusterWatchRule follows across all namespaces. func (t WatchedType) ClusterWide() bool { - _, ok := t.NamespaceOps[""] + _, ok := t.NamespaceSelections[""] return ok } @@ -83,8 +93,8 @@ func (t WatchedType) SnapshotNamespaces() []string { if t.ClusterWide() { return nil } - out := make([]string, 0, len(t.NamespaceOps)) - for ns := range t.NamespaceOps { + out := make([]string, 0, len(t.NamespaceSelections)) + for ns := range t.NamespaceSelections { out = append(out, ns) } sort.Strings(out) @@ -100,71 +110,116 @@ type WatchedTypeTable struct { GitDest types.ResourceReference // Dest is the GitTarget's write destination fingerprint (provider/branch/path), // carried so the effective-plan hash can be derived from the table alone. - Dest string + Dest string + // ClusterID is the source cluster this GitTarget mirrors FROM: the cluster its watches + // open against and its types were resolved on. LocalClusterID is the cluster the + // operator runs in. + ClusterID string Types []WatchedType ResolvedAt uint64 } // watchSelection is one followable registry record a rule selected for a GitTarget, -// with the namespace it was selected under ("" = cluster-wide stream) and the rule's -// operation filters. +// with the namespace it was selected under ("" = cluster-wide stream), the rule's +// operation filters, and the rule's write exclusions. type watchSelection struct { record typeset.TypeRecord namespace string ops []configv1alpha3.OperationType + exclusion WriteExclusion } -// watchedTypeAccum accumulates one followable record's namespace/operation scope while -// folding a GitTarget's selections. +// watchedTypeAccum accumulates one followable record's namespace scope while folding a +// GitTarget's selections. Within a namespace, clauses are keyed by their exclusion +// fingerprint so two rules that decline the same writers fold their operations together, +// while rules that decline different writers stay distinct. type watchedTypeAccum struct { record typeset.TypeRecord - namespaceOps map[string]OperationSet + namespaces map[string]map[string]*RuleSelection + nsOrder []string + clauseOrders map[string][]string } // buildWatchedTypeTable folds a GitTarget's selected followable records into its -// watched-type table, unioning each record's per-namespace operation filters. Identity -// and followability are already settled by the registry, so this is a pure fold with no -// catalog lookup and no conflict decision. +// watched-type table. Identity and followability are already settled by the registry, so +// this is a pure fold with no catalog lookup and no conflict decision. func buildWatchedTypeTable( gitDest types.ResourceReference, generation uint64, selections []watchSelection, ) WatchedTypeTable { byGVR := map[schema.GroupVersionResource]*watchedTypeAccum{} + var gvrOrder []schema.GroupVersionResource for _, sel := range selections { gvr := sel.record.Identity.GVR acc := byGVR[gvr] if acc == nil { - acc = &watchedTypeAccum{record: sel.record, namespaceOps: map[string]OperationSet{}} + acc = &watchedTypeAccum{ + record: sel.record, + namespaces: map[string]map[string]*RuleSelection{}, + clauseOrders: map[string][]string{}, + } byGVR[gvr] = acc + gvrOrder = append(gvrOrder, gvr) } - opSet := acc.namespaceOps[sel.namespace] - if opSet == nil { - opSet = OperationSet{} - acc.namespaceOps[sel.namespace] = opSet - } - opSet.add(sel.ops) + acc.add(sel) } table := WatchedTypeTable{GitDest: gitDest, ResolvedAt: generation} - for _, acc := range byGVR { - table.Types = append(table.Types, watchedTypeFromRecord(acc.record, acc.namespaceOps)) + for _, gvr := range gvrOrder { + acc := byGVR[gvr] + table.Types = append(table.Types, watchedTypeFromRecord(acc.record, acc.namespaceSelections())) } sortWatchedTypes(table.Types) return table } +// add folds one rule's selection into the accumulator, merging it with any earlier rule +// that declines exactly the same writers. +func (a *watchedTypeAccum) add(sel watchSelection) { + clauses := a.namespaces[sel.namespace] + if clauses == nil { + clauses = map[string]*RuleSelection{} + a.namespaces[sel.namespace] = clauses + a.nsOrder = append(a.nsOrder, sel.namespace) + } + key := sel.exclusion.Key() + clause := clauses[key] + if clause == nil { + clause = &RuleSelection{Ops: OperationSet{}, Exclusion: sel.exclusion} + clauses[key] = clause + a.clauseOrders[sel.namespace] = append(a.clauseOrders[sel.namespace], key) + } + clause.Ops.add(sel.ops) +} + +// namespaceSelections materializes the accumulated clauses in a deterministic order, so +// the watch-spec fingerprint is stable across reconciles. +func (a *watchedTypeAccum) namespaceSelections() map[string]RuleSelections { + out := make(map[string]RuleSelections, len(a.namespaces)) + for _, ns := range a.nsOrder { + keys := append([]string(nil), a.clauseOrders[ns]...) + sort.Strings(keys) + selections := make(RuleSelections, 0, len(keys)) + for _, key := range keys { + selections = append(selections, *a.namespaces[ns][key]) + } + out[ns] = selections + } + return out +} + // watchedTypeFromRecord copies a followable registry record's identity into a -// WatchedType, attaching the per-namespace operation scope the rules folded. -func watchedTypeFromRecord(rec typeset.TypeRecord, namespaceOps map[string]OperationSet) WatchedType { +// WatchedType, attaching the per-namespace rule clauses the rules folded. +func watchedTypeFromRecord(rec typeset.TypeRecord, namespaceSelections map[string]RuleSelections) WatchedType { return WatchedType{ - GVK: rec.Identity.GVK, - GVR: rec.Identity.GVR, - Namespaced: rec.Identity.Scope == typeset.ScopeNamespaced, - Scope: resourceScopeFor(rec.Identity.Scope), - ServedVersion: rec.Identity.GVR.Version, - Preferred: rec.Preferred, - NamespaceOps: namespaceOps, + GVK: rec.Identity.GVK, + GVR: rec.Identity.GVR, + Namespaced: rec.Identity.Scope == typeset.ScopeNamespaced, + Scope: resourceScopeFor(rec.Identity.Scope), + ServedVersion: rec.Identity.GVR.Version, + Preferred: rec.Preferred, + NamespaceSelections: namespaceSelections, } } diff --git a/internal/watch/watched_type_table_test.go b/internal/watch/watched_type_table_test.go index db0867e0..57269b0c 100644 --- a/internal/watch/watched_type_table_test.go +++ b/internal/watch/watched_type_table_test.go @@ -77,8 +77,8 @@ func TestBuildWatchedTypeTable_ClusterWideOverridesNamedNamespaces(t *testing.T) wt := table.Types[0] assert.True(t, wt.ClusterWide()) assert.Empty(t, wt.SnapshotNamespaces()) - assert.Contains(t, wt.NamespaceOps, "") - assert.Contains(t, wt.NamespaceOps, "team-a") + assert.Contains(t, wt.NamespaceSelections, "") + assert.Contains(t, wt.NamespaceSelections, "team-a") } func TestBuildWatchedTypeTable_OperationsUnionPerNamespace(t *testing.T) { @@ -93,8 +93,8 @@ func TestBuildWatchedTypeTable_OperationsUnionPerNamespace(t *testing.T) { require.Len(t, table.Types, 1) wt := table.Types[0] - assert.Equal(t, []string{"CREATE", "UPDATE"}, wt.NamespaceOps["team-a"].Sorted()) - assert.Equal(t, []string{"*"}, wt.NamespaceOps["team-b"].Sorted()) + assert.Equal(t, []string{"CREATE", "UPDATE"}, wt.NamespaceOps("team-a").Sorted()) + assert.Equal(t, []string{"*"}, wt.NamespaceOps("team-b").Sorted()) } func TestBuildWatchedTypeTable_EmptyOperationsAreAllOperations(t *testing.T) { @@ -105,7 +105,7 @@ func TestBuildWatchedTypeTable_EmptyOperationsAreAllOperations(t *testing.T) { table := buildWatchedTypeTable(testGitDest(), 1, selections) require.Len(t, table.Types, 1) - assert.Equal(t, []string{"*"}, table.Types[0].NamespaceOps["team-a"].Sorted()) + assert.Equal(t, []string{"*"}, table.Types[0].NamespaceOps("team-a").Sorted()) } func TestBuildWatchedTypeTable_ClusterScopedType(t *testing.T) { diff --git a/internal/watch/write_exclusion.go b/internal/watch/write_exclusion.go new file mode 100644 index 00000000..e9c3b6f9 --- /dev/null +++ b/internal/watch/write_exclusion.go @@ -0,0 +1,338 @@ +// SPDX-License-Identifier: Apache-2.0 + +package watch + +import ( + "context" + "sort" + "strings" + + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/metric" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + + configv1alpha3 "github.com/ConfigButler/gitops-reverser/api/v1alpha3" + "github.com/ConfigButler/gitops-reverser/internal/telemetry" + "github.com/ConfigButler/gitops-reverser/internal/types" +) + +// WriteExclusion is one rule's negative clause: the field managers and identities whose +// writes that rule declines to mirror. The zero value excludes nothing. +type WriteExclusion struct { + // FieldManagers is the sorted, de-duplicated set from ResourceRule.ExcludeFieldManagers. + FieldManagers []string + // Users is the sorted, de-duplicated set from ResourceRule.ExcludeUsers. + Users []string +} + +// newWriteExclusion normalizes a rule's two exclusion slices so two rules that declare the +// same exclusions in a different order fold together in the watched-type table. +func newWriteExclusion(fieldManagers, users []string) WriteExclusion { + return WriteExclusion{ + FieldManagers: normalizeExclusionList(fieldManagers), + Users: normalizeExclusionList(users), + } +} + +func normalizeExclusionList(in []string) []string { + if len(in) == 0 { + return nil + } + seen := make(map[string]struct{}, len(in)) + out := make([]string, 0, len(in)) + for _, v := range in { + v = strings.TrimSpace(v) + if v == "" { + continue + } + if _, dup := seen[v]; dup { + continue + } + seen[v] = struct{}{} + out = append(out, v) + } + if len(out) == 0 { + return nil + } + sort.Strings(out) + return out +} + +// Empty reports whether this clause excludes nothing, the common case. +func (e WriteExclusion) Empty() bool { + return len(e.FieldManagers) == 0 && len(e.Users) == 0 +} + +// Key is a stable fingerprint used to fold rules with identical exclusions and to detect +// when a rule change must restart a watch. +func (e WriteExclusion) Key() string { + if e.Empty() { + return "" + } + return strings.Join(e.FieldManagers, ",") + "|" + strings.Join(e.Users, ",") +} + +// excludesUser reports whether this clause names the attributed identity. An unresolved +// author (empty username) never matches: see admits. +func (e WriteExclusion) excludesUser(username string) bool { + if username == "" { + return false + } + return containsString(e.Users, username) +} + +// excludesLastWriters reports whether this clause names the writer of a change. +// +// lastWriters is the set of field managers that share the newest managedFields timestamp. +// The clause excludes the change only when EVERY one of them is listed: a tie means we +// cannot tell which manager produced this particular write, and mirroring a machine's +// write is a smaller harm than dropping a human's. An object with no managedFields (an +// empty set) is never excluded, for the same reason. +func (e WriteExclusion) excludesLastWriters(lastWriters []string) bool { + if len(e.FieldManagers) == 0 || len(lastWriters) == 0 { + return false + } + for _, writer := range lastWriters { + if !containsString(e.FieldManagers, writer) { + return false + } + } + return true +} + +func containsString(haystack []string, needle string) bool { + for _, v := range haystack { + if v == needle { + return true + } + } + return false +} + +// RuleSelection is one compiled rule's admission clause for a watched type in one +// namespace: the operations it selects, and the writers it declines. +type RuleSelection struct { + Ops OperationSet + Exclusion WriteExclusion +} + +// RuleSelections are every rule clause that selects one (GitTarget, type, namespace). +// Rules are a logical OR, so an event is admitted when ANY clause admits it. +type RuleSelections []RuleSelection + +// NeedsAuthor reports whether resolving the event's author is required to decide +// admission. Attribution costs a bounded wait (the grace window), so it is only paid when +// some clause actually declares excludeUsers. +func (s RuleSelections) NeedsAuthor() bool { + for _, sel := range s { + if len(sel.Exclusion.Users) > 0 { + return true + } + } + return false +} + +// HasExclusions reports whether any clause declines any writer. +func (s RuleSelections) HasExclusions() bool { + for _, sel := range s { + if !sel.Exclusion.Empty() { + return true + } + } + return false +} + +// Ops is the union of every clause's operations — what the watch must actually stream. A +// coarse op filter runs against this before admission is evaluated. +func (s RuleSelections) Ops() OperationSet { + union := OperationSet{} + for _, sel := range s { + for op := range sel.Ops { + union[op] = struct{}{} + } + } + return union +} + +// Key fingerprints the whole selection set so a rule change that alters operations or +// exclusions restarts the affected watch, and one that alters neither does not. It reads +// as "CREATE+UPDATE" for an unrestricted clause and "UPDATE/flux|;*/" when clauses differ. +func (s RuleSelections) Key() string { + parts := make([]string, 0, len(s)) + for _, sel := range s { + part := strings.Join(sel.Ops.Sorted(), "+") + if !sel.Exclusion.Empty() { + part += "/" + sel.Exclusion.Key() + } + parts = append(parts, part) + } + sort.Strings(parts) + return strings.Join(parts, ";") +} + +// Admits decides whether a live change is mirrored, implementing the OR over rules: +// +// route ⟺ ∃ rule : rule selects op ∧ writer ∉ rule.excludeFieldManagers ∧ user ∉ rule.excludeUsers +// +// An exclusion is a veto within its own rule, never a global filter, so an unrestricted +// rule for the same type re-admits everything a restricted one excluded. +// +// lastWriters is empty for a DELETE: managedFields names who last wrote the object, not +// who deleted it, so a field-manager exclusion must not suppress deletes. username is +// empty when the author was not resolved, which makes every user clause fail open. +func (s RuleSelections) Admits(op string, lastWriters []string, username string) bool { + for _, sel := range s { + if !sel.Ops.Match(op) { + continue + } + if sel.Exclusion.excludesLastWriters(lastWriters) { + continue + } + if sel.Exclusion.excludesUser(username) { + continue + } + return true + } + return false +} + +// ExclusionReason names why an event was dropped, for the excluded-events metric. It +// re-walks the clauses only on the drop path, which is off the hot path by construction. +func (s RuleSelections) ExclusionReason(op string, lastWriters []string, username string) string { + byFieldManager := false + for _, sel := range s { + if !sel.Ops.Match(op) { + continue + } + if sel.Exclusion.excludesLastWriters(lastWriters) { + byFieldManager = true + continue + } + if sel.Exclusion.excludesUser(username) { + return exclusionReasonUser + } + } + if byFieldManager { + return exclusionReasonFieldManager + } + return exclusionReasonUnknown +} + +const ( + exclusionReasonFieldManager = "field_manager" + exclusionReasonUser = "user" + exclusionReasonUnknown = "unknown" +) + +// warnIfExcludeUsersWithoutAttribution logs, once per rule, that an excludeUsers clause +// can never match because no author is ever resolved. The clause fails open, so the +// symptom is a rule that silently does nothing — which is worth one loud line. Rule +// resolution runs on every reconcile, so the warning is edge-triggered on first sight. +func (m *Manager) warnIfExcludeUsersWithoutAttribution(ruleRef string, exclusion WriteExclusion) { + if len(exclusion.Users) == 0 || m.AuthorResolver != nil { + return + } + if _, warned := m.excludeUsersWarned.LoadOrStore(ruleRef, struct{}{}); warned { + return + } + m.Log.Info("WARNING: excludeUsers has no effect without author attribution; "+ + "every write will be mirrored. Enable --author-attribution, or use excludeFieldManagers, "+ + "which reads the object's managedFields and needs no audit fact.", + "rule", ruleRef, "excludeUsers", exclusion.Users) +} + +// recordExcludedWatchEvent counts one live event a rule declined to mirror, so an operator +// can see the forward leg's applies being dropped rather than committed. No-op until the +// counter is registered. +func (m *Manager) recordExcludedWatchEvent( + gitDest types.ResourceReference, + gvr schema.GroupVersionResource, + reason string, +) { + if telemetry.WatchEventsExcludedTotal == nil { + return + } + telemetry.WatchEventsExcludedTotal.Add(context.Background(), 1, metric.WithAttributes( + attribute.String("gittarget_namespace", gitDest.Namespace), + attribute.String("gittarget_name", gitDest.Name), + attribute.String("group", gvr.Group), + attribute.String("resource", gvr.Resource), + attribute.String("reason", reason), + )) +} + +// lastWriteFieldManagers returns the field managers of the managedFields entries carrying +// the newest timestamp — the managers that could have produced this write. +// +// Entries with no timestamp are ignored: they cannot be compared, and an entry the API +// server wrote without one is not evidence of recency. Entries for a subresource are +// counted like any other, because a status write is still a write by that manager; the +// content dedup upstream already drops status-only churn that carries no Git change. +// +// The result is sorted and de-duplicated. It is empty when the object has no usable +// managedFields, which makes every field-manager exclusion fail open. +func lastWriteFieldManagers(u *unstructured.Unstructured) []string { + if u == nil { + return nil + } + entries := u.GetManagedFields() + newest := newestManagedFieldsTime(entries) + if newest == nil { + return nil + } + return managersAt(entries, newest) +} + +// newestManagedFieldsTime returns the newest usable timestamp across the entries, or nil +// when none carries one. +func newestManagedFieldsTime(entries []metav1.ManagedFieldsEntry) *metav1.Time { + var newest *metav1.Time + for i := range entries { + ts := entries[i].Time + if ts == nil || ts.IsZero() { + continue + } + if newest == nil || ts.After(newest.Time) { + newest = ts + } + } + return newest +} + +// managersAt returns the sorted, de-duplicated managers of every entry stamped at exactly +// the given time. +func managersAt(entries []metav1.ManagedFieldsEntry, at *metav1.Time) []string { + seen := map[string]struct{}{} + var writers []string + for i := range entries { + ts := entries[i].Time + if ts == nil || !ts.Equal(at) { + continue + } + manager := strings.TrimSpace(entries[i].Manager) + if manager == "" { + continue + } + if _, dup := seen[manager]; dup { + continue + } + seen[manager] = struct{}{} + writers = append(writers, manager) + } + sort.Strings(writers) + return writers +} + +// lastWritersForOperation resolves the field managers a field-manager exclusion may be +// evaluated against. A removal returns none: managedFields names the last writer, and a +// human deleting a Flux-managed object would otherwise be silently ignored — the exact +// failure a label selector has, and the reason this filter reads the write rather than +// the object. +func lastWritersForOperation(op string, u *unstructured.Unstructured) []string { + if op == string(configv1alpha3.OperationDelete) { + return nil + } + return lastWriteFieldManagers(u) +} diff --git a/internal/watch/write_exclusion_routing_test.go b/internal/watch/write_exclusion_routing_test.go new file mode 100644 index 00000000..a18d705d --- /dev/null +++ b/internal/watch/write_exclusion_routing_test.go @@ -0,0 +1,222 @@ +// SPDX-License-Identifier: Apache-2.0 + +package watch + +import ( + "context" + "testing" + "time" + + "github.com/go-logr/logr" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + k8stypes "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/watch" + + "github.com/ConfigButler/gitops-reverser/internal/git" + "github.com/ConfigButler/gitops-reverser/internal/reconcile" + "github.com/ConfigButler/gitops-reverser/internal/types" +) + +// stubAuthorResolver names every event's author without touching Redis, and counts how +// often it was consulted so the tests can pin that attribution is not paid twice. +type stubAuthorResolver struct { + username string + found bool + calls int +} + +func (s *stubAuthorResolver) ResolveAuthor( + context.Context, schema.GroupVersionResource, k8stypes.UID, string, bool, +) (git.UserInfo, bool) { + s.calls++ + if !s.found { + return git.UserInfo{}, false + } + return git.UserInfo{Username: s.username}, true +} + +type exclusionHarness struct { + manager *Manager + enqueuer *recordingEnqueuer + gitDest types.ResourceReference + key targetWatchKey +} + +func newExclusionHarness(t *testing.T, resolver AuthorResolver) *exclusionHarness { + t.Helper() + gitDest := types.NewResourceReference("target", "default") + enqueuer := &recordingEnqueuer{} + stream := reconcile.NewGitTargetEventStream(gitDest.Name, gitDest.Namespace, enqueuer, logr.Discard()) + router := &EventRouter{ + Log: logr.Discard(), + gitTargetStreams: map[string]*reconcile.GitTargetEventStream{gitDest.Key(): stream}, + } + return &exclusionHarness{ + manager: &Manager{EventRouter: router, AuthorResolver: resolver}, + enqueuer: enqueuer, + gitDest: gitDest, + key: targetWatchKey{GVR: configmapsGVR, Namespace: "apps"}, + } +} + +func (h *exclusionHarness) route(t *testing.T, filter watchFilter, ev watch.Event) { + t.Helper() + _, err := h.manager.routeLiveTargetWatchEvent(context.Background(), logr.Discard(), h.gitDest, h.key, filter, ev) + require.NoError(t, err) +} + +// configMapWrittenBy returns a live ConfigMap whose newest managedFields entry names +// manager — the object a watch delivers after that manager applied it. +func configMapWrittenBy(rv, manager string) *unstructured.Unstructured { + obj := configMapObject(rv) + now := metav1.NewTime(time.Now()) + older := metav1.NewTime(now.Add(-time.Hour)) + obj.SetManagedFields([]metav1.ManagedFieldsEntry{ + {Manager: "kubectl", Operation: metav1.ManagedFieldsOperationUpdate, Time: &older}, + {Manager: manager, Operation: metav1.ManagedFieldsOperationApply, Time: &now}, + }) + return obj +} + +// configMapWrittenByWithData is configMapWrittenBy with a distinct data value, so the event +// carries a real Git-content change rather than one the content dedup would drop anyway. +func configMapWrittenByWithData(t *testing.T, rv, manager, flavor string) *unstructured.Unstructured { + t.Helper() + obj := configMapWrittenBy(rv, manager) + require.NoError(t, unstructured.SetNestedField(obj.Object, flavor, "data", "flavor")) + return obj +} + +// The whole point: a GitOps forward leg applies this branch back into the cluster, and +// that apply must not be committed as a new change. +func TestRouteLiveTargetWatchEvent_ExcludesForwardLegApply(t *testing.T) { + h := newExclusionHarness(t, nil) + filter := exclusionFilter(allOps(), []string{"kustomize-controller"}, nil) + + h.route(t, filter, watch.Event{Type: watch.Modified, Object: configMapWrittenBy("12", "kustomize-controller")}) + assert.Empty(t, h.enqueuer.events, "the forward leg's own apply must not reach the branch worker") + + h.route(t, filter, watch.Event{Type: watch.Modified, Object: configMapWrittenBy("13", "kubectl-edit")}) + require.Len(t, h.enqueuer.events, 1, "a human's later edit of the same object is still mirrored") + assert.Equal(t, "UPDATE", h.enqueuer.events[0].Operation) +} + +// A field-manager exclusion must never suppress a delete: managedFields still names the +// forward leg as the last writer, but the actor who deleted the object may be a human. +func TestRouteLiveTargetWatchEvent_FieldManagerExclusionNeverSuppressesDelete(t *testing.T) { + h := newExclusionHarness(t, nil) + filter := exclusionFilter(allOps(), []string{"kustomize-controller"}, nil) + + h.route(t, filter, watch.Event{Type: watch.Deleted, Object: configMapWrittenBy("14", "kustomize-controller")}) + + require.Len(t, h.enqueuer.events, 1) + assert.Equal(t, "DELETE", h.enqueuer.events[0].Operation) +} + +func TestRouteLiveTargetWatchEvent_ExcludesAttributedUser(t *testing.T) { + flux := "system:serviceaccount:flux-system:kustomize-controller" + resolver := &stubAuthorResolver{username: flux, found: true} + h := newExclusionHarness(t, resolver) + filter := exclusionFilter(allOps(), nil, []string{flux}) + + h.route(t, filter, watch.Event{Type: watch.Modified, Object: configMapObject("12")}) + assert.Empty(t, h.enqueuer.events) + + // A delete by the same identity is excluded too — the audit fact names the deleter. + h.route(t, filter, watch.Event{Type: watch.Deleted, Object: configMapObject("13")}) + assert.Empty(t, h.enqueuer.events) +} + +// Dropping a change because we failed to identify its author would silently lose a +// human's edit. An unresolved author is mirrored. +func TestRouteLiveTargetWatchEvent_UnresolvedAuthorFailsOpen(t *testing.T) { + resolver := &stubAuthorResolver{found: false} + h := newExclusionHarness(t, resolver) + filter := exclusionFilter(allOps(), nil, []string{"system:serviceaccount:flux-system:kustomize-controller"}) + + h.route(t, filter, watch.Event{Type: watch.Modified, Object: configMapObject("12")}) + + require.Len(t, h.enqueuer.events, 1) + assert.Empty(t, h.enqueuer.events[0].UserInfo.Username) +} + +// Attribution costs a bounded grace-window wait. It is resolved once when excludeUsers +// forces it early, and the result is reused for the commit rather than looked up again. +func TestRouteLiveTargetWatchEvent_AuthorResolvedOnceWhenExcludeUsersIsSet(t *testing.T) { + resolver := &stubAuthorResolver{username: "jane@acme.com", found: true} + h := newExclusionHarness(t, resolver) + filter := exclusionFilter(allOps(), nil, []string{"someone-else"}) + + h.route(t, filter, watch.Event{Type: watch.Added, Object: configMapObject("12")}) + + require.Len(t, h.enqueuer.events, 1) + assert.Equal(t, "jane@acme.com", h.enqueuer.events[0].UserInfo.Username) + assert.Equal(t, 1, resolver.calls, "the early lookup must be reused, not repeated") +} + +// A field-manager exclusion reads watch state alone, so it must not consult the +// attribution index at all — that is the property that makes it race-free and usable in +// configured-author mode. +func TestRouteLiveTargetWatchEvent_FieldManagerExclusionDoesNotResolveAuthorEarly(t *testing.T) { + resolver := &stubAuthorResolver{username: "jane@acme.com", found: true} + h := newExclusionHarness(t, resolver) + filter := exclusionFilter(allOps(), []string{"kustomize-controller"}, nil) + + h.route(t, filter, watch.Event{Type: watch.Modified, Object: configMapWrittenBy("12", "kustomize-controller")}) + + assert.Empty(t, h.enqueuer.events) + assert.Zero(t, resolver.calls, "an excluded write must not pay the attribution grace window") +} + +// An excluded write must not seed the content-dedup cache: if it did, a human later +// writing that exact content would be deduped against a change that never reached Git. +func TestRouteLiveTargetWatchEvent_ExcludedWriteDoesNotSeedDedupCache(t *testing.T) { + h := newExclusionHarness(t, nil) + filter := exclusionFilter(allOps(), []string{"kustomize-controller"}, nil) + + // A human's write seeds the cache with "vanilla". + h.route( + t, + filter, + watch.Event{Type: watch.Added, Object: configMapWrittenByWithData(t, "11", "kubectl", "vanilla")}, + ) + require.Len(t, h.enqueuer.events, 1) + + // The forward leg writes "chocolate". Excluded, so it must not become the cached hash. + h.route(t, filter, watch.Event{ + Type: watch.Modified, Object: configMapWrittenByWithData(t, "12", "kustomize-controller", "chocolate"), + }) + require.Len(t, h.enqueuer.events, 1, "the forward leg's write is not mirrored") + + // A human now writes the very content the forward leg wrote. If the excluded write had + // seeded the cache, this would be deduped away and the human's edit lost. + h.route(t, filter, watch.Event{ + Type: watch.Modified, Object: configMapWrittenByWithData(t, "13", "kubectl-edit", "chocolate"), + }) + require.Len(t, h.enqueuer.events, 2, + "the human's write must reach Git; it was never deduped against the excluded one") +} + +// A GitOps tool's own labels and annotations are stripped from Git content by +// internal/sanitize, so an apply that only stamps them produces no Git-writable change and +// the content dedup drops it before any exclusion is consulted. Worth pinning: it is why the +// e2e drives the forward leg with a data change rather than a label, and why an operator who +// sees no excluded-event metric should look for a content change, not a label. +func TestRouteLiveTargetWatchEvent_OperationalLabelsAreNotGitContent(t *testing.T) { + h := newExclusionHarness(t, nil) + filter := opsFilter(allOps()) + + h.route(t, filter, watch.Event{Type: watch.Added, Object: configMapWrittenBy("11", "kubectl")}) + require.Len(t, h.enqueuer.events, 1) + + labelled := configMapWrittenBy("12", "kustomize-controller") + labelled.SetLabels(map[string]string{"kustomize.toolkit.fluxcd.io/name": "demo"}) + h.route(t, filter, watch.Event{Type: watch.Modified, Object: labelled}) + + assert.Len(t, h.enqueuer.events, 1, + "a Flux label is not Git content, so the update carries no change to mirror") +} diff --git a/internal/watch/write_exclusion_test.go b/internal/watch/write_exclusion_test.go new file mode 100644 index 00000000..af51bbea --- /dev/null +++ b/internal/watch/write_exclusion_test.go @@ -0,0 +1,243 @@ +// SPDX-License-Identifier: Apache-2.0 + +package watch + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" +) + +func managedFieldsAt(entries ...metav1.ManagedFieldsEntry) *unstructured.Unstructured { + u := &unstructured.Unstructured{} + u.SetAPIVersion("v1") + u.SetKind("ConfigMap") + u.SetName("settings") + u.SetNamespace("apps") + u.SetManagedFields(entries) + return u +} + +func entry(manager string, at time.Time) metav1.ManagedFieldsEntry { + ts := metav1.NewTime(at) + return metav1.ManagedFieldsEntry{Manager: manager, Operation: metav1.ManagedFieldsOperationApply, Time: &ts} +} + +func TestLastWriteFieldManagers(t *testing.T) { + t.Parallel() + + base := time.Date(2026, 7, 9, 12, 0, 0, 0, time.UTC) + + tests := map[string]struct { + obj *unstructured.Unstructured + want []string + }{ + "newest entry wins": { + obj: managedFieldsAt( + entry("kubectl", base), + entry("kustomize-controller", base.Add(time.Minute)), + ), + want: []string{"kustomize-controller"}, + }, + "older machine write does not mask a newer human write": { + obj: managedFieldsAt( + entry("kustomize-controller", base), + entry("kubectl-edit", base.Add(time.Minute)), + ), + want: []string{"kubectl-edit"}, + }, + "a timestamp tie reports both": { + obj: managedFieldsAt( + entry("kustomize-controller", base), + entry("kubectl", base), + ), + want: []string{"kubectl", "kustomize-controller"}, + }, + "entries without a timestamp are ignored": { + obj: managedFieldsAt( + metav1.ManagedFieldsEntry{Manager: "no-clock"}, + entry("kustomize-controller", base), + ), + want: []string{"kustomize-controller"}, + }, + "all entries without a timestamp yield nothing": { + obj: managedFieldsAt(metav1.ManagedFieldsEntry{Manager: "no-clock"}), + want: nil, + }, + "no managedFields at all": { + obj: managedFieldsAt(), + want: nil, + }, + "nil object": {obj: nil, want: nil}, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + t.Parallel() + assert.Equal(t, tc.want, lastWriteFieldManagers(tc.obj)) + }) + } +} + +// managedFields names who last WROTE an object, never who deleted it. Evaluating a field +// manager exclusion on a DELETE would silently ignore a human deleting a Flux-managed +// resource — the exact failure a label selector has. +func TestLastWritersForOperation_DeleteHasNoFieldManager(t *testing.T) { + t.Parallel() + + obj := managedFieldsAt(entry("kustomize-controller", time.Now())) + assert.Equal(t, []string{"kustomize-controller"}, lastWritersForOperation("UPDATE", obj)) + assert.Equal(t, []string{"kustomize-controller"}, lastWritersForOperation("CREATE", obj)) + assert.Nil(t, lastWritersForOperation("DELETE", obj)) +} + +func allOps() OperationSet { return OperationSet{"*": struct{}{}} } + +func TestRuleSelections_Admits_FieldManagerVeto(t *testing.T) { + t.Parallel() + + excludingFlux := RuleSelections{{ + Ops: allOps(), + Exclusion: newWriteExclusion([]string{"kustomize-controller"}, nil), + }} + + assert.False(t, excludingFlux.Admits("UPDATE", []string{"kustomize-controller"}, ""), + "the forward leg's own apply must not be mirrored back") + assert.True(t, excludingFlux.Admits("UPDATE", []string{"kubectl"}, ""), + "a human's write is always mirrored") + assert.True(t, excludingFlux.Admits("UPDATE", nil, ""), + "an object with no usable managedFields fails open") +} + +// A tie means we cannot tell which manager produced this write. Mirroring a machine's +// write is a smaller harm than dropping a human's, so a tie only excludes when every tied +// manager is excluded. +func TestRuleSelections_Admits_TimestampTieFailsOpen(t *testing.T) { + t.Parallel() + + sel := RuleSelections{{Ops: allOps(), Exclusion: newWriteExclusion([]string{"kustomize-controller"}, nil)}} + assert.True(t, sel.Admits("UPDATE", []string{"kubectl", "kustomize-controller"}, "")) + + both := RuleSelections{{ + Ops: allOps(), + Exclusion: newWriteExclusion([]string{"kustomize-controller", "helm-controller"}, nil), + }} + assert.False(t, both.Admits("UPDATE", []string{"helm-controller", "kustomize-controller"}, "")) +} + +func TestRuleSelections_Admits_UserVeto(t *testing.T) { + t.Parallel() + + flux := "system:serviceaccount:flux-system:kustomize-controller" + sel := RuleSelections{{Ops: allOps(), Exclusion: newWriteExclusion(nil, []string{flux})}} + + assert.False(t, sel.Admits("DELETE", nil, flux), "excludeUsers does apply to deletes") + assert.True(t, sel.Admits("UPDATE", nil, "jane@acme.com")) + assert.True(t, sel.Admits("UPDATE", nil, ""), + "an unresolved author fails open: losing a human's edit is worse than mirroring a machine's") +} + +// Rules are a logical OR. An exclusion vetoes only inside the rule that declares it, so an +// unrestricted rule for the same type re-admits everything a restricted one excluded. +func TestRuleSelections_Admits_ExclusionIsPerRuleNotGlobal(t *testing.T) { + t.Parallel() + + restricted := RuleSelection{Ops: allOps(), Exclusion: newWriteExclusion([]string{"kustomize-controller"}, nil)} + unrestricted := RuleSelection{Ops: allOps()} + + assert.False(t, RuleSelections{restricted}.Admits("UPDATE", []string{"kustomize-controller"}, "")) + assert.True(t, RuleSelections{restricted, unrestricted}.Admits("UPDATE", []string{"kustomize-controller"}, ""), + "a second, unrestricted rule for the same type admits the write") +} + +// An exclusion only vetoes writes its own rule selects. A rule that excludes Flux but +// watches only CREATE must not suppress Flux's UPDATE when another rule watches UPDATE. +func TestRuleSelections_Admits_RespectsPerRuleOperations(t *testing.T) { + t.Parallel() + + createOnlyExcludingFlux := RuleSelection{ + Ops: OperationSet{"CREATE": struct{}{}}, + Exclusion: newWriteExclusion([]string{"kustomize-controller"}, nil), + } + updateOnlyPlain := RuleSelection{Ops: OperationSet{"UPDATE": struct{}{}}} + selections := RuleSelections{createOnlyExcludingFlux, updateOnlyPlain} + + assert.False(t, selections.Admits("CREATE", []string{"kustomize-controller"}, "")) + assert.True(t, selections.Admits("UPDATE", []string{"kustomize-controller"}, "")) +} + +func TestRuleSelections_Admits_NoRuleSelectsTheOperation(t *testing.T) { + t.Parallel() + sel := RuleSelections{{Ops: OperationSet{"CREATE": struct{}{}}}} + assert.False(t, sel.Admits("DELETE", nil, "")) +} + +func TestRuleSelections_NeedsAuthor(t *testing.T) { + t.Parallel() + + assert.False(t, RuleSelections{{Ops: allOps()}}.NeedsAuthor()) + assert.False(t, RuleSelections{{ + Ops: allOps(), + Exclusion: newWriteExclusion([]string{"kustomize-controller"}, nil), + }}.NeedsAuthor(), "a field-manager exclusion never needs the attribution grace window") + assert.True(t, RuleSelections{{ + Ops: allOps(), + Exclusion: newWriteExclusion(nil, []string{"flux"}), + }}.NeedsAuthor()) +} + +func TestRuleSelections_HasExclusionsAndOpsUnion(t *testing.T) { + t.Parallel() + + plain := RuleSelections{{Ops: OperationSet{"CREATE": struct{}{}}}} + assert.False(t, plain.HasExclusions()) + + mixed := RuleSelections{ + {Ops: OperationSet{"CREATE": struct{}{}}}, + {Ops: OperationSet{"UPDATE": struct{}{}}, Exclusion: newWriteExclusion([]string{"flux"}, nil)}, + } + assert.True(t, mixed.HasExclusions()) + assert.Equal(t, []string{"CREATE", "UPDATE"}, mixed.Ops().Sorted(), + "the stream must carry every operation any clause selects") +} + +func TestRuleSelections_ExclusionReason(t *testing.T) { + t.Parallel() + + byManager := RuleSelections{{Ops: allOps(), Exclusion: newWriteExclusion([]string{"flux"}, nil)}} + assert.Equal(t, exclusionReasonFieldManager, byManager.ExclusionReason("UPDATE", []string{"flux"}, "")) + + byUser := RuleSelections{{Ops: allOps(), Exclusion: newWriteExclusion(nil, []string{"sa:flux"})}} + assert.Equal(t, exclusionReasonUser, byUser.ExclusionReason("DELETE", nil, "sa:flux")) +} + +func TestNewWriteExclusion_NormalizesAndDeduplicates(t *testing.T) { + t.Parallel() + + e := newWriteExclusion([]string{" b ", "a", "b", ""}, []string{"z", "z"}) + require.Equal(t, []string{"a", "b"}, e.FieldManagers) + require.Equal(t, []string{"z"}, e.Users) + assert.False(t, e.Empty()) + + assert.True(t, newWriteExclusion(nil, nil).Empty()) + assert.True(t, newWriteExclusion([]string{"", " "}, nil).Empty(), + "a list of blanks excludes nobody") + assert.Empty(t, newWriteExclusion(nil, nil).Key()) +} + +// Two rules declaring the same exclusions in a different order must fold into one clause, +// or the watched-type table would grow a clause per reconcile ordering. +func TestWriteExclusion_KeyIsOrderIndependent(t *testing.T) { + t.Parallel() + + a := newWriteExclusion([]string{"x", "y"}, []string{"1", "2"}) + b := newWriteExclusion([]string{"y", "x"}, []string{"2", "1"}) + assert.Equal(t, a.Key(), b.Key()) + + c := newWriteExclusion([]string{"x"}, []string{"1", "2"}) + assert.NotEqual(t, a.Key(), c.Key()) +} diff --git a/internal/watch/write_exclusion_test_helpers_test.go b/internal/watch/write_exclusion_test_helpers_test.go new file mode 100644 index 00000000..b670fceb --- /dev/null +++ b/internal/watch/write_exclusion_test_helpers_test.go @@ -0,0 +1,26 @@ +// SPDX-License-Identifier: Apache-2.0 + +package watch + +// nsSelections lifts the pre-exclusion "namespace -> operations" shape most tests care +// about into the per-rule clause shape the table now carries: one unrestricted clause per +// namespace, excluding nobody. +func nsSelections(ops map[string]OperationSet) map[string]RuleSelections { + out := make(map[string]RuleSelections, len(ops)) + for ns, set := range ops { + out[ns] = RuleSelections{{Ops: set}} + } + return out +} + +// opsFilter builds the watch admission state for a rule that excludes nobody. +func opsFilter(ops OperationSet) watchFilter { + return watchFilter{ops: ops, selections: RuleSelections{{Ops: ops}}} +} + +// exclusionFilter builds the watch admission state for a single rule that declines the +// given field managers and users. +func exclusionFilter(ops OperationSet, fieldManagers, users []string) watchFilter { + selections := RuleSelections{{Ops: ops, Exclusion: newWriteExclusion(fieldManagers, users)}} + return watchFilter{ops: selections.Ops(), selections: selections} +} diff --git a/internal/webhook/author_assertion.go b/internal/webhook/author_assertion.go new file mode 100644 index 00000000..5bd34941 --- /dev/null +++ b/internal/webhook/author_assertion.go @@ -0,0 +1,155 @@ +// SPDX-License-Identifier: Apache-2.0 + +package webhook + +import ( + "context" + "encoding/json" + "fmt" + + authnv1 "k8s.io/api/authentication/v1" + authzv1 "k8s.io/api/authorization/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + authzv1client "k8s.io/client-go/kubernetes/typed/authorization/v1" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" +) + +const ( + // AssertAuthorVerb is the RBAC verb a requester must hold on a GitTarget to set + // CommitRequest.spec.author. It follows the precedent of `bind`, `escalate` and + // `impersonate`: a verb on a resource, granted deliberately, that authorizes saying + // something the API cannot otherwise verify. + AssertAuthorVerb = "assert-author" + // gitTargetsResource is the resource the verb is checked against. + gitTargetsResource = "gittargets" + // configbutlerGroup is the API group of that resource. + configbutlerGroup = "configbutler.ai" +) + +// AuthorAssertionAuthorizer answers "may this requester assert a commit author on this +// GitTarget?". The real implementation issues a SubjectAccessReview; the interface keeps +// the admission handler unit-testable without an API server. +type AuthorAssertionAuthorizer interface { + // CanAssertAuthor reports whether user holds AssertAuthorVerb on the named GitTarget. + // The reason is the authorizer's own explanation, surfaced in the denial message. + CanAssertAuthor( + ctx context.Context, + user authnv1.UserInfo, + namespace, gitTargetName string, + ) (allowed bool, reason string, err error) +} + +// subjectAccessReviewAuthorizer implements AuthorAssertionAuthorizer against the +// apiserver's authorization API, delegating the decision to whatever authorizers the +// cluster has configured (RBAC, webhook, node). It creates SubjectAccessReviews, so the +// operator's ServiceAccount needs `create` on `subjectaccessreviews.authorization.k8s.io`. +type subjectAccessReviewAuthorizer struct { + client authzv1client.SubjectAccessReviewInterface +} + +// NewSubjectAccessReviewAuthorizer builds the production authorizer. +func NewSubjectAccessReviewAuthorizer(client authzv1client.SubjectAccessReviewInterface) AuthorAssertionAuthorizer { + return &subjectAccessReviewAuthorizer{client: client} +} + +func (a *subjectAccessReviewAuthorizer) CanAssertAuthor( + ctx context.Context, + user authnv1.UserInfo, + namespace, gitTargetName string, +) (bool, string, error) { + // resourceNames scopes the grant to one GitTarget, so a tenant granted assert-author + // on their own target cannot author commits into someone else's repository. + review := &authzv1.SubjectAccessReview{ + Spec: authzv1.SubjectAccessReviewSpec{ + User: user.Username, + UID: user.UID, + Groups: user.Groups, + Extra: extraToSubjectAccessReviewExtra(user.Extra), + ResourceAttributes: &authzv1.ResourceAttributes{ + Namespace: namespace, + Verb: AssertAuthorVerb, + Group: configbutlerGroup, + Resource: gitTargetsResource, + Name: gitTargetName, + }, + }, + } + result, err := a.client.Create(ctx, review, metav1.CreateOptions{}) + if err != nil { + return false, "", fmt.Errorf("create SubjectAccessReview: %w", err) + } + if result.Status.EvaluationError != "" && !result.Status.Allowed { + return false, result.Status.EvaluationError, nil + } + return result.Status.Allowed, result.Status.Reason, nil +} + +// extraToSubjectAccessReviewExtra converts admission's user.extra to the authorization +// API's near-identical type, so a webhook authorizer keyed on an OIDC claim sees it. +func extraToSubjectAccessReviewExtra(extra map[string]authnv1.ExtraValue) map[string]authzv1.ExtraValue { + if len(extra) == 0 { + return nil + } + out := make(map[string]authzv1.ExtraValue, len(extra)) + for key, values := range extra { + out[key] = authzv1.ExtraValue(values) + } + return out +} + +// assertedAuthor is the subset of a CommitRequest an admission request needs to decide +// authorization: the asserted identity and the GitTarget it is asserted against. Like +// commandObjectUID it reads only the fields it needs, so it never depends on the command +// kind being registered in a decoder scheme. +type assertedAuthor struct { + Name string + Email string + GitTargetName string +} + +// parseAssertedAuthor extracts spec.author and spec.targetRef.name from a raw +// CommitRequest. ok=false means the object asserts no author, which is the common case +// and requires no authorization at all. +func parseAssertedAuthor(raw []byte) (assertedAuthor, bool) { + if len(raw) == 0 { + return assertedAuthor{}, false + } + var probe struct { + Spec struct { + Author *struct { + Name string `json:"name"` + Email string `json:"email"` + } `json:"author"` + TargetRef struct { + Name string `json:"name"` + } `json:"targetRef"` + } `json:"spec"` + } + if err := json.Unmarshal(raw, &probe); err != nil { + return assertedAuthor{}, false + } + if probe.Spec.Author == nil || probe.Spec.Author.Name == "" { + return assertedAuthor{}, false + } + return assertedAuthor{ + Name: probe.Spec.Author.Name, + Email: probe.Spec.Author.Email, + GitTargetName: probe.Spec.TargetRef.Name, + }, true +} + +// denyAuthorAssertion builds the admission denial for an unauthorized spec.author. The +// message names the exact RBAC rule that would grant it, because "forbidden" without the +// remedy is the least useful thing an admission webhook can say. +func denyAuthorAssertion(user, namespace, gitTargetName, reason string) admission.Response { + msg := fmt.Sprintf( + "user %q may not set spec.author on a CommitRequest for GitTarget %q: "+ + "asserting a commit author requires the %q verb on gittargets.%s. Grant it with a Role in "+ + "namespace %q: {apiGroups: [%q], resources: [%q], resourceNames: [%q], verbs: [%q]}", + user, gitTargetName, AssertAuthorVerb, configbutlerGroup, namespace, + configbutlerGroup, gitTargetsResource, gitTargetName, AssertAuthorVerb) + if reason != "" { + msg += " (authorizer: " + reason + ")" + } + return admission.Denied(msg) +} diff --git a/internal/webhook/author_assertion_test.go b/internal/webhook/author_assertion_test.go new file mode 100644 index 00000000..bc215656 --- /dev/null +++ b/internal/webhook/author_assertion_test.go @@ -0,0 +1,203 @@ +// SPDX-License-Identifier: Apache-2.0 + +package webhook + +import ( + "context" + "errors" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + authnv1 "k8s.io/api/authentication/v1" +) + +// fakeAuthorizer stands in for the SubjectAccessReview client and records what it was asked. +type fakeAuthorizer struct { + allowed bool + reason string + err error + + calls []struct { + user authnv1.UserInfo + namespace string + gitTargetName string + } +} + +func (f *fakeAuthorizer) CanAssertAuthor( + _ context.Context, user authnv1.UserInfo, namespace, gitTargetName string, +) (bool, string, error) { + f.calls = append(f.calls, struct { + user authnv1.UserInfo + namespace string + gitTargetName string + }{user, namespace, gitTargetName}) + return f.allowed, f.reason, f.err +} + +const commitRequestWithAuthor = `{ + "metadata": {"uid": "cr-uid"}, + "spec": {"targetRef": {"name": "tenants"}, "author": {"name": "Ada Lovelace", "email": "ada@example.com"}} +}` + +const commitRequestWithoutAuthor = `{"metadata":{"uid":"cr-uid"},"spec":{"targetRef":{"name":"tenants"}}}` + +func TestParseAssertedAuthor(t *testing.T) { + t.Parallel() + + tests := map[string]struct { + raw string + wantOK bool + wantName string + wantRef string + }{ + "asserts an author": { + raw: commitRequestWithAuthor, + wantOK: true, + wantName: "Ada Lovelace", + wantRef: "tenants", + }, + "no author": {raw: commitRequestWithoutAuthor, wantOK: false}, + "empty author name": {raw: `{"spec":{"author":{"email":"a@b.co"}}}`, wantOK: false}, + "null author": {raw: `{"spec":{"author":null}}`, wantOK: false}, + "empty body": {raw: "", wantOK: false}, + "malformed json": {raw: "{not json", wantOK: false}, + "author but no name": {raw: `{"spec":{"author":{"name":""}}}`, wantOK: false}, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + t.Parallel() + got, ok := parseAssertedAuthor([]byte(tc.raw)) + require.Equal(t, tc.wantOK, ok) + if !tc.wantOK { + return + } + assert.Equal(t, tc.wantName, got.Name) + assert.Equal(t, tc.wantRef, got.GitTargetName) + }) + } +} + +func TestValidateOperatorTypesHandler_AllowsAuthorizedAssertion(t *testing.T) { + rec := &fakeCommandAuthorRecorder{} + authz := &fakeAuthorizer{allowed: true} + h := &ValidateOperatorTypesHandler{Store: rec, Authorizer: authz} + + user := authnv1.UserInfo{Username: "gitops-api", Groups: []string{"platform"}} + resp := h.Handle(context.Background(), commandReview(commitRequestResource(), commitRequestWithAuthor, user, nil)) + + assert.True(t, resp.Allowed) + require.Len(t, authz.calls, 1) + assert.Equal(t, "gitops-api", authz.calls[0].user.Username) + assert.Equal(t, "team-a", authz.calls[0].namespace, "the review is scoped to the CommitRequest's namespace") + assert.Equal(t, "tenants", authz.calls[0].gitTargetName, "and to the GitTarget it names") + + require.Len(t, rec.calls, 1) + assert.True(t, rec.calls[0].author.AssertAuthorAllowed, + "the verdict must be recorded: the controller honors spec.author only against it") +} + +func TestValidateOperatorTypesHandler_DeniesUnauthorizedAssertion(t *testing.T) { + rec := &fakeCommandAuthorRecorder{} + h := &ValidateOperatorTypesHandler{Store: rec, Authorizer: &fakeAuthorizer{allowed: false, reason: "no RBAC rule"}} + + resp := h.Handle(context.Background(), + commandReview(commitRequestResource(), commitRequestWithAuthor, authnv1.UserInfo{Username: "mallory"}, nil)) + + assert.False(t, resp.Allowed) + require.NotNil(t, resp.Result) + assert.Contains(t, resp.Result.Message, "assert-author") + assert.Contains(t, resp.Result.Message, "tenants", "the denial names the GitTarget") + assert.Contains(t, resp.Result.Message, "no RBAC rule", "and the authorizer's own reason") + assert.Empty(t, rec.calls, "an unauthorized assertion records nothing") +} + +// An unverifiable privilege is not a granted one: without an authorizer the handler cannot +// know whether the requester holds the verb, so it must not let the assertion through. +func TestValidateOperatorTypesHandler_DeniesAssertionWithoutAuthorizer(t *testing.T) { + rec := &fakeCommandAuthorRecorder{} + h := &ValidateOperatorTypesHandler{Store: rec} + + resp := h.Handle(context.Background(), + commandReview(commitRequestResource(), commitRequestWithAuthor, authnv1.UserInfo{Username: "alice"}, nil)) + + assert.False(t, resp.Allowed) + assert.Contains(t, resp.Result.Message, "SubjectAccessReview") + assert.Empty(t, rec.calls) +} + +// An authorizer we could not reach has not said yes. +func TestValidateOperatorTypesHandler_DeniesAssertionWhenReviewErrors(t *testing.T) { + rec := &fakeCommandAuthorRecorder{} + h := &ValidateOperatorTypesHandler{Store: rec, Authorizer: &fakeAuthorizer{err: errors.New("apiserver down")}} + + resp := h.Handle(context.Background(), + commandReview(commitRequestResource(), commitRequestWithAuthor, authnv1.UserInfo{Username: "alice"}, nil)) + + assert.False(t, resp.Allowed) + assert.Contains(t, resp.Result.Message, "apiserver down") + assert.Empty(t, rec.calls) +} + +// A dry-run apply must report the same forbidden error a real one would, so the guard runs +// before the dry-run short-circuit. +func TestValidateOperatorTypesHandler_DeniesUnauthorizedAssertionOnDryRun(t *testing.T) { + rec := &fakeCommandAuthorRecorder{} + h := &ValidateOperatorTypesHandler{Store: rec, Authorizer: &fakeAuthorizer{allowed: false}} + dryRun := true + + resp := h.Handle(context.Background(), + commandReview(commitRequestResource(), commitRequestWithAuthor, authnv1.UserInfo{Username: "mallory"}, &dryRun)) + + assert.False(t, resp.Allowed) + assert.Empty(t, rec.calls, "dry-run still records nothing") +} + +func TestValidateOperatorTypesHandler_AuthorizedDryRunRecordsNothing(t *testing.T) { + rec := &fakeCommandAuthorRecorder{} + h := &ValidateOperatorTypesHandler{Store: rec, Authorizer: &fakeAuthorizer{allowed: true}} + dryRun := true + + resp := h.Handle(context.Background(), + commandReview(commitRequestResource(), commitRequestWithAuthor, authnv1.UserInfo{Username: "alice"}, &dryRun)) + + assert.True(t, resp.Allowed) + assert.Empty(t, rec.calls, "sideEffects: NoneOnDryRun") +} + +// Without Redis the assertion is authorized but never recorded, so the controller ignores +// it. The create is still allowed — a missing store never blocks a command. +func TestValidateOperatorTypesHandler_AuthorizedAssertionWithoutStore(t *testing.T) { + h := &ValidateOperatorTypesHandler{Authorizer: &fakeAuthorizer{allowed: true}} + + resp := h.Handle(context.Background(), + commandReview(commitRequestResource(), commitRequestWithAuthor, authnv1.UserInfo{Username: "alice"}, nil)) + + assert.True(t, resp.Allowed) +} + +// The overwhelmingly common case: no spec.author, so no review is issued at all. +func TestValidateOperatorTypesHandler_NoAssertionSkipsAuthorization(t *testing.T) { + rec := &fakeCommandAuthorRecorder{} + authz := &fakeAuthorizer{allowed: false} + h := &ValidateOperatorTypesHandler{Store: rec, Authorizer: authz} + + resp := h.Handle(context.Background(), + commandReview(commitRequestResource(), commitRequestWithoutAuthor, authnv1.UserInfo{Username: "alice"}, nil)) + + assert.True(t, resp.Allowed) + assert.Empty(t, authz.calls, "a CommitRequest without spec.author needs no privilege") + require.Len(t, rec.calls, 1) + assert.False(t, rec.calls[0].author.AssertAuthorAllowed) +} + +func TestExtraToSubjectAccessReviewExtra(t *testing.T) { + t.Parallel() + + assert.Nil(t, extraToSubjectAccessReviewExtra(nil)) + got := extraToSubjectAccessReviewExtra(map[string]authnv1.ExtraValue{"claim": {"a", "b"}}) + require.Len(t, got, 1) + assert.Equal(t, []string{"a", "b"}, []string(got["claim"])) +} diff --git a/internal/webhook/validate_operator_types_handler.go b/internal/webhook/validate_operator_types_handler.go index e8a13a11..508c9620 100644 --- a/internal/webhook/validate_operator_types_handler.go +++ b/internal/webhook/validate_operator_types_handler.go @@ -7,6 +7,7 @@ import ( "encoding/json" "time" + "github.com/go-logr/logr" authnv1 "k8s.io/api/authentication/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" @@ -55,22 +56,33 @@ type CommandAuthorRecorder interface { RecordCommandAuthor(ctx context.Context, uid types.UID, author queue.CommandAuthor) error } -// ValidateOperatorTypesHandler is the admission handler for our operator CRDs. Today it -// does one thing: for a command kind (a CommitRequest) it captures the authenticated -// submitter into the CommandAuthorStore and always allows — pure observation with a -// single side effect (a Redis upsert), never a rejection, so a user's command never -// depends on it succeeding (a missed capture degrades to a committer-authored commit; -// see docs/design/commitrequest-admission-authorship.md §2, §4). It dispatches on the -// resource (isCommandKind today), so a future config-validation branch for non-command -// kinds slots in alongside without disturbing this one. +// ValidateOperatorTypesHandler is the admission handler for our operator CRDs. For a +// command kind (a CommitRequest) it captures the authenticated submitter into the +// CommandAuthorStore. That capture is pure observation with a single side effect (a Redis +// upsert): a missed capture degrades to a committer-authored commit, so a user's command +// never depends on it succeeding (docs/design/commitrequest-admission-authorship.md §2, §4). +// +// It rejects exactly one thing: a CommitRequest whose spec.author asserts an identity the +// requester is not authorized to assert. That check needs the requester, which only +// admission sees. Its verdict is recorded on the author record rather than trusted from +// the allow itself, because this webhook is failurePolicy: Ignore — a bypassed webhook +// writes no record, and the controller then ignores the assertion rather than honoring an +// unchecked one. See docs/design/multi-tenant/asserted-commit-author.md. +// +// It dispatches on the resource (isCommandKind today), so a future config-validation +// branch for non-command kinds slots in alongside without disturbing this one. type ValidateOperatorTypesHandler struct { Store CommandAuthorRecorder + // Authorizer decides whether a requester may set CommitRequest.spec.author. A nil + // Authorizer denies every assertion: an unverifiable privilege is not a granted one. + Authorizer AuthorAssertionAuthorizer } // Handle records {uid → author} for an admitted command CREATE before the object // persists (the authorship invariant, §2), then allows. Every early return still // allows: a non-command kind, a dry-run, a missing uid, or an unauthenticated request -// simply records nothing and falls back to the committer downstream. +// simply records nothing and falls back to the committer downstream. The one denial is an +// unauthorized spec.author. func (h *ValidateOperatorTypesHandler) Handle(ctx context.Context, req admission.Request) admission.Response { log := logf.FromContext(ctx).WithName("validate-operator-types") @@ -79,6 +91,20 @@ func (h *ValidateOperatorTypesHandler) Handle(ctx context.Context, req admission // Belt-and-suspenders; the webhook rules already scope us to command kinds. return admission.Allowed("not a command kind") } + + // The author assertion is authorized before the dry-run and store short-circuits: a + // dry-run apply must report the same forbidden error a real one would, and an install + // without Redis must not silently accept a privileged field it will then ignore. + asserted, asserts := parseAssertedAuthor(req.Object.Raw) + assertAuthorAllowed := false + if asserts { + allowed, denial := h.authorizeAuthorAssertion(ctx, log, req, asserted) + if !allowed { + return denial + } + assertAuthorAllowed = true + } + // Dry-run never persists, so the controller will never read this record — and we // declare sideEffects: NoneOnDryRun, so we must honor it. if req.DryRun != nil && *req.DryRun { @@ -87,8 +113,14 @@ func (h *ValidateOperatorTypesHandler) Handle(ctx context.Context, req admission // No store means the webhook is running without a Redis backend (admission is on by // default, but command-author capture is Redis-backed). Allow without recording; the - // controller then finalizes as the committer (AuthorAttributed=False). + // controller then finalizes as the committer (AuthorAttributed=False), and — because + // no record exists — ignores any spec.author it just authorized. if h.Store == nil { + if asserts { + log.Info("spec.author authorized but not recorded: no author store configured; "+ + "the CommitRequest will commit as the committer. Set --redis-addr.", + "namespace", req.Namespace, "name", req.Name) + } return admission.Allowed("no author store: not recorded") } @@ -102,10 +134,11 @@ func (h *ValidateOperatorTypesHandler) Handle(ctx context.Context, req admission } author := queue.CommandAuthor{ - Author: req.UserInfo.Username, // effective (impersonated) user — the apiserver resolved it - DisplayName: firstExtraValue(req.UserInfo.Extra, displayNameExtraKey), - Email: firstExtraValue(req.UserInfo.Extra, emailExtraKey), - RequestedAt: time.Now().UTC().Format(time.RFC3339Nano), + Author: req.UserInfo.Username, // effective (impersonated) user — the apiserver resolved it + DisplayName: firstExtraValue(req.UserInfo.Extra, displayNameExtraKey), + Email: firstExtraValue(req.UserInfo.Extra, emailExtraKey), + RequestedAt: time.Now().UTC().Format(time.RFC3339Nano), + AssertAuthorAllowed: assertAuthorAllowed, } if author.Author == "" { return admission.Allowed("no user to attribute") // anonymous never reaches a persisted CREATE @@ -128,6 +161,50 @@ func (h *ValidateOperatorTypesHandler) Handle(ctx context.Context, req admission return admission.Allowed("author recorded") } +// authorizeAuthorAssertion runs the SubjectAccessReview behind CommitRequest.spec.author. +// allowed=false carries the admission denial to return. +// +// Three ways to be denied, all fail-closed: +// - no Authorizer is wired (the operator cannot create SubjectAccessReviews), because an +// unverifiable privilege is not a granted one; +// - the review errors, because an authorizer we could not reach has not said yes; +// - the review says no. +func (h *ValidateOperatorTypesHandler) authorizeAuthorAssertion( + ctx context.Context, + log logr.Logger, + req admission.Request, + asserted assertedAuthor, +) (bool, admission.Response) { + if h.Authorizer == nil { + log.Info("spec.author denied: no author-assertion authorizer configured", + "namespace", req.Namespace, "name", req.Name, "user", req.UserInfo.Username) + return false, denyAuthorAssertion(req.UserInfo.Username, req.Namespace, asserted.GitTargetName, + "the operator cannot create SubjectAccessReviews, so the assertion cannot be authorized") + } + + allowed, reason, err := h.Authorizer.CanAssertAuthor(ctx, req.UserInfo, req.Namespace, asserted.GitTargetName) + if err != nil { + log.Error(err, "author-assertion review failed; denying", + "namespace", req.Namespace, "name", req.Name, "user", req.UserInfo.Username) + return false, denyAuthorAssertion(req.UserInfo.Username, req.Namespace, asserted.GitTargetName, + "the authorization review could not be completed: "+err.Error()) + } + if !allowed { + log.Info("spec.author denied: requester lacks the assert-author verb", + "namespace", req.Namespace, "name", req.Name, + "user", req.UserInfo.Username, "gitTarget", asserted.GitTargetName) + return false, denyAuthorAssertion(req.UserInfo.Username, req.Namespace, asserted.GitTargetName, reason) + } + + // Info, not V(1): asserting an author is a privileged act on the repository's history. + // Every use of it belongs in the audit trail of the operator's own logs. + log.Info("spec.author authorized", + "namespace", req.Namespace, "name", req.Name, + "user", req.UserInfo.Username, "gitTarget", asserted.GitTargetName, + "assertedAuthor", asserted.Name) + return true, admission.Response{} +} + // commandObjectUID extracts metadata.uid from an admission request's raw object. It // reads only the field it needs (like the audit handler's metadata probes), so it never // depends on the command kind being registered in a decoder scheme. diff --git a/pkg/manifestanalyzer/doc.go b/pkg/manifestanalyzer/doc.go new file mode 100644 index 00000000..64cf6439 --- /dev/null +++ b/pkg/manifestanalyzer/doc.go @@ -0,0 +1,34 @@ +// SPDX-License-Identifier: Apache-2.0 + +// Package manifestanalyzer is the public, stable answer to two questions a tool built +// around GitOps Reverser needs to ask about a Git repository, without a cluster and +// without writing anything: +// +// ScanFolder — may this folder become a GitTarget, and if not, why? +// ScanRepo — which folders in this repository could, and what shape is each one? +// +// The decisions come from the same acceptance gate the operator's writer enforces before +// it commits a byte, so a tool built on this package cannot drift from the operator that +// will later refuse the folder. Nothing here re-implements a rule. +// +// # Stability +// +// This package is the supported surface. The types below carry JSON tags and a +// [SchemaVersion]; fields are added, never repurposed or removed within a schema major. +// Everything under internal/ is not covered by that promise and is not importable from +// another module. +// +// The command-line equivalents are `manifest-analyzer --mode scan --format json` and +// `--mode repo-walker --format json`, which emit exactly the documents [FolderReport] +// and [RepoReport] marshal to. Exec the binary if Go is not your language; import this +// package if it is. +// +// # What it does not do +// +// Neither entry point resolves types against a live cluster, so neither reports whether a +// document's kind is actually served, nor produces a write plan. Both are structure-only: +// they read bytes, never follow symlinks, and never write. The operator applies the same +// gate plus the cluster-aware checks when a GitTarget adopts the folder — a folder this +// package accepts can still be refused for a reason only a cluster can see (an unresolved +// kind, an out-of-scope resource). +package manifestanalyzer diff --git a/pkg/manifestanalyzer/folder.go b/pkg/manifestanalyzer/folder.go new file mode 100644 index 00000000..e58a745d --- /dev/null +++ b/pkg/manifestanalyzer/folder.go @@ -0,0 +1,188 @@ +// SPDX-License-Identifier: Apache-2.0 + +package manifestanalyzer + +import ( + "context" + "encoding/json" + "io" + "io/fs" + + internalanalyzer "github.com/ConfigButler/gitops-reverser/internal/manifestanalyzer" +) + +// SchemaVersion identifies the JSON contract [FolderReport] and [RepoReport] marshal to. +// It bumps only on a breaking change — a field removed, renamed, or given a new meaning. +// Adding a field is not breaking, so consumers must ignore fields they do not know. +const SchemaVersion = "v1" + +// IssueKind classifies why a folder was not accepted. The values are the operator's own +// refusal codes, stable across releases. +type IssueKind string + +const ( + // IssueDuplicate marks a document that duplicates an earlier manifest identity. + IssueDuplicate IssueKind = "duplicate-identity" + // IssueNonKRM marks YAML that does not parse as a Kubernetes manifest. + IssueNonKRM IssueKind = "non-krm-yaml" + // IssueInvalidYAML marks a document that does not parse as YAML. + IssueInvalidYAML IssueKind = "invalid-yaml" + // IssueImpureManagedFile marks a file holding managed resources that also holds a + // non-managed document. A managed file may contain only valid KRM documents. + IssueImpureManagedFile IssueKind = "impure-managed-file" + // IssueMixedFile marks a managed file that also holds an allowlisted non-API KRM + // document (a kustomization). Allowlisted KRM must be retained in its own file. + IssueMixedFile IssueKind = "mixed-managed-allowlisted" + // IssueUnresolvedKRM marks recognized KRM that cannot be tied to a single served, + // followable resource. Only a cluster-aware scan reports this. + IssueUnresolvedKRM IssueKind = "unresolved-krm" + // IssueOutOfScope marks a watched kind whose resource falls outside the GitTarget's + // scope (right kind, wrong namespace). Only a cluster-aware scan reports this. + IssueOutOfScope IssueKind = "out-of-scope" + // IssueUnsupportedKustomize marks a kustomization.yaml using a feature the writer + // cannot map back to editable source documents (generators, patches, components, + // Helm inflation, replacements, transformers, name prefixes, remote bases). + IssueUnsupportedKustomize IssueKind = "unsupported-kustomize" + // IssueForeignFile marks a non-YAML regular file the operator cannot manage. + IssueForeignFile IssueKind = "foreign-file" + // IssueForeignSymlink marks a symlink, which a writer could follow out of the subtree. + IssueForeignSymlink IssueKind = "foreign-symlink" + // IssueForeignSubmodule marks a nested Git submodule. + IssueForeignSubmodule IssueKind = "foreign-submodule" + // IssueIgnoreShadowsManaged marks a .gittargetignore pattern that matches a path the + // operator writes, which would blind it to its own file. + IssueIgnoreShadowsManaged IssueKind = "ignore-shadows-managed" + // IssueWriteEscapesScope marks a planned write that would leave the GitTarget's path. + IssueWriteEscapesScope IssueKind = "write-escapes-scope" + // IssueWriteFanIn marks an in-place edit of a source file that more than one kustomize + // render root reaches. + IssueWriteFanIn IssueKind = "write-fan-in" +) + +// Issue is one reason a folder is not accepted, or one fact a stricter policy may treat +// as blocking. +type Issue struct { + Kind IssueKind `json:"kind"` + // Path is the offending file, slash-separated and relative to the scan root. Empty + // when the issue is about the folder as a whole. + Path string `json:"path"` + // DocumentIndex is the zero-based index of the offending document within Path. + DocumentIndex int `json:"documentIndex"` + // Message is a human-readable explanation. It is not a stable string. + Message string `json:"message"` +} + +// Identity names one Kubernetes document as it appears in the file. +type Identity struct { + APIVersion string `json:"apiVersion"` + Kind string `json:"kind"` + Namespace string `json:"namespace"` + Name string `json:"name"` +} + +// RetainedDocument is content the operator keeps but never writes: build directives such +// as a kustomization.yaml. They are read as context, never edited as resources. +type RetainedDocument struct { + Path string `json:"path"` + DocumentIndex int `json:"documentIndex"` + // Identity is set only when a named Kubernetes resource is hiding inside an + // allowlisted build-directive file — the mixed-file case, which is refused. It is nil + // for the ordinary whole-file retention of a kustomization.yaml, which names no + // resource. + Identity *Identity `json:"identity,omitempty"` + // Unsupported reports that this retained file uses a construct the writer cannot map + // back to editable source. It is the cause of an IssueUnsupportedKustomize. + Unsupported bool `json:"unsupported,omitempty"` +} + +// FolderReport answers "may this folder become a GitTarget?". +type FolderReport struct { + SchemaVersion string `json:"schemaVersion"` + // Root is the scanned folder as passed to ScanFolder. Informational; empty for + // ScanFolderFS. + Root string `json:"root,omitempty"` + // Accepted is the gate decision. When false, Issues says why. + Accepted bool `json:"accepted"` + // Issues is empty when Accepted, and never nil in the marshaled JSON. + Issues []Issue `json:"issues"` + // Retained lists the build directives read as context. + Retained []RetainedDocument `json:"retained,omitempty"` +} + +// ScanFolder runs the adoption gate over a folder on disk. It is read-only, needs no +// cluster, and never follows symlinks. +// +// The returned error covers only I/O: a folder that cannot be adopted is a successful +// scan with Accepted=false, not an error. +func ScanFolder(ctx context.Context, root string) (FolderReport, error) { + result, err := internalanalyzer.ScanDir(ctx, root, nil, nil, folderScanPolicy()) + if err != nil { + return FolderReport{}, err + } + report := folderReportFrom(result.Acceptance) + report.Root = root + return report, nil +} + +// ScanFolderFS is ScanFolder over an arbitrary [io/fs.FS] — an in-memory tree, a tarball, +// or a Git tree exposed as a filesystem. It cannot fail: an unreadable file becomes an +// issue, not an error. +func ScanFolderFS(ctx context.Context, fsys fs.FS) FolderReport { + return folderReportFrom(internalanalyzer.Scan(ctx, fsys, nil, nil, folderScanPolicy()).Acceptance) +} + +// WriteJSON writes the report as indented JSON — byte-for-byte what +// `manifest-analyzer --mode scan --format json` prints. +func (r FolderReport) WriteJSON(w io.Writer) error { + if r.Issues == nil { + r.Issues = []Issue{} + } + enc := json.NewEncoder(w) + enc.SetIndent("", " ") + return enc.Encode(r) +} + +// folderScanPolicy is the structure-only adoption gate: the default allowlist, no +// followability lookup, and no desired state. It is the same policy the CLI's scan mode +// applies, kept in one place so the two can never disagree. +func folderScanPolicy() internalanalyzer.ScanPolicy { + return internalanalyzer.ScanPolicy{ + Acceptance: internalanalyzer.AcceptancePolicy{Allowlist: internalanalyzer.DefaultAllowlist()}, + } +} + +// folderReportFrom projects the internal acceptance decision onto the public contract. +// It is deliberately a copy rather than a type alias: the public shape must be free to +// stay still while the internal one moves. +func folderReportFrom(acc internalanalyzer.Acceptance) FolderReport { + report := FolderReport{ + SchemaVersion: SchemaVersion, + Accepted: acc.Accepted, + Issues: make([]Issue, 0, len(acc.Issues)), + } + for _, issue := range acc.Issues { + report.Issues = append(report.Issues, Issue{ + Kind: IssueKind(issue.Kind), + Path: issue.Path, + DocumentIndex: issue.DocumentIndex, + Message: issue.Message, + }) + } + for _, rd := range acc.Retained { + doc := RetainedDocument{ + Path: rd.Location.Path, + DocumentIndex: rd.Location.DocumentIndex, + Unsupported: rd.Unsupported, + } + if rd.Identity.Kind != "" { + doc.Identity = &Identity{ + APIVersion: rd.Identity.APIVersion, + Kind: rd.Identity.Kind, + Namespace: rd.Identity.Namespace, + Name: rd.Identity.Name, + } + } + report.Retained = append(report.Retained, doc) + } + return report +} diff --git a/pkg/manifestanalyzer/folder_test.go b/pkg/manifestanalyzer/folder_test.go new file mode 100644 index 00000000..fb343095 --- /dev/null +++ b/pkg/manifestanalyzer/folder_test.go @@ -0,0 +1,176 @@ +// SPDX-License-Identifier: Apache-2.0 + +package manifestanalyzer_test + +import ( + "bytes" + "context" + "encoding/json" + "os" + "path/filepath" + "testing" + "testing/fstest" + + "github.com/stretchr/testify/require" + + "github.com/ConfigButler/gitops-reverser/pkg/manifestanalyzer" +) + +const configMapYAML = `apiVersion: v1 +kind: ConfigMap +metadata: + name: settings + namespace: demo +data: + key: value +` + +const kustomizationHelmYAML = `apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +helmCharts: + - name: podinfo + repo: https://stefanprodan.github.io/podinfo +` + +func TestScanFolder_AcceptsPlainKRM(t *testing.T) { + t.Parallel() + + root := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(root, "cm.yaml"), []byte(configMapYAML), 0o600)) + + report, err := manifestanalyzer.ScanFolder(context.Background(), root) + require.NoError(t, err) + + require.True(t, report.Accepted) + require.Empty(t, report.Issues) + require.Equal(t, manifestanalyzer.SchemaVersion, report.SchemaVersion) + require.Equal(t, root, report.Root) +} + +// A folder the operator would refuse must come back as a successful scan with +// Accepted=false. Refusal is a verdict, never an error. +func TestScanFolder_RefusalIsNotAnError(t *testing.T) { + t.Parallel() + + root := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(root, "cm.yaml"), []byte(configMapYAML), 0o600)) + require.NoError(t, os.WriteFile(filepath.Join(root, "kustomization.yaml"), []byte(kustomizationHelmYAML), 0o600)) + + report, err := manifestanalyzer.ScanFolder(context.Background(), root) + require.NoError(t, err) + + require.False(t, report.Accepted) + require.NotEmpty(t, report.Issues) + kinds := make([]manifestanalyzer.IssueKind, 0, len(report.Issues)) + for _, issue := range report.Issues { + kinds = append(kinds, issue.Kind) + } + require.Contains(t, kinds, manifestanalyzer.IssueUnsupportedKustomize, + "Helm inflation is the permanent support boundary and must be named as such") +} + +func TestScanFolder_MissingDirIsAnError(t *testing.T) { + t.Parallel() + _, err := manifestanalyzer.ScanFolder(context.Background(), filepath.Join(t.TempDir(), "absent")) + require.Error(t, err) +} + +func TestScanFolderFS_ReportsDuplicateIdentity(t *testing.T) { + t.Parallel() + + fsys := fstest.MapFS{ + "a.yaml": {Data: []byte(configMapYAML)}, + "b.yaml": {Data: []byte(configMapYAML)}, + } + report := manifestanalyzer.ScanFolderFS(context.Background(), fsys) + + require.False(t, report.Accepted) + require.Equal(t, manifestanalyzer.SchemaVersion, report.SchemaVersion) + require.Empty(t, report.Root, "an fs.FS has no path to report") + + var found bool + for _, issue := range report.Issues { + if issue.Kind == manifestanalyzer.IssueDuplicate { + found = true + require.NotEmpty(t, issue.Path) + require.NotEmpty(t, issue.Message) + } + } + require.True(t, found, "two documents with the same identity must be refused as duplicates") +} + +// The retained set names the build directives the operator reads but never writes. A +// consumer relies on it to explain "we understand this kustomization; we will not edit it". +func TestScanFolder_ReportsRetainedKustomization(t *testing.T) { + t.Parallel() + + root := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(root, "cm.yaml"), []byte(configMapYAML), 0o600)) + require.NoError(t, os.WriteFile(filepath.Join(root, "kustomization.yaml"), + []byte("apiVersion: kustomize.config.k8s.io/v1beta1\nkind: Kustomization\nresources:\n - cm.yaml\n"), 0o600)) + + report, err := manifestanalyzer.ScanFolder(context.Background(), root) + require.NoError(t, err) + require.True(t, report.Accepted) + require.Len(t, report.Retained, 1) + require.Equal(t, "kustomization.yaml", report.Retained[0].Path) + require.False(t, report.Retained[0].Unsupported) + require.Nil(t, report.Retained[0].Identity, + "a whole-file retention names no resource; only the refused mixed-file case does") +} + +// A kustomization whose feature set the writer cannot map back to source is retained and +// flagged, so a consumer can point at the exact file that made the folder unadoptable. +func TestScanFolder_FlagsUnsupportedRetainedKustomization(t *testing.T) { + t.Parallel() + + root := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(root, "kustomization.yaml"), []byte(kustomizationHelmYAML), 0o600)) + + report, err := manifestanalyzer.ScanFolder(context.Background(), root) + require.NoError(t, err) + require.False(t, report.Accepted) + require.Len(t, report.Retained, 1) + require.True(t, report.Retained[0].Unsupported) +} + +// The JSON document is the published contract: schemaVersion is always present, and +// issues is an empty array rather than null so a consumer can iterate it unconditionally. +func TestFolderReport_WriteJSON_Contract(t *testing.T) { + t.Parallel() + + root := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(root, "cm.yaml"), []byte(configMapYAML), 0o600)) + report, err := manifestanalyzer.ScanFolder(context.Background(), root) + require.NoError(t, err) + + var buf bytes.Buffer + require.NoError(t, report.WriteJSON(&buf)) + + require.Contains(t, buf.String(), `"issues": []`, "issues must marshal as [] and never null") + + var raw map[string]any + require.NoError(t, json.Unmarshal(buf.Bytes(), &raw)) + require.Equal(t, "v1", raw["schemaVersion"]) + require.Equal(t, true, raw["accepted"]) + require.NotContains(t, raw, "retained", "an empty retained set is omitted") +} + +// The public report must survive a marshal/unmarshal round trip unchanged, or a consumer +// storing it and reading it back would see a different verdict than the one produced. +func TestFolderReport_JSONRoundTrip(t *testing.T) { + t.Parallel() + + root := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(root, "cm.yaml"), []byte(configMapYAML), 0o600)) + require.NoError(t, os.WriteFile(filepath.Join(root, "kustomization.yaml"), []byte(kustomizationHelmYAML), 0o600)) + report, err := manifestanalyzer.ScanFolder(context.Background(), root) + require.NoError(t, err) + + var buf bytes.Buffer + require.NoError(t, report.WriteJSON(&buf)) + + var decoded manifestanalyzer.FolderReport + require.NoError(t, json.Unmarshal(buf.Bytes(), &decoded)) + require.Equal(t, report, decoded) +} diff --git a/pkg/manifestanalyzer/repo.go b/pkg/manifestanalyzer/repo.go new file mode 100644 index 00000000..f4d22bca --- /dev/null +++ b/pkg/manifestanalyzer/repo.go @@ -0,0 +1,179 @@ +// SPDX-License-Identifier: Apache-2.0 + +package manifestanalyzer + +import ( + "context" + "encoding/json" + "io" + + internalanalyzer "github.com/ConfigButler/gitops-reverser/internal/manifestanalyzer" +) + +// Layout is the structural shape of a candidate folder. +type Layout string + +const ( + // LayoutPlain is a directory of raw KRM documents with explicit namespaces and no + // kustomization. Accepted. + LayoutPlain Layout = "plain" + // LayoutKustomizeSingle is a self-contained render root: one kustomization whose + // resources graph stays within its own subtree. Accepted. + LayoutKustomizeSingle Layout = "kustomize-single" + // LayoutKustomizeOverlay is a render root reaching a base outside its own subtree + // (the classic base/ + overlays/{env} shape). Refused today, with a forward-looking + // reason: it becomes accepted when render-root scoping ships. + LayoutKustomizeOverlay Layout = "kustomize-overlay" + // LayoutRefusedStructural is a render root whose kustomization uses a construct the + // writer cannot map back to editable source. This is the permanent support boundary, + // never a "not yet". + LayoutRefusedStructural Layout = "refused-structural" +) + +// Refusal reason codes. The distinction is load-bearing and a consumer must not collapse +// it: one is a "not yet", the other is permanent. +const ( + // ReasonOverlayFanOutNeedsF2 is the forward-looking refusal that flips to accepted + // when render-root scoping ships. + ReasonOverlayFanOutNeedsF2 = "overlay-fan-out-needs-f2" + // ReasonRefusedStructural is the permanent support boundary. + ReasonRefusedStructural = "refused-structural" +) + +// RefusalReason is one machine-readable reason a candidate is not accepted. +type RefusalReason struct { + Code string `json:"code"` + // Detail is human-readable and not a stable string. + Detail string `json:"detail"` +} + +// ResourceCounts splits the KRM a candidate covers into what it renders versus what it +// could actually edit. The two are equal for a plain or self-contained candidate and +// diverge for an overlay, which renders documents it cannot own. +type ResourceCounts struct { + Rendered int `json:"rendered"` + Editable int `json:"editable"` + NonKRM int `json:"nonKrm"` +} + +// Candidate is one folder that could become a GitTarget. +type Candidate struct { + // Path is slash-separated and relative to the repository root. + Path string `json:"path"` + Layout Layout `json:"layout"` + // AcceptedByOperator reports whether the operator would adopt this folder today. + AcceptedByOperator bool `json:"acceptedByOperator"` + RefusalReasons []RefusalReason `json:"refusalReasons,omitempty"` + // RenderRoot reports whether the candidate is a kustomize render root. + RenderRoot bool `json:"renderRoot"` + // ReadScope lists base directories outside this candidate's subtree that its + // kustomization reads. Empty for plain and self-contained candidates. + ReadScope []string `json:"readScope,omitempty"` + // InferredNamespace is the namespace the candidate resolves to, when unambiguous. + InferredNamespace string `json:"inferredNamespace,omitempty"` + Resources ResourceCounts `json:"resources"` + // OverlapsWith lists candidate paths this one nests with. Two overlapping candidates + // can never both become GitTargets — a folder has exactly one owner. + OverlapsWith []string `json:"overlapsWith,omitempty"` +} + +// OverlapConflict records that Ancestor strictly contains Descendant. +type OverlapConflict struct { + Ancestor string `json:"ancestor"` + Descendant string `json:"descendant"` +} + +// RepoSummary is the repository-level roll-up. +type RepoSummary struct { + CandidatesByLayout map[Layout]int `json:"candidatesByLayout"` + Accepted int `json:"accepted"` + Refused int `json:"refused"` + OverlapConflicts []OverlapConflict `json:"overlapConflicts,omitempty"` + // FleetRoot reports that the repository root is a cluster/fleet root. A GitTarget + // points at an app subtree, never at such a root. + FleetRoot bool `json:"fleetRoot,omitempty"` + // UnsupportedConstructs is the sorted, de-duplicated set of unsupported kustomize + // features seen across refused candidates, so a tool can say "this repository uses + // Helm inflation, which the operator does not manage". + UnsupportedConstructs []string `json:"unsupportedConstructs,omitempty"` +} + +// RepoReport answers "which folders in this repository could become GitTargets?". +type RepoReport struct { + SchemaVersion string `json:"schemaVersion"` + // Root is the scanned repository root as passed to ScanRepo. Informational. + Root string `json:"root,omitempty"` + Candidates []Candidate `json:"candidates"` + Summary RepoSummary `json:"summary"` +} + +// ScanRepo walks a whole repository and enumerates the folders that could become +// GitTargets, classifying each one's layout and reporting why a folder is refused. It is +// read-only, needs no cluster, and never follows symlinks. +func ScanRepo(ctx context.Context, root string) (RepoReport, error) { + rep, err := internalanalyzer.WalkRepo(ctx, root) + if err != nil { + return RepoReport{}, err + } + return repoReportFrom(rep), nil +} + +// WriteJSON writes the report as indented JSON — byte-for-byte what +// `manifest-analyzer --mode repo-walker --format json` prints. +func (r RepoReport) WriteJSON(w io.Writer) error { + if r.Candidates == nil { + r.Candidates = []Candidate{} + } + enc := json.NewEncoder(w) + enc.SetIndent("", " ") + return enc.Encode(r) +} + +// repoReportFrom projects the internal discovery report onto the public contract. +func repoReportFrom(rep internalanalyzer.RepoReport) RepoReport { + out := RepoReport{ + SchemaVersion: SchemaVersion, + Root: rep.Root, + Candidates: make([]Candidate, 0, len(rep.Candidates)), + Summary: RepoSummary{ + CandidatesByLayout: make(map[Layout]int, len(rep.Summary.CandidatesByLayout)), + Accepted: rep.Summary.Accepted, + Refused: rep.Summary.Refused, + FleetRoot: rep.Summary.FleetRoot, + UnsupportedConstructs: rep.Summary.UnsupportedConstructs, + }, + } + for layout, n := range rep.Summary.CandidatesByLayout { + out.Summary.CandidatesByLayout[Layout(layout)] = n + } + for _, conflict := range rep.Summary.OverlapConflicts { + out.Summary.OverlapConflicts = append(out.Summary.OverlapConflicts, OverlapConflict{ + Ancestor: conflict.Ancestor, Descendant: conflict.Descendant, + }) + } + for _, cand := range rep.Candidates { + out.Candidates = append(out.Candidates, candidateFrom(cand)) + } + return out +} + +func candidateFrom(cand internalanalyzer.RepoCandidate) Candidate { + out := Candidate{ + Path: cand.Path, + Layout: Layout(cand.Layout), + AcceptedByOperator: cand.AcceptedByOperator, + RenderRoot: cand.RenderRoot, + ReadScope: cand.ReadScope, + InferredNamespace: cand.InferredNamespace, + Resources: ResourceCounts{ + Rendered: cand.Resources.Rendered, + Editable: cand.Resources.Editable, + NonKRM: cand.Resources.NonKRM, + }, + OverlapsWith: cand.OverlapsWith, + } + for _, reason := range cand.RefusalReasons { + out.RefusalReasons = append(out.RefusalReasons, RefusalReason{Code: reason.Code, Detail: reason.Detail}) + } + return out +} diff --git a/pkg/manifestanalyzer/repo_test.go b/pkg/manifestanalyzer/repo_test.go new file mode 100644 index 00000000..9a3d63de --- /dev/null +++ b/pkg/manifestanalyzer/repo_test.go @@ -0,0 +1,168 @@ +// SPDX-License-Identifier: Apache-2.0 + +package manifestanalyzer_test + +import ( + "bytes" + "context" + "encoding/json" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/ConfigButler/gitops-reverser/pkg/manifestanalyzer" +) + +// corpusRoot is the discovery corpus the engine's own golden tests drive. The public +// package reads the same fixtures rather than growing a second copy: the point of this +// package is that it cannot disagree with the engine, and a duplicated corpus is the +// first way that guarantee rots. Tests run with the package directory as cwd. +const corpusRoot = "../../internal/manifestanalyzer/testdata/repo-walker" + +func fixture(t *testing.T, group, name string) string { + t.Helper() + path := filepath.Join(corpusRoot, group, name) + info, err := os.Stat(path) + require.NoError(t, err, "discovery corpus fixture %s must exist", path) + require.True(t, info.IsDir()) + return path +} + +func TestScanRepo_PlainPerEnvironmentFoldersAreCandidates(t *testing.T) { + t.Parallel() + + report, err := manifestanalyzer.ScanRepo(context.Background(), fixture(t, "supported", "plain-per-env")) + require.NoError(t, err) + + require.Equal(t, manifestanalyzer.SchemaVersion, report.SchemaVersion) + require.NotEmpty(t, report.Candidates) + for _, cand := range report.Candidates { + require.Equal(t, manifestanalyzer.LayoutPlain, cand.Layout) + require.True(t, cand.AcceptedByOperator, "plain KRM folders are the launch layout") + require.Empty(t, cand.RefusalReasons) + require.False(t, cand.RenderRoot) + } + require.Equal(t, len(report.Candidates), report.Summary.Accepted) + require.Zero(t, report.Summary.Refused) +} + +// The two refusal codes mean different things and a consumer must be able to tell them +// apart: one is "not yet, when render-root scoping ships", the other is the permanent +// support boundary. Collapsing them would mislabel an onboardable repository as hopeless. +func TestScanRepo_RefusalCodesStayDistinct(t *testing.T) { + t.Parallel() + + tests := map[string]struct { + group, name string + wantCode string + wantLayout manifestanalyzer.Layout + }{ + "overlay fan-out is a not-yet": { + group: "unsupported", name: "base-overlays", + wantCode: manifestanalyzer.ReasonOverlayFanOutNeedsF2, + wantLayout: manifestanalyzer.LayoutKustomizeOverlay, + }, + "helm inflation is permanent": { + group: "unsupported", name: "helm-inflation", + wantCode: manifestanalyzer.ReasonRefusedStructural, + wantLayout: manifestanalyzer.LayoutRefusedStructural, + }, + "unsupported kustomize is permanent": { + group: "unsupported", name: "unsupported-kustomize", + wantCode: manifestanalyzer.ReasonRefusedStructural, + wantLayout: manifestanalyzer.LayoutRefusedStructural, + }, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + t.Parallel() + report, err := manifestanalyzer.ScanRepo(context.Background(), fixture(t, tc.group, tc.name)) + require.NoError(t, err) + + var seen bool + for _, cand := range report.Candidates { + if cand.Layout != tc.wantLayout { + continue + } + seen = true + require.False(t, cand.AcceptedByOperator) + codes := make([]string, 0, len(cand.RefusalReasons)) + for _, reason := range cand.RefusalReasons { + codes = append(codes, reason.Code) + } + require.Contains(t, codes, tc.wantCode) + } + require.True(t, seen, "expected at least one %s candidate", tc.wantLayout) + }) + } +} + +func TestScanRepo_ReportsOverlapConflicts(t *testing.T) { + t.Parallel() + + report, err := manifestanalyzer.ScanRepo(context.Background(), fixture(t, "supported", "overlapping")) + require.NoError(t, err) + + require.NotEmpty(t, report.Summary.OverlapConflicts, + "two nested candidates can never both own a folder; the conflict must surface") + for _, conflict := range report.Summary.OverlapConflicts { + require.NotEmpty(t, conflict.Ancestor) + require.NotEmpty(t, conflict.Descendant) + require.NotEqual(t, conflict.Ancestor, conflict.Descendant) + } +} + +func TestScanRepo_FleetRootIsNeverACandidate(t *testing.T) { + t.Parallel() + + root := fixture(t, "supported", "fleet-root") + report, err := manifestanalyzer.ScanRepo(context.Background(), root) + require.NoError(t, err) + + require.True(t, report.Summary.FleetRoot) + for _, cand := range report.Candidates { + require.NotEqual(t, ".", cand.Path, "a GitTarget points at an app subtree, never the fleet root") + } +} + +func TestScanRepo_MissingRootIsAnError(t *testing.T) { + t.Parallel() + _, err := manifestanalyzer.ScanRepo(context.Background(), filepath.Join(t.TempDir(), "absent")) + require.Error(t, err) +} + +func TestRepoReport_WriteJSON_Contract(t *testing.T) { + t.Parallel() + + report, err := manifestanalyzer.ScanRepo(context.Background(), fixture(t, "supported", "kustomize-single")) + require.NoError(t, err) + + var buf bytes.Buffer + require.NoError(t, report.WriteJSON(&buf)) + + var raw map[string]any + require.NoError(t, json.Unmarshal(buf.Bytes(), &raw)) + require.Equal(t, "v1", raw["schemaVersion"]) + require.Contains(t, raw, "candidates") + require.Contains(t, raw, "summary") + + var decoded manifestanalyzer.RepoReport + require.NoError(t, json.Unmarshal(buf.Bytes(), &decoded)) + require.Equal(t, report, decoded) +} + +// An empty repository still marshals candidates as an array, so a consumer can iterate it +// without a nil check. +func TestRepoReport_WriteJSON_EmptyCandidatesIsArray(t *testing.T) { + t.Parallel() + + report, err := manifestanalyzer.ScanRepo(context.Background(), t.TempDir()) + require.NoError(t, err) + + var buf bytes.Buffer + require.NoError(t, report.WriteJSON(&buf)) + require.Contains(t, buf.String(), `"candidates": []`) +} diff --git a/test/e2e/asserted_author_e2e_test.go b/test/e2e/asserted_author_e2e_test.go new file mode 100644 index 00000000..284d2e45 --- /dev/null +++ b/test/e2e/asserted_author_e2e_test.go @@ -0,0 +1,241 @@ +// SPDX-License-Identifier: Apache-2.0 + +package e2e + +import ( + "fmt" + "strings" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// CommitRequest.spec.author lets a trusted control plane state who a commit is for, instead +// of deriving it from an apiserver audit fact. Asserting an author is a privilege: it is +// honored only when the requester holds the `assert-author` verb on the named GitTarget. +// +// Two properties matter, and both need a real API server with the admission webhook: +// +// 1. An UNAUTHORIZED assertion is denied at admission, with a message naming the RBAC rule +// that would grant it. +// 2. An AUTHORIZED assertion becomes the commit's author signature — while the committer +// stays the operator's configured identity, so a reader can always tell the reverser +// committed on someone's behalf. +// +// The controller, not the webhook, is the real gate (the webhook is failurePolicy: Ignore), +// but the webhook's denial is what an authorized caller sees first, so both are asserted. +// +// Serial-safe by construction: the spec owns its own repo, GitTarget, and namespace. +var _ = Describe("Asserted commit author", Label("commit-request", "audit-consumer"), Ordered, func() { + var ( + testNs string + repo *RepoArtifacts + provider string + target string + watchRule string + basePath string + asserterUser string + unprivileged string + bindingPrefix string + ) + + const ( + // Long enough that the silence timer cannot be what produces the commit. + commitWindow = "300s" + assertedName = "Ada Lovelace" + assertedEmail = "ada@example.com" + ) + + BeforeAll(func() { + By("creating the asserted-author namespace and Git credentials") + testNs = testNamespaceFor("asserted-author") + _, _ = kubectlRun("create", "namespace", testNs) + repo = SetupRepo(resolveE2EContext(), testNs, fmt.Sprintf("e2e-asserted-author-%d", GinkgoRandomSeed())) + _, err := kubectlRunInNamespace(testNs, "apply", "-f", repo.SecretsYAML) + Expect(err).NotTo(HaveOccurred(), "failed to apply git secrets to namespace") + applySOPSAgeKeyToNamespace(testNs) + + seed := GinkgoRandomSeed() + provider = fmt.Sprintf("asserted-author-provider-%d", seed) + target = fmt.Sprintf("asserted-author-target-%d", seed) + watchRule = fmt.Sprintf("asserted-author-rule-%d", seed) + basePath = "e2e/asserted-author" + asserterUser = fmt.Sprintf("gitops-api-%d@e2e", seed) + unprivileged = fmt.Sprintf("nobody-%d@e2e", seed) + bindingPrefix = fmt.Sprintf("asserted-author-%d", seed) + + createGitProviderWithCommitWindow(provider, testNs, repo.GitSecretHTTP, repo.RepoURLHTTP, commitWindow) + verifyResourceStatus("gitprovider", provider, testNs, "True", "Ready", "Repository connectivity validated") + + createGitTarget(target, testNs, provider, basePath, "main") + verifyResourceCondition("gittarget", target, testNs, "Validated", "True", "OK", "") + + rule := fmt.Sprintf(`apiVersion: configbutler.ai/v1alpha3 +kind: WatchRule +metadata: + name: %s + namespace: %s +spec: + targetRef: + kind: GitTarget + name: %s + rules: + - resources: ["configmaps"] +`, watchRule, testNs, target) + _, err = kubectlRunWithStdin(testNs, rule, "apply", "-f", "-") + Expect(err).NotTo(HaveOccurred(), "failed to apply WatchRule") + verifyResourceStatus("watchrule", watchRule, testNs, "True", "Ready", "") + waitForStreamsRunning(target, testNs) + + By("granting only asserterUser the assert-author verb, scoped to this one GitTarget") + // Both users may create CommitRequests; only one may assert an author. The grant is + // resourceNames-scoped, exactly as the documented example is. + rbac := fmt.Sprintf(`apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: %s-commit + namespace: %s +rules: + - apiGroups: ["configbutler.ai"] + resources: ["commitrequests"] + verbs: ["create", "get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: %s-commit + namespace: %s +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: %s-commit +subjects: + - kind: User + name: %s + apiGroup: rbac.authorization.k8s.io + - kind: User + name: %s + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: %s-assert + namespace: %s +rules: + - apiGroups: ["configbutler.ai"] + resources: ["gittargets"] + resourceNames: ["%s"] + verbs: ["assert-author"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: %s-assert + namespace: %s +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: %s-assert +subjects: + - kind: User + name: %s + apiGroup: rbac.authorization.k8s.io +`, + bindingPrefix, testNs, + bindingPrefix, testNs, bindingPrefix, asserterUser, unprivileged, + bindingPrefix, testNs, target, + bindingPrefix, testNs, bindingPrefix, asserterUser) + _, err = kubectlRunWithStdin(testNs, rbac, "apply", "-f", "-") + Expect(err).NotTo(HaveOccurred(), "failed to apply assert-author RBAC") + }) + + AfterAll(func() { + cleanupWatchRule(watchRule, testNs) + cleanupGitTarget(target, testNs) + cleanupNamespace(testNs) + }) + + // assertingCommitRequest renders a CommitRequest that asserts an author. + // + // closeDelaySeconds is an extra collect window, anchored at the moment the worker receives + // the attach. Without it a CommitRequest created right after an edit resolves + // NoWindowInGrace: the edit's watch event has not reached the branch worker yet, so there + // is no open window to attach to. + assertingCommitRequest := func(name string) string { + return fmt.Sprintf(`apiVersion: configbutler.ai/v1alpha3 +kind: CommitRequest +metadata: + name: %s + namespace: %s +spec: + targetRef: + name: %s + message: "save: asserted author" + closeDelaySeconds: 20 + author: + name: %q + email: %q +`, name, testNs, target, assertedName, assertedEmail) + } + + It("denies an unauthorized assertion, and names the RBAC rule that would grant it", func() { + // create, not apply: apply would need patch/update on an object that never persists, + // and the point here is the admission denial, not the client-side merge. + out, err := kubectlRunWithStdin(testNs, assertingCommitRequest("denied-assertion"), + "create", "--as="+unprivileged, "-f", "-") + + Expect(err).To(HaveOccurred(), "a user without assert-author must not be able to set spec.author") + Expect(out).To(ContainSubstring("assert-author")) + Expect(out).To(ContainSubstring(target), "the denial must name the GitTarget the verb is checked against") + Expect(out).To(ContainSubstring("resourceNames"), + "the denial must show the RBAC rule that would grant the privilege, not just refuse") + }) + + It("lets an authorized caller name the commit author, with the operator as committer", func() { + const cmName = "asserted-author-configmap" + const crName = "asserted-author-save" + + By("an edit opens a commit window that the long commitWindow will not close on its own") + _, err := kubectlRunInNamespace(testNs, "create", "configmap", cmName, "--from-literal=flavor=vanilla") + Expect(err).NotTo(HaveOccurred()) + + By("the authorized caller creates a CommitRequest asserting the author") + _, err = kubectlRunWithStdin(testNs, assertingCommitRequest(crName), "create", "--as="+asserterUser, "-f", "-") + Expect(err).NotTo(HaveOccurred(), "an authorized assertion must be admitted") + + By("the request reports AuthorAttributed=True with reason AuthorAsserted") + Eventually(func(g Gomega) { + g.Expect(commitRequestCondition(g, testNs, crName, "Ready")).To(Equal("True")) + g.Expect(commitRequestCondition(g, testNs, crName, "Pushed")).To(Equal("True")) + + reason, readErr := kubectlRunInNamespace(testNs, "get", "commitrequest", crName, "-o", + `jsonpath={.status.conditions[?(@.type=="AuthorAttributed")].reason}`) + g.Expect(readErr).NotTo(HaveOccurred()) + g.Expect(strings.TrimSpace(reason)).To(Equal("AuthorAsserted")) + + status, readErr := kubectlRunInNamespace(testNs, "get", "commitrequest", crName, "-o", + `jsonpath={.status.conditions[?(@.type=="AuthorAttributed")].status}`) + g.Expect(readErr).NotTo(HaveOccurred()) + g.Expect(strings.TrimSpace(status)).To(Equal("True")) + }, 2*time.Minute, 3*time.Second).Should(Succeed()) + + By("the commit is authored by the asserted identity, and committed by the operator") + Eventually(func(g Gomega) { + pullLatestRepoState(g, repo.CheckoutDir) + + author, logErr := gitRun(repo.CheckoutDir, "log", "-1", "--pretty=%an <%ae>") + g.Expect(logErr).NotTo(HaveOccurred()) + g.Expect(strings.TrimSpace(author)).To(Equal(fmt.Sprintf("%s <%s>", assertedName, assertedEmail))) + + committer, logErr := gitRun(repo.CheckoutDir, "log", "-1", "--pretty=%cn") + g.Expect(logErr).NotTo(HaveOccurred()) + g.Expect(strings.TrimSpace(committer)).NotTo(Equal(assertedName), + "the committer must stay the operator: a reader can always tell it committed on someone's behalf") + }, 2*time.Minute, 3*time.Second).Should(Succeed()) + + _, _ = kubectlRunInNamespace(testNs, "delete", "configmap", cmName, "--ignore-not-found=true") + _, _ = kubectlRunInNamespace(testNs, "delete", "commitrequest", crName, "--ignore-not-found=true") + }) +}) diff --git a/test/e2e/gittarget_retarget_e2e_test.go b/test/e2e/gittarget_retarget_e2e_test.go new file mode 100644 index 00000000..0e640e98 --- /dev/null +++ b/test/e2e/gittarget_retarget_e2e_test.go @@ -0,0 +1,171 @@ +// SPDX-License-Identifier: Apache-2.0 + +package e2e + +import ( + "fmt" + "os" + "path" + "path/filepath" + "strings" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// A GitTarget's spec.path and spec.branch are mutable: changing either is a supported +// retarget. The controller tears the old materialization down — watches, event stream, and +// the durable resume cursors — and rebuilds the folder from a fresh full snapshot at the new +// destination. +// +// Three things have to hold, and only an e2e can show them together: +// +// 1. The new folder receives the resources that ALREADY existed, not just the ones that +// change afterwards. That is what dropping the resume cursors buys; without it a +// retargeted GitTarget resumes mid-stream into an empty folder. +// 2. The old folder is left in place. Deleting from Git is the one irreversible thing this +// operator does, and a destination change is the moment an operator is least sure of +// what they meant. +// 3. status.observedDestination follows the content, not the spec: it is written only once +// a snapshot has actually landed at the new destination. +// +// Not Serial: the spec owns a dedicated Gitea repo, so nothing else writes its branch. +var _ = Describe("GitTarget retarget", Label("manager"), Ordered, func() { + var ( + testNs string + repo *RepoArtifacts + provider string + target string + watchRule string + ) + + const ( + originalPath = "e2e/retarget/before" + movedPath = "e2e/retarget/after" + cmName = "retargeted-configmap" + ) + + BeforeAll(func() { + By("creating the retarget namespace and Git credentials") + testNs = testNamespaceFor("retarget") + _, _ = kubectlRun("create", "namespace", testNs) + repo = SetupRepo(resolveE2EContext(), testNs, fmt.Sprintf("e2e-retarget-%d", GinkgoRandomSeed())) + _, err := kubectlRunInNamespace(testNs, "apply", "-f", repo.SecretsYAML) + Expect(err).NotTo(HaveOccurred(), "failed to apply git secrets to namespace") + applySOPSAgeKeyToNamespace(testNs) + + seed := GinkgoRandomSeed() + provider = fmt.Sprintf("retarget-provider-%d", seed) + target = fmt.Sprintf("retarget-target-%d", seed) + watchRule = fmt.Sprintf("retarget-rule-%d", seed) + + createGitProviderWithURLInNamespace(provider, testNs, repo.GitSecretHTTP, repo.RepoURLHTTP) + verifyResourceStatus("gitprovider", provider, testNs, "True", "Ready", "Repository connectivity validated") + + createGitTarget(target, testNs, provider, originalPath, "main") + verifyResourceCondition("gittarget", target, testNs, "Validated", "True", "OK", "") + + rule := fmt.Sprintf(`apiVersion: configbutler.ai/v1alpha3 +kind: WatchRule +metadata: + name: %s + namespace: %s +spec: + targetRef: + kind: GitTarget + name: %s + rules: + - resources: ["configmaps"] +`, watchRule, testNs, target) + _, err = kubectlRunWithStdin(testNs, rule, "apply", "-f", "-") + Expect(err).NotTo(HaveOccurred(), "failed to apply WatchRule") + verifyResourceStatus("watchrule", watchRule, testNs, "True", "Ready", "") + waitForStreamsRunning(target, testNs) + }) + + AfterAll(func() { + cleanupWatchRule(watchRule, testNs) + cleanupGitTarget(target, testNs) + cleanupNamespace(testNs) + }) + + observedPath := func() string { + GinkgoHelper() + out, err := kubectlRunInNamespace(testNs, "get", "gittarget", target, + "-o", "jsonpath={.status.observedDestination.path}") + Expect(err).NotTo(HaveOccurred()) + return strings.TrimSpace(out) + } + + It("rebuilds the folder at the new path, leaves the old one, and follows with observedDestination", func() { + beforeRel := path.Join(originalPath, fmt.Sprintf("%s/configmaps/%s.yaml", testNs, cmName)) + afterRel := path.Join(movedPath, fmt.Sprintf("%s/configmaps/%s.yaml", testNs, cmName)) + beforeAbs := filepath.Join(repo.CheckoutDir, beforeRel) + afterAbs := filepath.Join(repo.CheckoutDir, afterRel) + + By("creating a ConfigMap that lands in the original folder") + _, err := kubectlRunInNamespace(testNs, "create", "configmap", cmName, "--from-literal=flavor=vanilla") + Expect(err).NotTo(HaveOccurred()) + + Eventually(func(g Gomega) { + pullLatestRepoState(g, repo.CheckoutDir) + content, readErr := os.ReadFile(beforeAbs) + g.Expect(readErr).NotTo(HaveOccurred(), "the ConfigMap must be committed at %s", beforeRel) + g.Expect(string(content)).To(ContainSubstring("flavor: vanilla")) + }, 2*time.Minute, 2*time.Second).Should(Succeed()) + + By("status.observedDestination names the folder the content is actually in") + Eventually(observedPath, 90*time.Second, 2*time.Second).Should(Equal(originalPath)) + + By("repointing spec.path — which used to be rejected by the API server") + _, err = kubectlRunInNamespace(testNs, "patch", "gittarget", target, "--type=merge", + "--patch", fmt.Sprintf(`{"spec":{"path":%q}}`, movedPath)) + Expect(err).NotTo(HaveOccurred(), "spec.path must be mutable: changing it is a supported retarget") + + By("the pre-existing ConfigMap appears in the new folder from a fresh full snapshot") + // Nothing changed the ConfigMap after the move. If the resume cursors survived the + // retarget, the new folder would stay empty forever — which is the whole reason the + // teardown drops them. + Eventually(func(g Gomega) { + pullLatestRepoState(g, repo.CheckoutDir) + content, readErr := os.ReadFile(afterAbs) + g.Expect(readErr).NotTo(HaveOccurred(), "the retargeted folder must be rebuilt at %s", afterRel) + g.Expect(string(content)).To(ContainSubstring("flavor: vanilla")) + }, 3*time.Minute, 3*time.Second).Should(Succeed()) + + By("the abandoned folder is left in place, as unmanaged Git content") + _, statErr := os.Stat(beforeAbs) + Expect(statErr).NotTo(HaveOccurred(), + "a retarget must never delete the old folder: %s should still exist", beforeRel) + + By("observedDestination follows the content, and Retargeting settles") + Eventually(observedPath, 2*time.Minute, 2*time.Second).Should(Equal(movedPath)) + verifyResourceCondition("gittarget", target, testNs, + "Retargeting", "False", "DestinationSettled", "was abandoned") + + By("a live edit now lands in the new folder") + _, err = kubectlRunInNamespace(testNs, "patch", "configmap", cmName, "--type=merge", + "--patch", `{"data":{"flavor":"chocolate"}}`) + Expect(err).NotTo(HaveOccurred()) + + Eventually(func(g Gomega) { + pullLatestRepoState(g, repo.CheckoutDir) + content, readErr := os.ReadFile(afterAbs) + g.Expect(readErr).NotTo(HaveOccurred()) + g.Expect(string(content)).To(ContainSubstring("flavor: chocolate")) + }, 2*time.Minute, 2*time.Second).Should(Succeed()) + + _, _ = kubectlRunInNamespace(testNs, "delete", "configmap", cmName, "--ignore-not-found=true") + }) + + // The repository is a different object, with nothing to migrate and nothing to observe. + It("still rejects a providerRef change, and says which fields are mutable", func() { + out, err := kubectlRunInNamespace(testNs, "patch", "gittarget", target, "--type=merge", + "--patch", `{"spec":{"providerRef":{"name":"some-other-provider"}}}`) + Expect(err).To(HaveOccurred(), "spec.providerRef must stay immutable") + Expect(out).To(ContainSubstring("spec.providerRef is immutable")) + Expect(out).To(ContainSubstring("spec.branch and spec.path are mutable"), + "the refusal must point at the supported move rather than just refusing") + }) +}) diff --git a/test/e2e/write_exclusion_e2e_test.go b/test/e2e/write_exclusion_e2e_test.go new file mode 100644 index 00000000..46aad18f --- /dev/null +++ b/test/e2e/write_exclusion_e2e_test.go @@ -0,0 +1,215 @@ +// SPDX-License-Identifier: Apache-2.0 + +package e2e + +import ( + "fmt" + "os" + "path" + "path/filepath" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// Identity-based write exclusion is the fix for the loop a reverser forms with a GitOps +// forward leg on the same branch: the forward leg applies the branch back into the cluster, +// the reverser mirrors that apply as a live update, and the new commit re-triggers the +// forward leg. +// +// These specs stand in for the forward leg with `kubectl apply --server-side +// --field-manager=kustomize-controller`. That is exactly what a GitOps tool does, and the +// field manager is the whole signal `excludeFieldManagers` reads, so the substitution is +// faithful and needs no Flux install. +// +// The forward leg's apply changes `data`, not a label: internal/sanitize already strips +// `kustomize.toolkit.fluxcd.io/*` labels and annotations from Git content, so a stamped +// label alone would carry no change to mirror and the content dedup would drop it before any +// exclusion was consulted. A content change is the case the exclusion actually decides. +// +// Two GitTargets watch the same ConfigMaps into sibling folders: one whose rule declines the +// forward leg's writes, one whose rule does not. The control target is what makes the +// assertion sharp — once the forward leg's value appears in the mirrored folder, the event +// has demonstrably been processed, so its absence in the excluded folder is the exclusion +// doing its job rather than a race we did not wait long enough for. +// +// Not Serial: the spec owns a dedicated Gitea repo, so nothing else writes its branch. +var _ = Describe("Write exclusion", Label("manager"), Ordered, func() { + var ( + testNs string + repo *RepoArtifacts + provider string + ) + + const ( + fluxFieldManager = "kustomize-controller" + humanFieldManager = "kubectl-e2e-human" + + excludedPath = "e2e/write-exclusion/excluded" + mirroredPath = "e2e/write-exclusion/mirrored" + ) + + var excludedTarget, mirroredTarget string + + // applyRule applies a WatchRule for one GitTarget, optionally declining a field manager. + applyRule := func(name, target, excludeFieldManager string) { + GinkgoHelper() + exclude := "" + if excludeFieldManager != "" { + exclude = fmt.Sprintf("\n excludeFieldManagers: [%q]", excludeFieldManager) + } + rule := fmt.Sprintf(`apiVersion: configbutler.ai/v1alpha3 +kind: WatchRule +metadata: + name: %s + namespace: %s +spec: + targetRef: + kind: GitTarget + name: %s + rules: + - resources: ["configmaps"]%s +`, name, testNs, target, exclude) + _, err := kubectlRunWithStdin(testNs, rule, "apply", "-f", "-") + Expect(err).NotTo(HaveOccurred(), "failed to apply WatchRule %s", name) + verifyResourceStatus("watchrule", name, testNs, "True", "Ready", "") + } + + BeforeAll(func() { + By("creating the write-exclusion namespace and Git credentials") + testNs = testNamespaceFor("write-exclusion") + _, _ = kubectlRun("create", "namespace", testNs) + repo = SetupRepo(resolveE2EContext(), testNs, fmt.Sprintf("e2e-write-exclusion-%d", GinkgoRandomSeed())) + _, err := kubectlRunInNamespace(testNs, "apply", "-f", repo.SecretsYAML) + Expect(err).NotTo(HaveOccurred(), "failed to apply git secrets to namespace") + applySOPSAgeKeyToNamespace(testNs) + + seed := GinkgoRandomSeed() + provider = fmt.Sprintf("write-exclusion-provider-%d", seed) + excludedTarget = fmt.Sprintf("write-exclusion-excluded-%d", seed) + mirroredTarget = fmt.Sprintf("write-exclusion-mirrored-%d", seed) + + createGitProviderWithURLInNamespace(provider, testNs, repo.GitSecretHTTP, repo.RepoURLHTTP) + verifyResourceStatus("gitprovider", provider, testNs, "True", "Ready", "Repository connectivity validated") + + By("creating one GitTarget that declines the forward leg's writes and one that does not") + createGitTarget(excludedTarget, testNs, provider, excludedPath, "main") + createGitTarget(mirroredTarget, testNs, provider, mirroredPath, "main") + verifyResourceCondition("gittarget", excludedTarget, testNs, "Validated", "True", "OK", "") + verifyResourceCondition("gittarget", mirroredTarget, testNs, "Validated", "True", "OK", "") + + applyRule(excludedTarget+"-rule", excludedTarget, fluxFieldManager) + applyRule(mirroredTarget+"-rule", mirroredTarget, "") + + waitForStreamsRunning(excludedTarget, testNs) + waitForStreamsRunning(mirroredTarget, testNs) + }) + + AfterAll(func() { + cleanupWatchRule(excludedTarget+"-rule", testNs) + cleanupWatchRule(mirroredTarget+"-rule", testNs) + cleanupGitTarget(excludedTarget, testNs) + cleanupGitTarget(mirroredTarget, testNs) + cleanupNamespace(testNs) + }) + + // applyConfigMap applies a ConfigMap under a named field manager, exactly as a GitOps + // tool's server-side apply does. + applyConfigMap := func(name, fieldManager, flavor string) { + GinkgoHelper() + manifest := fmt.Sprintf(`apiVersion: v1 +kind: ConfigMap +metadata: + name: %s + namespace: %s +data: + flavor: %s +`, name, testNs, flavor) + _, err := kubectlRunWithStdin(testNs, manifest, + "apply", "--server-side", "--force-conflicts", "--field-manager="+fieldManager, "-f", "-") + Expect(err).NotTo(HaveOccurred(), "apply as %s should succeed", fieldManager) + } + + // fileIn returns the ConfigMap's path in a GitTarget's folder, first repo-relative (for + // messages) and then absolute (for reads). + fileIn := func(targetPath, name string) (string, string) { + rel := path.Join(targetPath, fmt.Sprintf("%s/configmaps/%s.yaml", testNs, name)) + return rel, filepath.Join(repo.CheckoutDir, rel) + } + + eventuallyContains := func(absPath, want string) { + GinkgoHelper() + Eventually(func(g Gomega) { + pullLatestRepoState(g, repo.CheckoutDir) + content, err := os.ReadFile(absPath) + g.Expect(err).NotTo(HaveOccurred(), "expected a committed file at %s", absPath) + g.Expect(string(content)).To(ContainSubstring(want)) + }, 2*time.Minute, 2*time.Second).Should(Succeed()) + } + + It("mirrors a human's write, drops the forward leg's, and keeps mirroring the human's next edit", func() { + const cmName = "excluded-forward-leg" + excludedRel, excludedAbs := fileIn(excludedPath, cmName) + _, mirroredAbs := fileIn(mirroredPath, cmName) + + By("a human creates the ConfigMap") + applyConfigMap(cmName, humanFieldManager, "vanilla") + + By("the human's write reaches both folders") + eventuallyContains(excludedAbs, "flavor: vanilla") + eventuallyContains(mirroredAbs, "flavor: vanilla") + + By("the forward leg applies a different value back into the cluster, as it would after a Git change") + applyConfigMap(cmName, fluxFieldManager, "chocolate") + + By("the unrestricted GitTarget mirrors that apply — which proves the event was processed") + eventuallyContains(mirroredAbs, "flavor: chocolate") + + By("the excluding GitTarget did not: the forward leg's own apply is never committed back") + excludedContent, err := os.ReadFile(excludedAbs) + Expect(err).NotTo(HaveOccurred()) + Expect(string(excludedContent)).To(ContainSubstring("flavor: vanilla"), + "the forward leg's apply reached %s but must never reach %s — that is the loop this prevents", + mirroredPath, excludedRel) + Expect(string(excludedContent)).NotTo(ContainSubstring("flavor: chocolate")) + + By("a later human edit of the same object is still mirrored to the excluding target") + applyConfigMap(cmName, humanFieldManager, "strawberry") + eventuallyContains(excludedAbs, "flavor: strawberry") + + _, _ = kubectlRunInNamespace(testNs, "delete", "configmap", cmName, "--ignore-not-found=true") + }) + + // managedFields names who last WROTE an object, not who deleted it. A field-manager + // exclusion must therefore never suppress a delete, or a human deleting a Flux-managed + // resource would be silently ignored — the exact failure a label selector has. + It("still mirrors a delete of an object the excluded manager last wrote", func() { + const cmName = "deleted-after-forward-leg" + excludedRel, excludedAbs := fileIn(excludedPath, cmName) + _, mirroredAbs := fileIn(mirroredPath, cmName) + + By("a human creates it, so it reaches Git") + applyConfigMap(cmName, humanFieldManager, "vanilla") + eventuallyContains(excludedAbs, "flavor: vanilla") + + By("the forward leg applies it, becoming the object's last writer") + applyConfigMap(cmName, fluxFieldManager, "chocolate") + // Gate on the control target so the forward leg's write is known to be processed + // before the delete, rather than racing it. + eventuallyContains(mirroredAbs, "flavor: chocolate") + + By("a human deletes it") + _, err := kubectlRunInNamespace(testNs, "delete", "configmap", cmName) + Expect(err).NotTo(HaveOccurred()) + + By("the delete reaches Git even though the excluded manager wrote the object last") + Eventually(func(g Gomega) { + pullLatestRepoState(g, repo.CheckoutDir) + _, statErr := os.Stat(excludedAbs) + g.Expect(os.IsNotExist(statErr)).To(BeTrue(), + "the file must be removed at %s: managedFields names the last writer, not the deleter", excludedRel) + g.Expect(latestCommitSubjectForPath(g, repo.CheckoutDir, excludedRel)).To(ContainSubstring("[DELETE]")) + }, 2*time.Minute, 3*time.Second).Should(Succeed()) + }) +}) diff --git a/test/mutationlab/e2e/configmap_scenarios_test.go b/test/mutationlab/e2e/configmap_scenarios_test.go index 329487a7..8c6d5dd7 100644 --- a/test/mutationlab/e2e/configmap_scenarios_test.go +++ b/test/mutationlab/e2e/configmap_scenarios_test.go @@ -377,7 +377,11 @@ func TestDeletecollection(t *testing.T) { // N watch DELETED + one name-less audit deletecollection + N per-object // admission DELETE (validating admission fires once per object). - records := h.drain(t, s.id, drainSpec{minCount: 2*len(names) + 1, settle: 3 * time.Second, timeout: 60 * time.Second}) + records := h.drain( + t, + s.id, + drainSpec{minCount: 2*len(names) + 1, settle: 3 * time.Second, timeout: 60 * time.Second}, + ) if got := countSource(records, mutationlab.SourceWatch); got != len(names) { t.Errorf("deletecollection produced %d watch deletes; want %d (per-object fan-out)", got, len(names)) } diff --git a/test/mutationlab/e2e/crd_conversion_test.go b/test/mutationlab/e2e/crd_conversion_test.go index 6517650b..f92f6abd 100644 --- a/test/mutationlab/e2e/crd_conversion_test.go +++ b/test/mutationlab/e2e/crd_conversion_test.go @@ -32,8 +32,16 @@ const ( ) var ( - crdGVR = schema.GroupVersionResource{Group: "apiextensions.k8s.io", Version: "v1", Resource: "customresourcedefinitions"} - vwcGVR = schema.GroupVersionResource{Group: "admissionregistration.k8s.io", Version: "v1", Resource: "validatingwebhookconfigurations"} + crdGVR = schema.GroupVersionResource{ + Group: "apiextensions.k8s.io", + Version: "v1", + Resource: "customresourcedefinitions", + } + vwcGVR = schema.GroupVersionResource{ + Group: "admissionregistration.k8s.io", + Version: "v1", + Resource: "validatingwebhookconfigurations", + } widgetV1 = schema.GroupVersionResource{Group: widgetGroup, Version: "v1", Resource: "widgets"} widgetV2 = schema.GroupVersionResource{Group: widgetGroup, Version: "v2", Resource: "widgets"} vwcName = "gitops-reverser-validating-webhook" diff --git a/test/mutationlab/e2e/workload_scenarios_test.go b/test/mutationlab/e2e/workload_scenarios_test.go index ea047111..2c1db4a4 100644 --- a/test/mutationlab/e2e/workload_scenarios_test.go +++ b/test/mutationlab/e2e/workload_scenarios_test.go @@ -76,7 +76,9 @@ func TestStatusSubresource(t *testing.T) { ctx := context.Background() s := h.newScenario(ctx, t, "status-update") - if _, err := h.kube.AppsV1().Deployments(s.ns).Create(ctx, pausedDeployment(s, "d1"), metav1.CreateOptions{}); err != nil { + if _, err := h.kube.AppsV1(). + Deployments(s.ns). + Create(ctx, pausedDeployment(s, "d1"), metav1.CreateOptions{}); err != nil { t.Fatalf("create deployment: %v", err) } // create => admission CREATE + audit create + watch ADDED + the controller's @@ -113,7 +115,9 @@ func TestScaleSubresource(t *testing.T) { ctx := context.Background() s := h.newScenario(ctx, t, "scale-patch") - if _, err := h.kube.AppsV1().Deployments(s.ns).Create(ctx, pausedDeployment(s, "d1"), metav1.CreateOptions{}); err != nil { + if _, err := h.kube.AppsV1(). + Deployments(s.ns). + Create(ctx, pausedDeployment(s, "d1"), metav1.CreateOptions{}); err != nil { t.Fatalf("create deployment: %v", err) } h.quiesceAndClear(t, s.id, 3)