diff --git a/.coverage-baseline b/.coverage-baseline index 2ba01570..f9f577b8 100644 --- a/.coverage-baseline +++ b/.coverage-baseline @@ -1 +1 @@ -76.6 +76.7 diff --git a/.github/renovate.json b/.github/renovate.json index bd682b7d..07a70ed8 100644 --- a/.github/renovate.json +++ b/.github/renovate.json @@ -165,6 +165,38 @@ "matchStrings": ["COSIGN_VERSION=(?v[0-9.]+)"], "datasourceTemplate": "github-releases", "depNameTemplate": "sigstore/cosign" + }, + { + "customType": "regex", + "description": "devcontainer: tilt (second ENV block; release asset download)", + "managerFilePatterns": ["/^\\.devcontainer/Dockerfile$/"], + "matchStrings": ["TILT_VERSION=(?v[0-9.]+)"], + "datasourceTemplate": "github-releases", + "depNameTemplate": "tilt-dev/tilt" + }, + { + "customType": "regex", + "description": "devcontainer: delve (go install github.com/go-delve/delve/cmd/dlv)", + "managerFilePatterns": ["/^\\.devcontainer/Dockerfile$/"], + "matchStrings": ["DLV_VERSION=(?v[0-9.]+)"], + "datasourceTemplate": "go", + "depNameTemplate": "github.com/go-delve/delve" + }, + { + "customType": "regex", + "description": "devcontainer: gopls (go install golang.org/x/tools/gopls; its own module, versioned apart from x/tools)", + "managerFilePatterns": ["/^\\.devcontainer/Dockerfile$/"], + "matchStrings": ["GOPLS_VERSION=(?v[0-9.]+)"], + "datasourceTemplate": "go", + "depNameTemplate": "golang.org/x/tools/gopls" + }, + { + "customType": "regex", + "description": "devcontainer: staticcheck (go install honnef.co/go/tools/cmd/staticcheck)", + "managerFilePatterns": ["/^\\.devcontainer/Dockerfile$/"], + "matchStrings": ["STATICCHECK_VERSION=(?v[0-9.]+)"], + "datasourceTemplate": "go", + "depNameTemplate": "honnef.co/go/tools" } ] } diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e046c378..037a7024 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,8 +11,16 @@ name: CI on: + # Every pull request, whatever its base — deliberately NOT filtered to `branches: [main]`. + # A stacked PR (PR N based on PR N-1's branch) never matches a main-only filter, so it + # merged with no lint, no unit tests, and no e2e: PR #255 ran only the external review + # bots. Retargeting does not rescue it either — when the base branch merges, GitHub + # retargets the open PR onto main and fires `pull_request` with action `edited`, which + # is not in the default trigger set (opened/synchronize/reopened), so nothing re-runs. + # Validating every base costs a little runner time and is what makes a stack reviewable + # one PR at a time. See docs/design/watchrule-source-namespace/README.md for the stack + # this was found on. pull_request: - branches: [main] workflow_call: outputs: ci-image: diff --git a/AGENTS.md b/AGENTS.md index 9d0c87e6..7831d82a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -7,6 +7,12 @@ - **Skip conversational phrases** - No "Great!", "Certainly!", "Okay!" - get straight to the point - **One clear message per concept** - Don't repeat the same information in different ways +## PULL REQUESTS + +- Use a concise Conventional Commit-style PR title: `type(scope): imperative summary`, or omit the scope + when none adds useful context. +- Describe the final branch state, not the sequence of commits used to reach it. + ## MANDATORY PRE-COMPLETION VALIDATION **CRITICAL**: These commands MUST pass before any implementation is considered complete: diff --git a/README.md b/README.md index b6f7c45b..4aae73eb 100644 --- a/README.md +++ b/README.md @@ -74,6 +74,11 @@ actor — user, service account, or CI identity — while the committer never mo kube-apiserver audit delivery (managed control planes like EKS/GKE/AKS generally do not expose it) plus Valkey/Redis: see the [attribution setup guide](docs/attribution-setup-guide.md). +If attribution is enabled but a live change has no usable audit fact, its author is explicitly +`unknown (attribution unresolved)`, not the configured committer. For a change that should have a named +actor, treat that identity as a signal to verify the audit policy, webhook route, source identity, and +Redis connectivity. + **Valkey/Redis is optional but advised.** Without it the default mode works fine; adding it unlocks warm-restart cursors, `CommitRequest` author capture, and attribution. diff --git a/Taskfile.yml b/Taskfile.yml index c8e992a2..02ac12fc 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -21,7 +21,7 @@ includes: dir: . flatten: true e2e: - taskfile: ./test/e2e/Taskfile.yml + taskfile: ./test/e2e/Taskfile-e2e.yml dir: . flatten: true diff --git a/api/v1alpha3/commitrequest_types.go b/api/v1alpha3/commitrequest_types.go index dc0e9c56..d69082cc 100644 --- a/api/v1alpha3/commitrequest_types.go +++ b/api/v1alpha3/commitrequest_types.go @@ -32,12 +32,12 @@ type CommitRequestSpec struct { Message string `json:"message,omitempty"` // CloseDelaySeconds optionally delays closing the open commit window for this - // many seconds after the CommitRequest is attributed, acting as an extra collect - // window: changes the author makes in the meantime still join the open commit - // window and are included in the resulting commit. Omitted or 0 closes the window - // as soon as the CommitRequest is attributed to its author. The window can still - // be closed earlier by another author's change or by the provider's commit window - // timer, exactly as without a CommitRequest. + // many seconds after the CommitRequest attaches to a matching open window, acting as + // an extra collect window: matching changes that arrive in the meantime still join + // that window and are included in the resulting commit. Omitted or 0 closes the + // window as soon as the CommitRequest attaches. The window can still be closed + // earlier by another author's change or by the provider's commit window timer, + // exactly as without a CommitRequest. // +optional // +kubebuilder:validation:Minimum=0 // +kubebuilder:validation:Maximum=300 @@ -55,9 +55,10 @@ type CommitRequestSpec struct { // while finalizing; Stalled=True when the finalize failed and needs attention. // - AuthorAttributed (domain): binary and settled immediately. True // (AttributedFromAdmission) when the submitter captured at admission named the -// commit author; False (CommitterFallback) when no admission record exists — the -// validate-operator-types webhook is not configured — and the commit is authored by the -// configured committer. False is not a failure and does not affect Ready. +// commit author; False (CommitterFallback) when capture ran but no admission record +// exists, or False (AuthorCaptureDisabled) when capture is disabled. In either False +// case the request claims no actor. False is not a failure and does not affect Ready; +// the final Git author remains the matching watch window's author. // - Pushed (domain): True once the commit is in the remote repository. type CommitRequestStatus struct { // ObservedGeneration is the most recent generation observed by the controller. diff --git a/charts/gitops-reverser/templates/validate-operator-types-webhook.yaml b/charts/gitops-reverser/templates/validate-operator-types-webhook.yaml index 1ca2408a..0fd57a40 100644 --- a/charts/gitops-reverser/templates/validate-operator-types-webhook.yaml +++ b/charts/gitops-reverser/templates/validate-operator-types-webhook.yaml @@ -4,8 +4,8 @@ # at admission, into the command-author Redis corner the controller reads back with no # wait. Narrow by construction: one rule per command kind. failurePolicy is Ignore and # the handler always allows, so a user's CommitRequest never depends on this webhook — -# a miss degrades to a committer-authored commit (AuthorAttributed=False). -# See docs/spec/commitrequest-admission-authorship.md §4. +# a miss leaves the request without a claimed actor (AuthorAttributed=False). +# See docs/spec/commitrequest-admission-authorship.md. apiVersion: admissionregistration.k8s.io/v1 kind: ValidatingWebhookConfiguration metadata: @@ -26,7 +26,7 @@ webhooks: namespace: {{ .Release.Namespace }} path: /validate-operator-types port: {{ .Values.servers.admission.port }} - # Ignore, not Fail: a missed author capture must degrade to the committer, never + # Ignore, not Fail: a missed author capture must leave the request unnamed, never # reject a user's CommitRequest. failurePolicy: Ignore matchPolicy: Equivalent diff --git a/charts/gitops-reverser/values.yaml b/charts/gitops-reverser/values.yaml index 5c94ac64..e8bab7e6 100644 --- a/charts/gitops-reverser/values.yaml +++ b/charts/gitops-reverser/values.yaml @@ -102,16 +102,16 @@ servers: secretNameOverride: "" # Validating admission server hosting the validate-operator-types webhook, which captures - # the submitter of each CommitRequest at admission and names them as the commit author + # the submitter of each CommitRequest at admission and records it as the request's named actor # (read back by the controller with no wait). Independent of `attribution` below: # attribution governs *mirrored-resource* authorship (from audit), this governs - # *command* authorship (from admission). Off → CommitRequests commit as the configured - # committer and report AuthorAttributed=False. See + # *command* authorship (from admission). Off → CommitRequests claim no actor and report + # AuthorAttributed=False. See # docs/spec/commitrequest-admission-authorship.md. # # Command-author capture is Redis-backed: the captured author is written to Redis at admission and # read back at reconcile. Enabled by default so the webhook is always installed, but it is a no-op - # without Redis: the pod runs healthy and CommitRequests commit as the configured committer + # without Redis: the pod runs healthy and CommitRequests claim no actor # (AuthorAttributed=False) until queue.redis.addr is set. admission: enabled: true diff --git a/cmd/main.go b/cmd/main.go index cc2c4cc5..de2b5fb7 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -312,11 +312,11 @@ func main() { // lives in its own Redis corner (author:v1:command), independent of // --author-attribution (which governs mirrored-resource attribution). It is wired // whenever the admission server is on; the controller reads the captured submitter - // back with no wait (docs/spec/commitrequest-admission-authorship.md §2, §6). + // back with no wait (docs/spec/commitrequest-admission-authorship.md). // // AuthorLookup must be a nil interface — not a non-nil interface wrapping a nil // *CommandAuthorStore — when the webhook is off, so the controller's nil check - // selects finalize-as-committer immediately (AuthorAttributed=False) instead of + // selects the no-actor path immediately (AuthorAttributed=False) instead of // dereferencing a nil store. var commandAuthorStore *queue.CommandAuthorStore var commandAuthorLookup controller.CommandAuthorLookup @@ -325,10 +325,10 @@ func main() { commandAuthorStore = redisStore.CommandAuthorStore() commandAuthorLookup = commandAuthorStore setupLog.Info("validate-operator-types webhook enabled: command submitters are captured at admission " + - "and named as the commit author") + "and recorded as the request's named actor") } else { setupLog.Info("validate-operator-types webhook enabled without Redis: command author capture is a " + - "no-op; CommitRequests commit as the configured committer (AuthorAttributed=False). " + + "no-op; CommitRequests claim no actor (AuthorAttributed=False). " + "Set --redis-addr to capture command authors.") } } @@ -478,7 +478,7 @@ func parseFlagsWithArgs(fs *flag.FlagSet, args []string) (appConfig, error) { "and, when author attribution is enabled, the attribution facts. Leave empty to run without "+ "Redis: watches cold-replay on restart instead of resuming. Required by "+ "--author-attribution=true. --admission-webhook still runs without it, but command-author "+ - "capture is a no-op: CommitRequests commit as the configured committer.") + "capture is a no-op: CommitRequests claim no actor.") fs.BoolVar(&cfg.authorAttribution, "author-attribution", true, "Name the real actor (human or service account) who caused each change as the Git commit author, "+ "resolved from matching audit facts; this runs the audit webhook ingress (default true). When "+ @@ -661,7 +661,7 @@ func validateAdmissionWebhookConfig(cfg appConfig) error { return nil } // No redis-addr requirement here: the admission webhook stays enabled without Redis and - // simply no-ops command-author capture (commits fall back to the committer). Redis is + // simply no-ops command-author capture (CommitRequests claim no actor). Redis is // wired only when configured; see the commandAuthorStore setup. if _, _, err := splitBindAddress(cfg.admissionWebhookBindAddress); err != nil { return fmt.Errorf("invalid admission-webhook-bind-address %q: %w", cfg.admissionWebhookBindAddress, err) diff --git a/config/crd/bases/configbutler.ai_commitrequests.yaml b/config/crd/bases/configbutler.ai_commitrequests.yaml index 60f2e22a..acd2f841 100644 --- a/config/crd/bases/configbutler.ai_commitrequests.yaml +++ b/config/crd/bases/configbutler.ai_commitrequests.yaml @@ -73,12 +73,12 @@ spec: closeDelaySeconds: description: |- CloseDelaySeconds optionally delays closing the open commit window for this - many seconds after the CommitRequest is attributed, acting as an extra collect - window: changes the author makes in the meantime still join the open commit - window and are included in the resulting commit. Omitted or 0 closes the window - as soon as the CommitRequest is attributed to its author. The window can still - be closed earlier by another author's change or by the provider's commit window - timer, exactly as without a CommitRequest. + many seconds after the CommitRequest attaches to a matching open window, acting as + an extra collect window: matching changes that arrive in the meantime still join + that window and are included in the resulting commit. Omitted or 0 closes the + window as soon as the CommitRequest attaches. The window can still be closed + earlier by another author's change or by the provider's commit window timer, + exactly as without a CommitRequest. format: int32 maximum: 300 minimum: 0 diff --git a/config/deployment.yaml b/config/deployment.yaml index af5667c2..a03127ea 100644 --- a/config/deployment.yaml +++ b/config/deployment.yaml @@ -34,6 +34,16 @@ spec: - --redis-addr=valkey.valkey-e2e.svc.cluster.local:6379 # The e2e Valkey serves plain TCP, so opt out of the now-default Redis TLS. - --redis-insecure + # Attribution waits this long for a matching audit fact before committing with the explicit + # `unknown (attribution unresolved)` author. Widened from the 3s default to buy headroom + # over the apiserver's --audit-webhook-batch-max-wait=1s + # (test/e2e/cluster/start-cluster.sh): measured fact delivery is ~0.8-1.1s even on an + # idle cluster, so 3s leaves only ~2s of slack for a 4-proc run. + # + # This is not a substitute for correct audit attribution configuration. A resolution that + # still ends as `absent` exposes an audit policy, route, source-identity, Redis, or timing + # issue for the change being exercised; the after-suite report keeps that distinction visible. + - --author-attribution-grace=10s # Dev/e2e only: the in-cluster Gitea SSH host key is not pinned, so permit SSH # when no known_hosts source exists. Never set this in production. - --insecure-allow-missing-known-hosts diff --git a/config/webhook/validating-webhook.yaml b/config/webhook/validating-webhook.yaml index b4b96ea8..2d0b2c17 100644 --- a/config/webhook/validating-webhook.yaml +++ b/config/webhook/validating-webhook.yaml @@ -20,7 +20,7 @@ metadata: # kinds (a CommitRequest today) at admission. This one IS product behavior and is # packaged in the chart (charts/.../templates/validate-operator-types-webhook.yaml); the # entry here is the e2e SUT's copy. See -# docs/spec/commitrequest-admission-authorship.md §4. +# docs/spec/commitrequest-admission-authorship.md. # # The cert-manager.io/inject-ca-from annotation injects the admission server CA bundle # for every webhook in this configuration. diff --git a/docs/INDEX.md b/docs/INDEX.md index 877eb992..ce92c857 100644 --- a/docs/INDEX.md +++ b/docs/INDEX.md @@ -1,10 +1,8 @@ # Index: what to read, and what is history -> Rebuilt 2026-07-11, when the tree went from 180 documents to 117. - -There are 136 tracked markdown files here. **About 35 of them bind.** This page names -those. If a document is not on this page, it is either a user guide (see -[`README.md`](README.md)) or history you can safely not read. +This page names the documents that bind the current implementation. If a document is not +listed here, it is either a user guide (see [`README.md`](README.md)) or historical context +you can safely skip. ## The four folders, and what each one means @@ -53,7 +51,7 @@ misled. Full list in [`spec/README.md`](spec/README.md); the ones that carry a | [`gitpath-foreign-content-stringency.md`](spec/gitpath-foreign-content-stringency.md) | refusing a path that shadows foreign content | | [`unsupported-folder-refusal-plan.md`](spec/unsupported-folder-refusal-plan.md) | `GitPathAccepted`, and refusing what we cannot own | | [`commitrequest-design.md`](spec/commitrequest-design.md) | the CommitRequest window and its conditions | -| [`commitrequest-admission-authorship.md`](spec/commitrequest-admission-authorship.md) | how a real Kubernetes user becomes a commit author | +| [`commitrequest-admission-authorship.md`](spec/commitrequest-admission-authorship.md) | how command submitters are captured and matched to commit windows | | [`where-validation-lives.md`](spec/where-validation-lives.md) | schema → CEL → **the reconciler**; a webhook only for what exists solely at admission | | [`e2e-serial-registry.md`](spec/e2e-serial-registry.md) | which e2e specs must run Serial, and why | @@ -78,7 +76,6 @@ Eleven other open items: | [`release-image-reuse-plan.md`](design/release-image-reuse-plan.md) | PRs 2–5 unstarted | | [`e2e-coverage-gaps-and-improvements-plan.md`](design/e2e-coverage-gaps-and-improvements-plan.md) | tests A/B/C still proposals | | [`e2e-finish-plan.md`](design/e2e-finish-plan.md) | remaining e2e harness work | -| [`residual-e2e-flakes-2026-06-19.md`](design/residual-e2e-flakes-2026-06-19.md) | Flake B still open | | [`sensitive-resource-diagnostics-follow-up.md`](design/sensitive-resource-diagnostics-follow-up.md) | deferred diagnostics | | [`e2e-git-server-choice.md`](design/e2e-git-server-choice.md) | stay on Gitea or move to Forgejo — the `_csrf` pin is fixable in place on both, so the migration is now a preference call, not a fix; also why we adopt no SDK either way | | [`watchrule-source-namespace/`](design/watchrule-source-namespace/README.md) | letting a WatchRule address a differently-named namespace on its source cluster — a deny-by-default `allowedSourceNamespaces` on the **GitTarget** (so scope is per-tenant, not a provider-wide union), unlocked by a false-by-default delegation flag on the ClusterProvider. Split into five implementable PRs: three prerequisite scope fixes (the namespace-blind resync sweep that would delete other namespaces' manifests, the cluster-wide/named stream collapse, and ClusterWatchRule's unchecked GitTarget attachment), the field and its gate, and the ceiling that makes the allow-list bind ClusterWatchRule too — required because a multi-tenant deployment runs a ClusterWatchRule per tenant from day one to capture CRDs | diff --git a/docs/TODO.md b/docs/TODO.md index 43b6bdc0..5914d2ae 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -41,8 +41,8 @@ This file is meant to track the smaller current backlog, not historical notes. - [ ] Fix recurring full e2e flakiness around WatchRule/snapshot convergence. This has shown up more than once as timeout-based failures in manager SOPS bootstrap and - signing snapshot-message specs, then passed on rerun. Capture and mitigation notes live in - [docs/design/e2e-watchrule-cross-spec-interference.md](spec/e2e-serial-registry.md). + signing snapshot-message specs, then passed on rerun. Capture and mitigation notes live in the + [e2e serial registry](spec/e2e-serial-registry.md). This should be addressed before the next feature that expands commit-message, snapshot, or write-window behavior, otherwise new failures will be hard to separate from existing timing debt. diff --git a/docs/UPGRADING.md b/docs/UPGRADING.md index da2b4381..9a2a2a4e 100644 --- a/docs/UPGRADING.md +++ b/docs/UPGRADING.md @@ -7,6 +7,29 @@ guidance that the changelog's breaking-change entries link to. We are pre-1.0, so breaking changes bump the **minor** version (release-please is configured with `bump-minor-pre-major`) rather than the major. Read the relevant entry before upgrading across it. +## Unreleased — unresolved attribution is visible in Git (next minor; author identity changes) + +When `attribution.enabled=true` and the operator cannot match a live watch event to an audit fact within +`attribution.grace`, the Git author is now: + +```text +unknown (attribution unresolved) +``` + +Previously, this case used the configured committer identity, making an attribution miss +indistinguishable from configured-author mode. The Git **committer** remains the configured operator +identity in both cases. + +Configured-author mode is unchanged: when attribution is disabled, the configured committer is still both +the Git author and committer. A real Kubernetes actor is unchanged too. Only a live event for which +attribution ran but did not resolve now has the explicit author above. + +Update any automation that treats `GitOps Reverser` as meaning "attribution was not configured". Monitor +`gitopsreverser_commits_total{author_kind="unresolved"}` after the upgrade. For a live mutation that should +be attributable, this author normally means the audit policy, webhook route, source identity, Redis +connectivity, or grace window is not configured correctly. It can also represent a change with no usable +audit actor, so use the audit metrics before assigning a cause. + ## Unreleased — a `patches:` block no longer refuses the folder (next minor; more folders accepted) A kustomization declaring `patches:` used to refuse the whole `GitTarget`. Not the edit — the @@ -344,9 +367,10 @@ The Helm chart now defaults to the simple, Redis-free `configured-author` path, to `""` (was `valkey-auth`). Without a Redis endpoint the operator runs `configured-author` and watches cold-replay on restart. - `servers.admission.enabled` stays `true` by default, but the validate-operator-types admission - webhook no longer requires Redis. Without `queue.redis.addr` it runs as a no-op (CommitRequests - commit as the configured committer, `AuthorAttributed=False`); it captures authors once Redis is - configured. Previously enabling admission without Redis failed startup. + webhook no longer requires Redis. Without `queue.redis.addr` it runs as a no-op: CommitRequests + claim no actor (`AuthorAttributed=False`), while the Redis-free installation's matching windows are + configured-author. It captures submitters once Redis is configured. Previously enabling admission + without Redis failed startup. - The chart still rejects one invalid combination at render time: `attribution.enabled=true` without `queue.redis.addr` fails `helm install`/`upgrade` with an actionable message (attributed-author mode cannot run without Redis) instead of crash-looping the pod. @@ -459,8 +483,9 @@ kubectl get commitrequest/ -n -o jsonpath='{.status.sha}' ``` `AuthorAttributed=True` with reason `AttributedFromAdmission` means the internal commands admission -webhook captured the submitter. `AuthorAttributed=False` with reason `CommitterFallback` is a valid -fallback, not a failed request. +webhook captured the submitter. `AuthorAttributed=False` with reason `CommitterFallback` means capture +ran but found no record; `AuthorCaptureDisabled` means capture is not configured. Neither is a failed +request. ### 4. `GitTarget.status.phase` and materialization rollups moved to stream conditions diff --git a/docs/architecture.md b/docs/architecture.md index f56fabc0..85bb65e2 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -90,8 +90,10 @@ The solution, in the vocabulary used throughout this document: Older APIs that reject `sendInitialEvents` fall back to LIST plus buffered WATCH. The sweep fires on snapshot establishment, **never on a timer**. * **Audit, when enabled, only names the author.** It is an optional attribution lookup; a missing or - late fact costs author fidelity, never correctness, and with attribution disabled the product commits - as the configured committer. + late fact costs author fidelity, never correctness. With attribution disabled the product commits as the + configured committer. With attribution enabled, an unresolved live change is visibly authored as + `unknown (attribution unresolved)`, so an installation that expects attribution can investigate its audit + policy, ingress, Redis connection, or source identity instead of mistaking the result for configured-author mode. * **One `BranchWorker` per Git branch serializes all writes.** Every write to a branch funnels through a single worker and a single commit window, which keeps concurrent GitTargets and authors from racing each other into a corrupt tree. @@ -224,8 +226,9 @@ instead of waiting for the silence timer. The **entire spec is immutable**. Key * `status.conditions`: kstatus-compatible. **Ready** is the summary (True once the request reached a terminal outcome that is not an error — a pushed commit, or a benign no-commit); **Reconciling**/**Stalled** are the kstatus progress/blocked pair; **AuthorAttributed** reports - whether the `/validate-operator-types` validating admission webhook named the submitter or the request - fell back to the configured committer; **Pushed** reports whether the commit reached the remote. The `Ready` + whether the `/validate-operator-types` validating admission webhook named the submitter; an absent record + means the request claims no actor and can attach only to an unnamed window. **Pushed** reports whether the + commit reached the remote. The `Ready` condition's `reason` carries `Committed`, `NoWindowInGrace`, `WindowMismatch`, `AlreadyPresent`, or `FinalizeFailed`. A benign no-commit (e.g. `NoWindowInGrace`) is `Ready=True`, `Stalled=False` — a correct, non-error outcome — whereas a `FinalizeFailed` is `Ready=False`, `Stalled=True`. @@ -368,9 +371,10 @@ Following the ConfigMap edit: whose desired-state projection is unchanged) is dropped here. 3. **Resolve the author.** When attribution is enabled, the resolver waits a bounded grace window (`--author-attribution-grace`, default `3s`) for a matching audit fact in the attribution index, joining by - resourceVersion/UID. On a strong match the real user or named service account becomes the author; - otherwise (attribution off, no match, or expiry) the commit is authored by the configured committer. This - wait is per-event and never reorders a watch — see [Watch Event Ordering](#watch-event-ordering). + resourceVersion/UID. On a strong match the real user or named service account becomes the author. With + attribution disabled, the configured committer is the author; with attribution enabled but no usable fact, + the explicit `unknown (attribution unresolved)` author marks the unresolved change. This wait is per-event + and never reorders a watch — see [Watch Event Ordering](#watch-event-ordering). 4. **Route + window.** The event flows into the [GitTargetEventStream](../internal/reconcile/git_target_event_stream.go), and the [BranchWorker](../internal/git/branch_worker.go) appends it to the open commit window for @@ -403,15 +407,17 @@ flowchart TD FILTER --> AUTHOR{Attribution enabled?} AUTHOR -->|yes| AUDIT[Wait bounded grace for audit fact] AUTHOR -->|no| COMMITTER[Use configured committer as author] - AUDIT --> RESOLVED[Resolved author or committer fallback] - COMMITTER --> RESOLVED - RESOLVED --> WINDOW[BranchWorker open window
one author + one GitTarget] + AUDIT -->|fact matched| RESOLVED[Use resolved actor] + AUDIT -->|no usable fact| UNRESOLVED[Use explicit unresolved author] + COMMITTER --> WINDOW[BranchWorker open window
one author + one GitTarget] + RESOLVED --> WINDOW + UNRESOLVED --> WINDOW WINDOW --> TIMER{Close trigger} TIMER -->|silence timer| FLUSH[Plan YAML edits + commit] TIMER -->|buffer limit or resync boundary| FLUSH - CR[CommitRequest persisted] --> ADMISSION[Admission author cache lookup
present or committer fallback] + CR[CommitRequest persisted] --> ADMISSION[Admission submitter lookup
named or unnamed] ADMISSION --> ATTACH[Attach to matching open window] ATTACH -->|same author + GitTarget| FLUSH ATTACH -->|no matching window| BENIGN[Ready=True, Pushed=False] @@ -568,8 +574,10 @@ Attribution being optional does **not** make it casual or take-it-or-leave-it. W operator does everything it can to name the real actor — but it values **high certainty over a plausible guess**. Attaching a name to a change is a consequential, sometimes politically charged claim ("*this* person did *that*"), so it must not be made lightly. The design therefore fails toward honesty: a weak, -conflicting, late, or absent fact resolves to the committer rather than to a guessed author. **It is better -to clearly record that the operator made a change than to assert an actor we are not sure of.** +conflicting, late, or absent fact produces an explicit unresolved author rather than a guessed person or a +misleading committer identity. **It is better to say that attribution did not resolve than to assert an +actor we are not sure of.** In an installation that expects live mutations to be attributable, this outcome +is a concrete signal to check audit delivery and attribution configuration. Two engineering choices follow directly from that stance: @@ -580,7 +588,7 @@ Two engineering choices follow directly from that stance: * **Tests pin the behavior.** Because a misattribution is a real harm, the attribution and resolver paths carry unit and e2e tests that prove the concrete cases — strong match, weak/last-key match, deletes whose audit RV differs from the watch RV, missing/late/expired facts, service-account vs human actor, - impersonation, and the committer fallback — so these certainty guarantees cannot silently regress. + impersonation, and the explicit unresolved outcome — so these certainty guarantees cannot silently regress. ### Attribution fact shape @@ -608,8 +616,9 @@ A watch event waits a **bounded grace window** (`--author-attribution-grace`, de to arrive, then ships regardless. On a strong match the actor becomes the author — a human or a service account alike, always named by its own username (e.g. `system:serviceaccount:flux-system:kustomize-controller`). A weak, conflicting, missing, or expired fact -resolves to the committer. A late fact that arrives after a commit has shipped **never rewrites it**. -With attribution disabled the resolver is absent and every commit is committer-authored. +produces `unknown (attribution unresolved) ` instead of a +guessed actor. A late fact that arrives after a commit has shipped **never rewrites it**. With attribution +disabled the resolver is absent and every commit is committer-authored. The CommitRequest controller does **not** read this audit index. A CommitRequest's submitter is named by the `/validate-operator-types` validating admission webhook instead — captured synchronously at admission, @@ -625,12 +634,21 @@ and the *author* (who wrote the change) — and GitOps Reverser uses both on pur to `GitOps Reverser `. Every commit, attributed or not, is committed by the operator, because the operator is what actually wrote it to Git. * The **author is the real actor — but only when we are sure.** On a strong attribution match the author is - set to that actor; with no confident match the author is set to the operator too. Git always carries an - author, so it is **never left blank** — when we are not sure it is simply the operator (identical to the - committer), never a guessed person. A commit whose author differs from the committer is therefore a - positive statement that the operator is confident who made the change. - -The author identity is never invented — it is taken from the authenticated request. The name and email come + set to that actor. Git always carries an author, so it is **never left blank**, and it is never a guessed + person. What fills it when we are not sure depends on WHY: + * **Attribution is switched off** (configured-author mode, and reconcile/resync writes that have no actor + at all): the author is the operator, identical to the committer. That is honest — the operator really is + the author. + * **Attribution ran and did not resolve an actor**: the author is the explicit sentinel + `unknown (attribution unresolved) `. It is deliberately + NOT the operator, because authoring it as the operator made a lost actor byte-identical to a + configured-author commit — so the gap was invisible in Git history and only countable in a metric. + + A commit whose author is a **person** is therefore a positive statement that the operator is confident who + made the change; one authored by the sentinel is a positive statement that it tried and could not tell. + +The author identity is never fabricated as a person — a real author is always taken from the authenticated +request, and an unresolved one is labelled as unresolved rather than attributed to anybody. The name and email come from the **OIDC claims** when the apiserver maps them; otherwise they come from the actor's own **Kubernetes identity** — the username, which for a controller is its service account (`system:serviceaccount::`), with a stable derived email under `noreply.cluster.local`. @@ -800,8 +818,8 @@ namespaces. Desired state comes from one **raw watch per `(GitTarget, GVR, scope)`**. Each event is sanitized, diffed against current Git content, and applied — there is no separate per-type object store to reconstruct, -because Git already holds current state. The authoritative design is -[design/watch-first-ingestion-architecture.md](finished/watch-first-ingestion-architecture.md). +because Git already holds current state. This section is the authoritative contract; the +[watch-first design record](finished/watch-first-ingestion-architecture.md) is historical context. * **Manager**: [internal/watch/manager.go](../internal/watch/manager.go) * **Watch / replay / sweep**: [internal/watch/target_watch.go](../internal/watch/target_watch.go) @@ -1009,14 +1027,14 @@ counted in the resync summary (`placementSkipped`) — rather than written unsaf A `CommitRequest` finalizes the open commit window for its GitTarget. The request author is resolved from the `/validate-operator-types` validating admission webhook, which captures the authenticated submitter at admission (keyed by the object's UID) before the object is visible. The lookup is **present-or-never**: if -the record is missing, the request **finalizes as the configured committer** with `AuthorAttributed=False`, -and that fallback does not fail the request: +the record is missing, the request is marked `AuthorAttributed=False`, claims no actor, and still attaches +immediately. That is not a failure: 1. The controller stamps the in-progress conditions (`Reconciling=True`) and settles `AuthorAttributed` synchronously from the admission author cache. There is no audit wait on this path. 2. The controller eagerly **attaches** the request to the worker (`AttachCommitRequest`), anchoring the - finalize at `receipt + closeDelaySeconds`. The worker binds it to an open window only when author and - GitTarget match. It **never finalizes another author's window**; a window carries at most one request. + finalize at `receipt + closeDelaySeconds`. The worker binds it to an open window only when the author + state and GitTarget match. It **never finalizes another author's window**; a window carries at most one request. 3. The window finalizes on the deadline (or when it closes for any other reason). If a finalize closes an open window, the worker always schedules a push, so a window closed by an otherwise no-op resync is not stranded. @@ -1025,8 +1043,10 @@ and that fallback does not fail the request: `Pushed=False`; a failure sets `Ready=False` / `Stalled=True` with a message. The CommitRequest submitter is not recoverable from object state alone, so without an admission record the -finalize is committer-authored. There is no audit-fact join on this path: the submitter is captured at -admission by the validating webhook, not derived from the audit attribution index. +request cannot claim an actor. The final Git author remains the attached watch window's author: configured +committer when attribution was disabled, or the explicit unresolved author when live attribution ran but +could not resolve. There is no audit-fact join for the request itself: its submitter is captured at admission +by the validating webhook, not derived from the audit attribution index. *** @@ -1047,7 +1067,7 @@ Controllers watch their dependencies so dependents reconcile quickly after spec * `CommitRequestReconciler` runs with `MaxConcurrentReconciles=1` and attributes/attaches as above; its optional `AuthorLookup` is the command-author cache populated by the `/validate-operator-types` webhook (wired whenever the admission webhook is enabled — independent of `--author-attribution` — and nil - otherwise, so the request finalizes as the committer). + otherwise, so the request claims no actor). Dependency watches use narrow predicates to avoid status-only heartbeat churn: a `ClusterProvider` also admits a `Ready` transition, and a `Namespace` admits only label changes. `GitProvider`, `GitTarget`, and @@ -1109,8 +1129,8 @@ Metrics are exported over OTLP / the metrics server. The audit-attribution path (read by the restart-reconcile guarantee); * resync/background-apply failure counters so a silently-recovered fault stays visible. -Per-watch volume/restart/replay metrics and per-attribution result/wait histograms are designed -([watch-first metrics](finished/watch-first-ingestion-architecture.md#metrics)) but **not yet emitted**; see +Per-watch volume/restart/replay metrics and per-attribution result/wait histograms are designed in the +[metrics observability plan](design/metrics-observability-plan.md) but **not yet emitted**; see [Operational Boundaries](#operational-boundaries). *** @@ -1174,8 +1194,8 @@ Deeper dives live under [docs/design/](design/): **Ingestion, attribution, and reconcile:** -* [Watch-first ingestion architecture](finished/watch-first-ingestion-architecture.md) — the authoritative - model: watch is the only object-state source, audit is optional attribution. +* [Watch-first ingestion design record](finished/watch-first-ingestion-architecture.md) — historical context + for the current watch-only object-state model and optional audit attribution. * [Watch event ordering under the attribution grace window](facts/watch-event-ordering-and-attribution-grace.md) * [Reconcile via watchlist mark and sweep](spec/reconcile-via-watchlist-mark-and-sweep.md) * [CommitRequest design](spec/commitrequest-design.md) diff --git a/docs/attribution-setup-guide.md b/docs/attribution-setup-guide.md index ae3b9f12..404bddc3 100644 --- a/docs/attribution-setup-guide.md +++ b/docs/attribution-setup-guide.md @@ -5,6 +5,12 @@ configured committer identity. **Attribution** turns on a second identity — th user or service account that made the change becomes the commit *author*, while the committer stays constant. Git carries both. +Once attribution is on, a change whose actor cannot be resolved is authored +`unknown (attribution unresolved) ` rather than falling +back to the committer. Seeing that identity in your history means attribution ran and did not find a +matching audit fact — check the audit webhook path before assuming it is normal. Its rate is +`commits_total{author_kind="unresolved"}`. + Attribution is the only optional capability, and it is a real operational step up: it requires kube-apiserver audit delivery and a Redis/Valkey backing store. This guide covers what that costs and how the pieces fit. It assumes GitOps Reverser is already installed and mirroring state — see @@ -27,6 +33,7 @@ With `attribution.enabled`, only the **author** column moves: | Human user (`simon@example.com`) | Simon | `ConfigButler Bot` | | Service account (`system:serviceaccount:team-a:deployer`) | the `team-a/deployer` service account | `ConfigButler Bot` | | CI / GitHub App identity | that CI / App identity | `ConfigButler Bot` | +| No usable audit fact for a live change | `unknown (attribution unresolved)` | `ConfigButler Bot` | The committer column never moves. That is the point. @@ -149,14 +156,16 @@ Audit facts and watch events arrive on independent paths, so the controller join wait rather than blocking: - **`attribution.grace`** (default `3s`) — how long a watch event waits for a matching audit fact - before it ships authored by the committer. Larger raises the attribution hit-rate at the cost of - commit latency. + before it ships with the explicit unresolved author. Larger raises the attribution hit-rate at the cost + of commit latency. - **`attribution.ttl`** (default `10m`) — how long an unmatched audit fact is retained waiting for its watch event. Attribution is opportunistic: on a strong match the named user or service account is the author; with -no match in the grace window, the commit still lands, authored by the committer. No change is dropped -for lack of a match. +no match in the grace window, the commit still lands, authored as +`unknown (attribution unresolved)`. No change is dropped for lack of a match. For a live mutation that +should be attributable, treat this author as an audit-attribution configuration or delivery problem until +the audit policy, webhook route, source identity, and Redis connectivity are verified. ## Verifying and reverting diff --git a/docs/ci-overview.md b/docs/ci-overview.md index edf6cd97..bf1ed753 100644 --- a/docs/ci-overview.md +++ b/docs/ci-overview.md @@ -150,7 +150,7 @@ GitHub's OIDC identity and logged in the public Rekor transparency log. | Concern | Lives in | | --- | --- | -| What gets checked (lint, unit, e2e, packaging) | [Taskfile-build.yml](../Taskfile-build.yml), [test/e2e/Taskfile.yml](../test/e2e/Taskfile.yml) — see [tasks-overview.md](tasks-overview.md) | +| What gets checked (lint, unit, e2e, packaging) | [Taskfile-build.yml](../Taskfile-build.yml), [test/e2e/Taskfile-e2e.yml](../test/e2e/Taskfile-e2e.yml) — see [tasks-overview.md](tasks-overview.md) | | Tool versions | [.devcontainer/Dockerfile](../.devcontainer/Dockerfile) (single source for devcontainer *and* CI) | | Validation pipeline | [.github/workflows/ci.yml](../.github/workflows/ci.yml) | | Release pipeline | [.github/workflows/release.yml](../.github/workflows/release.yml) | diff --git a/docs/configuration.md b/docs/configuration.md index a59d2b73..8b1052a9 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -171,6 +171,12 @@ For mirrored-resource commits, the author comes from the configured committer id `attribution.enabled=true` and a matching kube-apiserver audit event names the Kubernetes user or service account. Snapshot/reconcile commits are operator-authored. +When attribution IS enabled and no matching audit fact arrives, the commit is authored +`unknown (attribution unresolved) ` — **not** the +committer. That distinction is the point: a committer-authored commit means attribution was never +attempted, while the sentinel means it was attempted and did not resolve, which is worth investigating. +Such commits also count under `author_kind="unresolved"` in `commits_total`. + That distinction is useful in practice: - `git log --author=alice` answers "what did Alice change?" @@ -230,6 +236,12 @@ spec: - `Username` - `GitTarget` +`Username` is empty whenever no actor was named — both in configured-author mode and when +attribution ran and did not resolve. The `attribution-unresolved` sentinel is scoped to the Git +**author header** and deliberately does not reach templates or message bodies, so a template +rendering `{{.Username}}` never has to special-case it. Use `git log` (or +`author_kind="unresolved"`) to tell the two apart. + `groupTemplate` can use: - `Author` @@ -794,9 +806,11 @@ Progress and outcome are reported through kstatus-compatible **conditions** (no request is finalizing or waiting through `closeDelaySeconds`; `Stalled=True` when the finalize failed and needs attention (kstatus reports the object Failed). - **AuthorAttributed**: `True` with reason `AttributedFromAdmission` when the internal commands - admission webhook captured the request submitter. `False` with reason `CommitterFallback` when no - admission record exists; that is not a failure, and the command still commits as the configured - committer. + admission webhook captured the request submitter. `False` with reason `CommitterFallback` means capture + ran but no admission record exists; `False` with reason `AuthorCaptureDisabled` means capture is not + configured. Neither is a failure. The request then claims no actor and can attach only to an unnamed + watch window, whose Git author remains either the configured committer or the explicit unresolved author + according to the watch attribution outcome. - **Pushed**: `True` once the commit is in the remote repository. ## Audit ingestion settings @@ -873,7 +887,9 @@ When attribution is enabled, these flags tune the join: - `--author-attribution-ttl` (default `10m`): how long an attribution fact is retained waiting for the matching watch event to join it. - `--author-attribution-grace` (default `3s`): bounded per-event wait for a matching audit fact before a - watch event ships authored by the committer. + watch event ships authored by the `attribution-unresolved` sentinel. Note the delivery floor: the + apiserver's own `--audit-webhook-batch-max-wait` delays every fact by up to that much, so a grace at or + below it will lose actors systematically. A matched actor is always named by its own username — humans and service accounts alike (e.g. `system:serviceaccount:flux-system:kustomize-controller`); there is no option to collapse service diff --git a/docs/design/metrics-observability-plan.md b/docs/design/metrics-observability-plan.md index 067a9100..01576243 100644 --- a/docs/design/metrics-observability-plan.md +++ b/docs/design/metrics-observability-plan.md @@ -51,8 +51,8 @@ serious, honest metrics to show, and the audit subsystem is glass-box. object's `name`/`namespace` in a label. Identity labels stay prefixed (`provider_*`, `gittarget_*`) to survive a `honor_labels=false` pod scrape — see the note in [exporter.go](../../internal/telemetry/exporter.go). -6. **Degradation is loud.** Committer fallback, absent Redis, `410` rebuilds, LIST fallback — each has - a metric, so running in a degraded shape is a visible state, not a silent one. +6. **Degradation is loud.** Unresolved attribution, absent Redis, `410` rebuilds, LIST fallback — each + has a metric, so running in a degraded shape is a visible state, not a silent one. ## 3. Current state (the map) diff --git a/docs/design/residual-e2e-flakes-2026-06-19.md b/docs/design/residual-e2e-flakes-2026-06-19.md deleted file mode 100644 index 3b127c53..00000000 --- a/docs/design/residual-e2e-flakes-2026-06-19.md +++ /dev/null @@ -1,233 +0,0 @@ -# Residual e2e flakes after the first-event-loss fix (2026-06-19) - -> **design** — open, not yet built. Index: [`../INDEX.md`](../INDEX.md) - -Status: ACTIVE FOR FLAKE B; FLAKE A CLOSED BY THE 2026-06-20 ATTRIBUTION FIX. After the -capture-before-baseline fix -(`first-event-loss-on-reclaim-plan.md`, committed in `5d85e7d`), -the IceCreamOrder / first-event-loss flake is gone (it passed every one of 4 warm validation runs). Two -**other**, pre-existing flaky specs remain and intermittently fail the `E2E (full)` job. This doc -explains both, leading with what was *observed*, then the *interpretation* with an explicit **certainty -number** on each conclusion. - -How to read the certainty numbers: they are my honest confidence that the stated conclusion is the true -explanation, given current evidence. **Facts** (observed) are stated separately and are ~100% — they are -measurements, not conclusions. - ---- - -## Flake A — CommitRequest "finalize on demand" times out - -### Facts (observed) - -- CI run `27830248680` (commit `5d85e7d`, the committed fix) failed **only** the `E2E (full)` job; all - other jobs (Build, Unit, Lint, Helm, Dev Container, **E2E quickstart**) passed. -- The failing spec: **"Commit Request — finalizes the open commit window on demand and reports the - resulting SHA"** ([commit_request_e2e_test.go:133](../../test/e2e/commit_request_e2e_test.go#L133)). -- The assertion that timed out: `status.phase == "Committed"` within **120 s** (`Eventually`). -- In the same run, a *different* CommitRequest (`commit-request-bundle-save-…`) reached `Committed` - fine (age 12.7 s) — so the commit/push path itself was working; one CR did not finalize. -- The run **before** this one on the branch (`27826911318`, 12:54) was **green**; an earlier era's - warm run also failed a commit_request spec (investigation §2/§5). So this spec fails *intermittently*. - -### What the spec does - -``` - 1. create Deployment ───────────────▶ opens a commit WINDOW (nothing committed; main not created yet) - 2. Consistently(10s): main is empty ─▶ proves the window is holding the edit - 3. create CommitRequest ────────────▶ asks the controller to FINALIZE the window NOW - 4. Eventually(120s): CR.status.phase == "Committed" ◀── THIS timed out -``` - -Finalizing a window must **attribute it to an author**. The controller does that by scanning the -`commitrequests` audit stream for the CommitRequest's *own* `create` event -(`LookupCommitRequestAuthor`); it **fails closed** (never finalizes someone else's window) if it cannot -find that event within its 60 s attribution window. - -``` - CommitRequest created - │ - ▼ - controller reconcile ──▶ scan commitrequests:audit:stream for THIS CR's create event - │ │ - │ found? ├── yes ─▶ attribute author ─▶ finalize window ─▶ phase=Committed ✅ - │ └── no ─▶ retry (2s) … up to 60s … ─▶ phase=Failed (fail-closed) ✗ - ▼ - spec polls phase==Committed for 120s ──▶ ✗ TIMEOUT (the observed failure) -``` - -### Interpretation - -- **This is a pre-existing flake, not caused by the first-event-loss fix.** **Certainty: 90%.** - Reasoning: the fix changed *when the demand gate opens* (claim-time); it did not touch the - CommitRequest finalize/attribution path, and `commitrequests` is force-mirrored by the static - `AlwaysAllow` set regardless. A commit_request spec also flaked in a pre-fix era (§2). *Residual 10%: - I could not pull this run's controller log (a re-run was in progress, logs locked), so I have not - *directly* confirmed the failure reason for this exact instance.* - -- **The likely root is the CommitRequest attribution-scan miss documented in - `e2e-flakes-2026-06-18-investigation.md §2`.** **Certainty: - 55%.** That investigation proved, for a sibling commit_request spec, that the CR's create event was - *provably present* in the scanned stream for the whole 60 s window — correct RV, within the 512-entry - scan bound, matching uid/name, not diverted, not trimmed — yet **missed by ~30 successive scans**. - Root cause was left **OPEN** (a suspected parse/skip edge or a live-scan race). That same mechanism - would make this `:133` finalize spec fail-closed and time out. *Why only 55%: (a) §2 itself never - closed the root cause; (b) the `:133` spec could instead time out for an adjacent reason — e.g. the - Deployment's window never opened, or the push lagged — and I have not pinned this instance's reason.* - -- **An ordering/sequencing cause is plausible and not yet ruled out.** **Certainty that ordering - contributes: 30%.** The spec checks `phase==Committed` **and** then `status.sha == HEAD`, the message - verbatim, and exactly one commit. A wrong/older/later HEAD (e.g. HEAD advancing past the - CommitRequest's commit, or the window finalizing a different commit) would fail the sha/message/count - checks even though *something* committed. The observed failure was the earlier `phase != Committed` - timeout — which points more at the attribution scan than at sha ordering — but with the commit/push - path proven working in the same run (a sibling CR committed), a sequencing mismatch cannot be excluded - without the diagnostic below. *Why only 30%: the recorded failure was the phase timeout, not a - sha/message mismatch; ordering is the secondary hypothesis.* - -- **It is a correctness fail-closed, not data corruption.** **Certainty: 95%.** By design the finalizer - refuses to finalize a window it cannot attribute; the cost is a failed/stuck CommitRequest, never a - mis-attributed commit. - -### What would raise certainty -The parallel agent has added a **"last 5 commits" diagnostic** (`recentCommitDiagnostics`, -[repo_assertions_test.go](../../test/e2e/repo_assertions_test.go)) to the phase/sha/message/count -assertions ([commit_request_e2e_test.go:123](../../test/e2e/commit_request_e2e_test.go#L123)): on a -failure it prints the latest 5 `origin/main` commits (sha, time, author, subject), path-scoped — so the -next reproduction makes "wrong / older / later commit, or a gap" obvious at a glance. Beyond that: the -one-line `scanForCommitRequestCreate` diagnostic from §2.3 (entries-scanned + whether a create-for-name -was seen on a miss) to separate attribution-miss from sequencing; or read the controller log on the next -reproduction. The durable fix candidate (deferred) is the early-ingestion author store -([internal-audit-type-demand.md](../architecture.md)), which removes the scan entirely. - ---- - -## Flake B — Commit Signing late-joining target: overlap ConfigMap missing under path B - -### Facts (observed) - -- Spec: **"Commit Signing — should not replay already-reconciled configmaps as per-event commits to a - late-joining target"** ([signing_e2e_test.go:507](../../test/e2e/signing_e2e_test.go#L507)). -- The convergence assertion: every `overlap-b-cm-NN` ConfigMap must be **present under path B**. It - runs under the **90 s** suite default (`SetDefaultEventuallyTimeout(90s)`, - [signing_e2e_test.go:82](../../test/e2e/signing_e2e_test.go#L82)); it does **not** override it. - *(Correction: an earlier doc draft said 30 s — that was the explicit timeout in the run-3/f3-era code - shown by `Timed out after 30.000s`; it has since been raised to the 90 s default.)* -- Observed failures: `overlap overlap-b-cm-17` (f3) and `overlap-b-cm-15` (run-3) not present under B. -- **The decisive new fact (a fresh reproduction by the parallel agent, at the 90 s timeout):** the - overlap band **split** — `overlap-b-cm-00 … 15` landed in B's **reconcile (backfill) commit**, - `overlap-b-cm-16` was **missing**, and `overlap-b-cm-17 … 19` arrived as **later per-event (live) - commits**. So an object was missing **even at 90 s**, in a contiguous gap between the backfilled - prefix and the live suffix. -- It did **not** fail the no-replay half (`"seed-cm-" must never appear as a per-event commit`); the - failure is the **convergence** half. -- It is intermittent: passed 3 of 4 warm validation runs and did **not** fail the CI run that Flake A - failed; a re-run after the missing-16 reproduction passed. - -### What the spec does — and what the code actually guarantees at the join - -B, joining late on a type A already tails, gets the object set under its own path from **two** sources: -its **initial backfill** (the splice folds the checkpoint `(R, head]` and writes a reconcile commit) and -the **coverage-watermarked tail** (forwards live entries with stream id `> Hc`). - -**Key code fact (checked this session):** the splice sets `coverageHead = the last entry it folded` -(redis_type_splice.go), and the watermark is published -as **exactly that value** — `publishTargetTypeWatermark(gitDest, gvr, snapshot.CoverageHead)` -([event_router.go:241](../../internal/watch/event_router.go#L241)) — and the tail forwards id `> Hc` -(audit_tail.go). So `Hc` is **contiguous with the fold by -construction**: everything `≤ coverageHead` is in the backfill, everything `> coverageHead` is forwarded -by the tail. There is no `(S, Hc]` window for an object to fall into — *unless its stream entry is -missing entirely*. **This demotes the "watermark set ahead of the backfill" boundary-gap I hypothesised -in the first two drafts.** - -The split that was actually observed (`00–15` backfilled, **`16` missing**, `17–19` live) fits a -**diverted cm-16** far better — the overlap band is one batch `kubectl apply`, so audit events can -arrive **out of RV order**; an event whose RV lands **below the stream high-water** is **diverted** -(rejected from the per-type stream, redis_bytype_queue.go). -A diverted cm-16 is in **neither** the fold (not in the stream) **nor** the tail (not in the stream), -and is healed only when the **divert nudge** re-anchors the type — which races the 90 s deadline: - -``` - overlap band applied in ONE batch → audit events arrive out of order - cm-00..15 ingested in order ─▶ in stream ─▶ folded by B's backfill (≤ coverageHead) ✅ - cm-16 arrives AFTER cm-17 (rv below high-water) ─▶ DIVERTED, never in stream - └─▶ not folded, not tailed ─▶ MISSING under B - └─▶ divert nudge → RequestResync → re-anchor re-LIST ─▶ heals cm-16 ⏱ races 90s - cm-17..19 ingested in order ─▶ in stream, id > coverageHead ─▶ forwarded by tail ✅ -``` - -### Interpretation - -- **Distinct, pre-existing race — not the first-event-loss bug, not caused by the fix.** - **Certainty: 90%.** `configmaps` is a common, already-Synced type; the per-event commits work (no-replay - half passes); the fix touched neither the splice/coverage code nor the divert path. *Residual 10%: the - fix mirrors a claimed type slightly earlier — could marginally shift join timing; not demonstrated.* - -> **Defer to the authoritative deep-dive:** -> [`signing-overlap-band-coverage-drop-investigation.md`](../finished/signing-snapshot-tail-replay-failure-investigation.md) -> is the rigorous (FACT/DERIVED/HYPOTHESIS-tagged) investigation of this exact flake, and it **measured -> the divert counter**. The bullets below are reconciled to it — and it walks back the "divert" lead I -> had given (the diagram above is now just *one — contradicted — candidate*). - -- **What is forced (DERIVED): cm-16 was not a normally-ordered main-stream entry within B's reconcile - cut.** **Certainty: 85%.** The splice's `Desired` (≤ `coverageHead`) and the tail (> `coverageHead`) - are one consistent cut, so a normally-ordered stream entry cannot be dropped — therefore the object - was off the main stream (never ingested, diverted, or an attach anomaly). This is §4.1 of the deep-dive. - -- **The specific path off the stream is UNCONFIRMED — and divert is *contradicted*, not confirmed.** - *(This corrects my earlier "divert leading, 55%".)* The deep-dive measured **`lateCount=0` for - ConfigMaps** at the failure (§4.4), so **H2 (divert) is argued *against*, not for** — the evidence is - the opposite of what I assumed. Current ranking (its §4.5): **H1 "never a normally-ordered stream - entry" is least-contradicted (~40%)**; H3 rv-missing-attach anomaly (`rvMissingCount=66`) open (~20%); - H4 `Hc`-seam **unlikely** because `Hc` is published as the fold's `coverageHead` (contiguous by - construction) *unless* a stale-high prior `Hc` is held (~15%); **H2 divert contradicted (~15%)**. No - mechanism is confirmed — a capture is required. - -- **On "just widen the timeout":** it depends on §5 of the deep-dive — *is "every object present under a - target within one reconcile" a guarantee, or only "by the next checkpoint/heal"?* If the latter, the - 90 s assertion is a **test over-assertion** and widening (or budgeting the heal cadence) is legitimate; - if the former, the join-time path has a real gap to fix. **Unresolved — certainty either way: ~50%.** - -- **Correctness: likely a bounded gap, but "self-healing" is NOT established.** *(Down from my earlier - confident "converges, 70%".)* The deep-dive's §8 says the object was **permanently absent** in that - run's repo (the heal did not visibly recover it within the window, and `DeferCleanup` removed B). So - whether it heals at the next re-anchor is **Q2/Q5 — open**, not a settled fact. The no-replay invariant - did hold. - -### What would raise certainty — and the instrumentation added this session -On the next reproduction, these now-added logs disambiguate divert vs boundary in one read: -- **`late audit event nudged a type resync`** (now **info**, [materialization.go](../../internal/watch/materialization.go)) — fires iff a divert happened; if it names `configmaps` around the join, the divert mechanism is confirmed. -- the scoped **`diag_all`** firehose already captures `configmaps`, so a diverted cm-16 shows as `outcome=older_than_high_water`. -- **`target-type watermark published / held`** (target_type_watermark.go) — shows B's actual `Hc` and whether a stale-high prior was held (the secondary hypothesis). -- **`audit-tail routed batch to target`** (V(1), audit_tail.go) — per-batch `routed / suppressedAtOrBelowHc / firstID / lastID`; if cm-16 appears here and is `suppressedAtOrBelowHc`, it is the boundary case; if it never appears, it was diverted upstream. - -If the repro shows divert: the durable answer is faster/again-reliable nudge-heal (or accept + widen the -test deadline). If it shows a held stale-high `Hc`: fix the publish to not hold a boundary ahead of the -fold. **No fix is applied yet — the mechanism is not confirmed (trust gate not met).** - ---- - -## Summary - -| | Flake A — CommitRequest finalize | Flake B — Signing late-join | -| --- | --- | --- | -| Failing spec | `commit_request_e2e_test.go:133` | `signing_e2e_test.go:507` | -| Symptom | `phase==Committed` not reached in 120 s | one `overlap-b-cm-NN` missing under B (at **90 s**) | -| Observed in | CI `27830248680`; era run-2 | local f3 + run-3 + a fresh split repro (00–15/✗16/17–19) | -| Pre-existing, not the fix | **90%** | **90%** | -| Likely mechanism | attribution-scan miss (§2, root OPEN) — **55%**; ordering a secondary — **30%** | object was **off the main stream** (DERIVED, 85%); *which* path **unconfirmed** — divert **contradicted** by `lateCount=0`; H1 never-ingested least-contradicted (see deep-dive) | -| Correctness | fail-closed, not mis-attributed — **95%** | bounded gap *likely* but self-heal **NOT established** (object was permanently absent in the repro) | - -**Bottom line (certainty 80%):** the first-event-loss fix is sound and landed; `E2E (full)` stays -intermittently red on these two *independent, pre-existing* flakes (the same commit `5d85e7d` failed -E2E(full) on attempt 1 and **passed on re-run** — direct proof they are flaky, not deterministic). -**Neither is evidence against the fix.** Two self-corrections this session: (1) the watermark is -published *as* the splice's `coverageHead`, so the boundary is contiguous by construction (the -`Hc := S` boundary-fix I first proposed is largely moot); (2) the divert lead I then gave is itself -**contradicted** by the deep-dive's `lateCount=0` measurement — so the honest state for Flake B is -"the object was off the main stream, **but by which path is unconfirmed**," tracked rigorously in -[`signing-overlap-band-coverage-drop-investigation.md`](../finished/signing-snapshot-tail-replay-failure-investigation.md). -I deliberately **did not apply a fix** (trust gate not met); instead I landed the instrumentation -(nudge→info, watermark publish/held, per-batch tail-route summary) that, with that doc's §6.1 -ingestion-side capture, collapses H1–H4 on the next reproduction. Flake A's leading hypothesis stays -the attribution-scan miss (55%), ordering a tracked secondary (30%). diff --git a/docs/design/support-boundary/finished/higher-level-krm-documents.md b/docs/design/support-boundary/finished/higher-level-krm-documents.md index 887df749..30cd4530 100644 --- a/docs/design/support-boundary/finished/higher-level-krm-documents.md +++ b/docs/design/support-boundary/finished/higher-level-krm-documents.md @@ -99,7 +99,7 @@ two reasons: Flux CRDs (`helmreleases.helm.toolkit.fluxcd.io`, `kustomizations.kustomize.toolkit.fluxcd.io`) are established during standard -e2e setup ([../../../../test/e2e/Taskfile.yml](../../../../test/e2e/Taskfile.yml), +e2e setup ([../../../../test/e2e/Taskfile-e2e.yml](../../../../test/e2e/Taskfile-e2e.yml), `_flux-installed`), so the live pin uses a `HelmRelease`. The Argo and KRO controllers are not in the base cluster; their kinds are pinned at the unit level, where the analyzer/editor need only the document bytes — no controller, diff --git a/docs/finished/documentation-triage.md b/docs/finished/documentation-triage.md deleted file mode 100644 index 99f174fd..00000000 --- a/docs/finished/documentation-triage.md +++ /dev/null @@ -1,276 +0,0 @@ -# Documentation triage: what binds, what is history, what to do - -> **finished** — this plan was executed on 2026-07-11. Kept as the record of *why* the -> tree looks the way it does. For what to read now, see [`../INDEX.md`](../INDEX.md). -> -> Outcome: 180 documents -> 117. `docs/spec/` created (25 docs the code depends on), -> `docs/design/manifest/` deleted (19 -> 1 summary + 4 promoted specs), ~50 dead documents -> removed, and every dangling citation repaired — including the 17 that were already broken -> before this started. -> -> Scope as surveyed: all 180 markdown files under `docs/` (~55,000 lines). - -## The headline - -**About 35 documents actually bind. The other ~145 are history.** You cannot -currently tell which is which without reading them, and that — not the volume — is -the problem. - -| Group | Files | Of which still binding | -|---|---:|---:| -| `docs/*.md` (user/operator guides) | ~20 | all — they are the product surface | -| `docs/facts/` | 4 | all — durable reference | -| `docs/design/*.md` + `design/stream/` | 57 | **9 active**, 9 reference | -| `docs/design/support-boundary/` | 13 | **all — the live workstream** | -| `docs/design/manifest/` + `version2/` | 19 | ~3 — the rest shipped | -| `docs/finished/` | 41 | **10 load-bearing** (see below — this is the bug) | -| `docs/future/` | 13 | 8 still wanted | -| `docs/ci/`, `docs/audit-setup/`, misc | 13 | reference | - -## The diagnosis: the folder names lie, in both directions - -This is the whole finding, and it explains why the tree feels unmanageable. - -**`docs/design/` contains 23 documents whose work has already shipped**, sitting -beside the 9 that are still open, with nothing to distinguish them. Several still -say `Status: PROPOSAL` while the code that implements them has been in `main` for -weeks (`unsupported-folder-refusal-plan.md` is the clearest case — it ships as -`GitPathAccepted` with an e2e test, and its header still says "PROPOSAL"). - -**`docs/finished/` contains 10 documents that are the *sole written record* of a -live invariant.** They are not finished at all. The worst case: - -> **`docs/finished/current-manifest-support-review.md`** is the live specification -> of `internal/manifestanalyzer`. It is cited by section name from eight Go files, -> and it is a cited input to active downstream work. It holds the -> non-negotiable rules — *a GitTarget makes an all-or-nothing claim on a folder*; -> *never partially materialize a multi-doc file*; *refuse the folder rather than -> prune unwatched KRM*. It is filed under "finished." - -Nine more like it are listed below. **Archive `finished/` wholesale and you delete -the contract.** - -Only 74 of 180 files carry a `> Status:` line at all, and several of those are -stale. There is no mechanical way to ask "what is still true?" - -## Hard constraint: 17 doc citations in the Go code are already broken - -A previous reorganisation moved documents and never fixed the pointers. Today, -`grep`ping the Go sources for `docs/**.md` yields **17 paths that do not exist**: - -``` -docs/guide.md docs/keep.md -docs/design/manifest/implementation-plan.md docs/design/commit-window-refactor.md -docs/design/manifest/current-manifest-support-review.md docs/design/audit-readiness-probe-plan.md -docs/design/stream/api-source-of-truth-reconcile.md docs/design/support-boundary/f1-...md -docs/future/secret-value-retention-plan.md docs/design/support-boundary/f7-...md -docs/design/stream/audit-diagnostic-streams-plan.md …and 6 more -``` - -Six of those point at documents that no longer exist anywhere. The rest point at -documents that *moved into* `docs/finished/`. - -Two consequences, and they shape the entire plan below: - -1. **The code already thinks the load-bearing docs live in `docs/design/`.** That - is an argument for moving them back rather than for leaving them where they are. -2. **Any move must fix citations in the same commit,** or it makes this worse. This - is exactly how the current mess was made. - -## What actually binds — the read-this list - -If you read nothing else, read these. This is the set that describes how the system -works *today* or what is being decided *now*. - -### The live workstream (13) — `docs/design/support-boundary/` - -All of it. This is the branch you are on. My separate proposal -([expansion-boundary-and-corpus-organisation.md](../design/support-boundary/expansion-boundary-and-corpus-organisation.md)) -recommends adding a `support-contract.md` front door and folding four restatements -of the fan-in rule into one. - -### Load-bearing, but misfiled under `finished/` (10) — **promote these** - -Each is the only place a still-current invariant is written down, and most are cited -from the code: - -| Doc | The invariant that lives only here | -|---|---| -| `current-manifest-support-review.md` | all-or-nothing folder claim; never partially materialize a multi-doc file; refuse rather than prune | -| `manifestedit-field-ownership-spike.md` | "the API wins" — full-object ownership, no field-subset ownership, plus the explicit do-not-build list | -| `sops-single-file-no-multidoc.md` | SOPS encrypts a *file* as one cryptographic unit → one document per encrypted file | -| `scale-subresource-audit-rehydration.md` | the `/scale` → `FieldPatch` translation rules and denied-subresource list | -| `commit-window-refactor.md` | one grouped commit = exactly one (author, GitTarget) tuple | -| `commitrequest-multi-finalize-design.md` | why `MaxConcurrentReconciles=1` on the CommitRequest controller | -| `type-lifecycle-events-and-wobble-settling.md` | the settle-window / flap-coalescing spec `internal/typeset` implements | -| `gittarget-isolation-on-rule-change.md` | a rule change on target A must never re-snapshot target B | -| `audit-readiness-probe-plan.md` | why liveness must never depend on Redis (the restart-loop trap) | -| `gvk-gvr-mapping-layer.md` | GVK↔GVR contract; still cited by four active design docs | - -### Still-open design (9) — `docs/design/` - -`e2e-coverage-gaps-and-improvements-plan.md`, `e2e-finish-plan.md`, -`metrics-observability-plan.md`, `multi-source-audit-ingress-hardening.md`, -`reconcile-triggering.md`, `release-image-reuse-plan.md`, -`sensitive-resource-diagnostics-follow-up.md`, `watch-and-catalog-architecture.md`, -`stream/residual-e2e-flakes-2026-06-19.md`. - -### Durable reference (9 + `docs/facts/`) - -`status-conditions-guide.md`, `e2e-serial-registry.md`, `e2e-test-design.md`, -`kubernetes-api-discovery.md`, `kubernetes-api-resource-catalog.md`, -`kubernetes-apf-and-inflight-tuning.md`, `audit-webhook-api-server-connectivity.md`, -`watch-event-ordering-and-attribution-grace.md`, `sops-repo-bootstrap-out-of-scope.md`. - -### Still wanted (8) — `docs/future/` - -`idea-application-editing.md` (the product seed of a future application-editing workstream — -still holds the branch/session grouping strategies nothing else covers), -`ha-gittarget-distribution-plan.md` (cited three times by `architecture.md`), -`least-privilege-remaining-work.md`, `design-commit-request-phase-2.md` (verified -live gap: nothing GCs `CommitRequest` objects), `idea-unify-pending-write-kinds.md`, -`idea-cross-kind-dependency-watches.md`, `publish-age-recipients-as-metadata.md`, -`idea-commit-signing-key-validation.md`. - -## What is history - -### Shipped, but still sitting in `docs/design/` (23) — move to `finished/` - -`audit-webhook-tls-design`, `commitrequest-admission-authorship`, -`deletecollection-attribution-expander`, `dynamic-analysis-fuzzing-plan`, -`e2e-aggregated-apiserver-test-design`, `e2e-bi-directional-corner`, -`e2e-ci-runner-sharding-plan`, `e2e-signing-followups`, `e2e-speedup-plan`, -`git-credentials-interop`, `gitpath-foreign-content-stringency`, -`image-refresh-test-design`, `mutation-capture-lab-design`, `redis-key-schema-v3`, -`sensitive-resource-classification-plan`, -`sops-repo-bootstrap-and-key-management-architecture`, -`startup-robustness-cert-and-crd-wobble`, `tilt-playground-plan`, -`typeset-owns-discovery-grace`, `unsupported-folder-refusal-plan`, -`watch-first-ingestion-architecture`, `stream/commitrequest-design`, -`stream/streaming-readiness-status-machine-design`. - -Note `watch-first-ingestion-architecture.md` is the *most* important of these — it -is the rewrite that deleted half the audit world — but it has landed, so it is now -an explainer, not a plan. It should arguably graduate into `docs/architecture.md` -rather than into `finished/`. - -### Superseded (6 + 13) — **delete** - -Git is the archive. These have named successors and nothing cites them: - -- In `design/`: `e2e-aggregated-apiserver-proxy-hookup-plan`, - `gittarget-lifecycle-and-repo-architecture`, - `gittarget-state-machine-bootstrap-assessment`, `status-design-git-target`, - `watchrule-wildcard-and-resolution-semantics`, - `stream/per-type-streaming-readiness-plan`. -- In `finished/`: `api-source-of-truth-reconcile`, - `commitrequest-attribution-divert-reliability`, `internal-audit-type-demand`, - `design-commit-request-api`, `deletecollection-resync-nudge-plan`, - `audit-metrics-overhaul-plan`, `materialization-tail-and-live-readiness-review`, - `watch-list-checkpoint-plan`, `per-resource-type-rv-keyed-streams-experiment`, - `idea-audit-enrichment-side-channel`, `rule-set-snapshot-discovery-lag-fix`, - `watchrule-gvr-resolution-plan`, `manifestedit-new-file-placement-spike`. - -One catch before deleting `materialization-tail-and-live-readiness-review.md`: -`internal/typeset/materializer.go:34` still cites its "Rec 6" as an open -optimisation. Move that note into the code comment first. - -### Closed investigations (9) — **delete** - -Forensics whose bugs are fixed: `e2e-full-suite-shared-state-investigation`, -`e2e-watchrule-cross-spec-interference`, `upgrade-finding`, -`watch-audit-rule-matching-improvement`, `watch-first-merge-readiness`, and four -under `stream/`. These are the genre with the worst read-value-to-length ratio — -long, dramatic, and entirely about bugs that no longer exist. - -Keep one exception: `stream/signing-snapshot-tail-replay-failure-investigation.md` -is cited from `internal/git/branch_worker.go:330` as the reason a watermark gate -exists. Move that reasoning into the code comment, then delete. - -### Archaeology (18 in `finished/`) — cold-store or delete - -Mostly the pre-watch-first audit world: `demand-gated-audit-ingestion`, -`canonical-stream-retirement`, `shallow-audit-event-misclassification`, -`partial-object-audit-event-handling`, `design-audit-ingestion-hardening`, and 13 -more. `internal/gate` and the audit-as-state pipeline they describe are **deleted -code**. Nothing forward-looking depends on them. - -### Overtaken / stale (5) — delete - -`future/design-snapshot-engine-evolution` (every part landed elsewhere in a -different shape), `future/idea-end-user-commit-messages` + -`future/addendum-end-user-commit-messages-audit-transports` (both rest on "the audit -stream is the source of truth", which watch-first made **false**), -`future/watchrule-wildcard-support-plan`, -`future/further-away/accesspolicy-design` (targets `GitRepoConfig`, a CRD two -renames dead), `design/voter-gitops-demo-instances`. - -## The plan - -Phased so that the cheap, zero-risk win lands first and nothing breaks. - -### Phase 0 — the index (zero risk, most of the benefit) - -**Write `docs/INDEX.md`: the ~35 documents that bind, grouped by topic.** Move -nothing. Delete nothing. You immediately get "read these 35, the rest is history", -which is the actual thing you asked for. Everything below is optional cleanup. - -### Phase 1 — status headers (mechanical) - -Put a `> Status:` line on every design doc, from a closed vocabulary: -`active` | `shipped` | `superseded-by: ` | `closed-investigation` | -`reference`. Then `grep '^> Status: active'` answers the question forever, and the -next person does not have to redo this triage. - -### Phase 2 — fix the 17 broken citations, and promote the 10 load-bearing docs - -Do these **in one commit**, because the fix and the move touch the same lines. The -code already points at `docs/design/` for most of them, so promoting them back -*is* the fix. - -Suggested destination: a `docs/spec/` folder meaning **"this is true now and the -code depends on it"** — distinct from `design/` ("we are still deciding") and -`finished/` ("this happened"). The three-way split is what the current two-way one -cannot express, and it is precisely the distinction that broke. - -### Phase 3 — move the 23 shipped docs out of `design/` - -Now `docs/design/` means what it says: undecided work. It shrinks from 57 files to -~9. - -### Phase 4 — delete the dead (~50 files) - -Superseded, closed investigations, archaeology, stale. Extract the three code-cited -facts first (noted above). Git keeps everything; nothing is lost that a -`git log --follow` cannot recover. - -### Phase 5 — documentation consolidation - -Separately proposed in -[expansion-boundary-and-corpus-organisation.md](../design/support-boundary/expansion-boundary-and-corpus-organisation.md): -add a `support-contract.md` front door, fold the four restatements of the fan-in -invariant into one, and de-duplicate the `--dry-run=server` evidence table. - -## What I would *not* do - -- **Do not bulk-archive `docs/finished/`.** Ten of those documents are the contract. - That is the single highest-risk action available here, and it is the one that - looks most obviously safe. -- **Do not move anything without fixing the Go citations in the same commit.** That - is how the current 17 broken pointers were created. -- **Do not delete the investigations to save space.** Delete them because their bugs - are gone. Two of them still hold the *reason* a guard exists in live code; those - reasons must land in the code comment first, or you will lose them and someone - will remove the guard. - -## Open question - -**Should `docs/design/manifest/` survive at all?** Its three subsystems — -`internal/typeset`, `internal/git/manifestedit`, `internal/manifestreport` — have -all shipped, so most of its 19 files are archaeology. But it is also where the -*reasoning* behind the manifest model lives, and downstream work is built -directly on top of it. My instinct is that two or three docs graduate to `spec/` -(`contextual-namespace-and-kustomize-folder-editing.md`, -`version2/gittarget-new-file-placement-rules.md`, -`version2/type-followability.md`) and the other sixteen go, but that is the call I -am least sure of and it is worth your eye rather than mine. diff --git a/docs/finished/redis-key-schema-v3.md b/docs/finished/redis-key-schema-v3.md index 3cccc312..41dd5f32 100644 --- a/docs/finished/redis-key-schema-v3.md +++ b/docs/finished/redis-key-schema-v3.md @@ -216,7 +216,8 @@ unique across resource types, and aggregated/CRD API servers can mint their own. an `objectRef` without a UID. Under the `RequestResponse` audit level this project relies on, `IdentityFromAuditEvent` ([`identity.go`](../../internal/auditutil/identity.go)) backfills the UID from the body even for `generateName` creates, so a no-UID fact is uncommon — but when it happens, `rv:` -is the difference between a precise author and a committer fallback. It is **cheap and shipped now**: one +is the difference between a precise author and an explicit unresolved author. It is **cheap and shipped +now**: one extra key, written *only* when the fact has no UID (a UID-bearing fact's `rv:` key would be dead — the watch side always carries a UID, so it would resolve via `object::…` first and never reach it). This is a deliberate improvement over v2, which wrote a dead rv-only key for *every* event. diff --git a/docs/finished/watch-first-ingestion-architecture.md b/docs/finished/watch-first-ingestion-architecture.md index 42ffa3a8..5d377006 100644 --- a/docs/finished/watch-first-ingestion-architecture.md +++ b/docs/finished/watch-first-ingestion-architecture.md @@ -1,506 +1,36 @@ -# Watch-first ingestion architecture - -> **finished** — shipped or closed. Kept for context only; **nothing here binds**. For current behaviour see [`../spec/`](../spec/). Index: [`../INDEX.md`](../INDEX.md) - -> Status: **accepted direction — big-bang rewrite** -> Date: 2026-06-25 -> -> Related: -> [Current architecture](../architecture.md), -> [Mutation Capture Lab](../spec/mutation-capture-lab-design.md), -> [API-source-of-truth reconcile](../architecture.md), -> [Mutation lab corpus](../../test/mutationlab/corpus/), -> [Mutation lab README](../../test/mutationlab/README.md) - -## Summary - -GitOps Reverser today treats the **audit webhook** as the authoritative live mutation stream, and -spends a large amount of machinery making that stream ordered, gap-free, demand-gated, and -attributable. The mutation-capture corpus proved that Kubernetes **WATCH** is a strictly better source -for persisted object state on exactly the cases that hurt today (aggregated APIs, shallow bodies, CRD -conversion, deletecollection fan-out). This document is the decided target: **make WATCH the only -source of object state, run one watch set per GitTarget, and demote audit to an optional attribution -lookup table.** - -Three decisions frame this rewrite: - -1. **Per-GitTarget watches.** Each GitTarget opens and owns the watches for the resource types it - claims. A GitTarget becomes a self-contained unit — its watches, its event stream, its branch - worker, its commit window — which makes horizontal scaling and HA a matter of *assigning GitTargets - to replicas*, not coordinating a shared per-type pipeline. It also keeps each watch's replay and - mark-and-sweep bound to a single Git tree, which is what makes them simple (see the dedup note under - [HA](#per-gittarget-ownership-and-ha)). Sharing watches across GitTargets is an explicit future - enhancement, not the first cut. -2. **Redis is required; audit attribution is the optional layer on top.** Redis/Valkey holds each - GitTarget's watch **resume cursors** (state continuity / work re-pickup) and is the substrate for HA - and the planned durable branch-worker queue, so it is a hard dependency in every mode. What is - optional is **audit attribution**, toggled by `--author-attribution` (chart - `attribution.enabled`). With attribution off the product runs **configured-author**: it mirrors cluster - state to Git, authored by the configured committer identity, and Redis still backs the resume cursors. - With attribution on (and the audit webhook configured to post to it), the product additionally - resolves the **author** from audit facts when the evidence is strong. Attribution off → committer. - Attribution on → committer **and** author. -3. **Big-bang.** The audit-as-correctness pipeline is removed, not kept behind a flag. The current - design is not perfect and is not worth preserving in parallel; the simplification is large enough - that a clean replacement is cheaper to reason about than a dual-mode coexistence. - -The conceptual core: **the Git tree is a projection of persisted state observed by watch.** Audit -explains *who* caused a change when it can; it never defines *what* changed. - -## The realization that deletes a subsystem - -Once a live watch is the source of truth, two facts follow that erase most of the current engine: - -- **Git is the materialized state; the watch is the only feed into it.** Each watch event is sanitized - and diffed against current Git content — there is no per-type Redis object log to reconstruct, and no - separate in-memory cache to keep authoritative (Git already holds current state). -- **The apiserver delivers events already ordered by `resourceVersion` per type.** There is nothing to - re-order. The entire reason the current system exists — making interleaved, out-of-order audit - batches into a per-type ordered stream — goes away. - -So the following all disappear, not as an optimization but as dead concepts: - -- the per-type Redis audit streams and their RV-anchored stream positions; -- the **late-lane / divert / nudge** machinery (out-of-order numeric cross-writes were an audit-batch - artifact; watch has no such thing) — the recurring flaky invariant retires with it; -- the **demand gate** (it existed to stop a per-type *mirror* exploding across cluster types; there is - no mirror to bound); -- the **audit log splice fold**, its RV-anchored stream position, and the **periodic checkpoint LIST + - timer-driven drift sweep** — a mark-and-sweep survives, but only on watch re-establishment, never on a - timer (see [Resume and recovery](#resume-and-recovery)); -- the **materialization phase machine** (Dormant→Requested→Syncing→Synced…) and its durable - `:objects:state` — the live watch plus the Redis RV cursor replace it, and the "re-claimed type stuck - in phase=removed" class of bug retires with the phase machine. - -The whole engine is one raw watch per `(GVR, scope)` with `sendInitialEvents`, a Redis resume cursor, -and a mark-and-sweep on replay — **no periodic LIST, no informer resync, no `SharedInformer`** (a -`SharedInformer`'s reflector re-LISTs on every process start and exposes no cross-restart resume cursor, -which is exactly what we rely on). See [State ingestion](#state-ingestion-raw-watch-per-type) and -[Resume and recovery](#resume-and-recovery). - -## What each question is answered by - -| Question | Source | -|---|---| -| **What changed?** | WATCH (with `sendInitialEvents` replay) over claimed, followable types. The only correctness path. | -| **Who caused it?** | Optionally, audit facts in Redis — only on strong evidence, only when audit attribution is enabled. | - -A missing, late, shallow, conflicting, failed, dry-run, or simply absent audit fact never blocks or -alters state capture. It can only change the commit *author* from the bot to a named user/service -account. - -## Target architecture - -```mermaid -flowchart TD - subgraph K8S["Kubernetes API"] - DISC["Discovery / CRDs / APIServices"] - WATCH["WATCH + sendInitialEvents replay (per claimed GVR + scope)"] - AUDIT["Audit webhook (optional)"] - end - - subgraph PERGT["Per GitTarget"] - OWN["GitTarget watch set\n(claimed ∩ followable (GVR, scope))"] - RELEVANCE["Relevance filter\n(sanitize · followability · no-op diff)"] - ESTREAM[("GitTarget event stream")] - WORKER["BranchWorker → writer → push"] - end - - subgraph ATTR["Attribution (only when enabled)"] - AFACTS["Audit fact extractor"] - AINDEX[("Redis attribution index\n(TTL, keyed for join)")] - RESOLVE["Conservative resolver\n(bounded grace window)"] - end - - DISC --> OWN - WATCH --> OWN - OWN --> RELEVANCE - RELEVANCE --> RESOLVE - RESOLVE --> ESTREAM - ESTREAM --> WORKER - - AUDIT -. configured .-> AFACTS - AFACTS --> AINDEX - AINDEX -. lookup .-> RESOLVE -``` - -In this shape: - -- A GitTarget reconcile resolves its claimed `(GVR, scope)` set, then opens one watch (replay + live) per - `(GVR, scope)`. **Scope is part of the watch identity:** a namespaced WatchRule opens a namespaced - watch (or a field-selector-bounded one), never a cluster-wide watch filtered in-process — that would - be both an RBAC/privilege escalation and a load multiplier. The existing namespace tracking - (`NamespaceOps` / `SnapshotNamespaces`) supplies the scope. There is no global per-type pipeline; - ownership is per GitTarget. -- The **relevance filter** — owned by product code — discards the controller noise the cluster's audit - policy used to discard for free (status churn, runtime-owned subresources, no-op diffs). -- Every Git write derives from persisted state observed by watch (live, or via the `sendInitialEvents` replay). -- The **resolver** attaches an author only when audit attribution is enabled *and* an audit fact matches - strongly; otherwise the commit is authored by the configured committer. -- Audit events never create object changes and never repair object bodies. - -This is still "watch + replay," not "watch from nothing": a reliable watch needs an initial replay of -current state (`sendInitialEvents`) and a fresh replay after `410`. "Watch-first" means the watch is the -*only* object-state ingestion mechanism — there is no separate LIST/checkpoint path. Audit is advisory -and optional. - -## State ingestion: raw watch per type - -A per-GitTarget set of raw watches replaces the per-type audit stream as the live object log. - -| Current audit-first concept | Watch-first replacement | -|---|---| -| `...:audit:stream` (per type, in Redis) | raw watch event loop → diff against Git (no separate object store) | -| Audit event payload | sanitized watch object | -| Audit tail reader (blocking XREAD per type) | raw watch per `(GVR, scope)` | -| Checkpoint + audit log splice | `sendInitialEvents` replay-as-ADD on every (re)connect; `(GVR,scope)→RV` resume cursor in Redis | -| Audit body joiner | removed (watch carries the body) | -| Shallow-audit drop / wait-for-body | not relevant to state content | -| Demand gate (bound the mirror) | open watches only for claimed ∩ followable `(GVR, scope)` | -| **Audit policy noise filter** | **product-owned relevance filter** (new responsibility) | - -Each watch event carries everything Git needs: GVR, scope, event type (`ADDED` / `MODIFIED` / `DELETED`, -plus the transport-level `BOOKMARK` and `ERROR` a raw watch delivers), -namespace/name/UID/resourceVersion/generation/deletionTimestamp, and the sanitized object for -object-bearing events. We consume `BOOKMARK` directly: the `initial-events-end` bookmark marks the end -of the replay, and bookmark RVs advance the resume cursor we persist to Redis; an `ERROR` (e.g. `410 -Gone`) triggers a fresh `sendInitialEvents` reconnect. Each event flows into the existing per-GitTarget -[`GitTargetEventStream`](../../internal/reconcile/git_target_event_stream.go) as a -[`git.Event`](../../internal/git/types.go), then to the BranchWorker — the same seam audit events use -today, so the entire downstream writer is untouched. - -### Resume and recovery - -Losing a watch — pod eviction, rollout, crash, `410 Gone` — is normal, and the design's job is to come -back *correctly*, never silently dropping a delete. There are exactly two resume modes, and the choice -between them is the choice between "we still know precisely where we were" and "we don't." - -**Mode A — resume from the exact point (efficient, preferred).** We persist -`(GVR, scope) → last resourceVersion` to Redis as events arrive, and keep it fresh with watch bookmarks -even when the type is quiet. On restart we re-open the watch *from that RV* — a plain delta watch, no -replay. If the apiserver still holds that RV in its watch cache, it streams every event since, -**including the `DELETED`s we missed**, and we simply continue. No replay, no sweep. This is the common -case for short gaps, and it is the efficient path: we pick up exactly where we left off. - -**Mode B — replay with mark-and-sweep (the correctness fallback).** If the exact RV is gone (compacted -→ `410 Gone`, no cursor yet, or a cold start), a delta is untrustworthy — objects may have been -deleted while we were away and a delta from an unknown point would never reveal it. So we open with -`sendInitialEvents=true` and run a **mark-and-sweep** over the replay: - -1. **Mark** every object the replay delivers as `ADDED`, up to the `initial-events-end` bookmark. -2. **Sweep** at the bookmark: for this `(GVR, scope)`, any Git file whose object was *not* marked no - longer exists — emit a `DELETED` for it (committer-authored; we never witnessed the actual delete). -3. Then stream live. - -**This mark-and-sweep is load-bearing and must not be dropped.** It is the *only* thing that reconciles -a delete that happened while no watch was running — replay-as-ADD alone tells us what *exists*, never -what is *gone*, so without the sweep an orphaned Git file would linger indefinitely. Mode B is what -makes watch-first safe to lose and restart. It reuses the existing reconcile/sweep writer machinery; -what changes from the old design is the **trigger** — it fires on watch re-establishment (a replay), -**not** on a periodic timer. We deliberately do **not** run an hourly sweep: it runs exactly when -continuity was lost, and never while the watch has been continuously live. - -Net: keep the watch live and every delete is seen directly; resume from the exact RV and the delta -carries the deletes; otherwise replay and the mark-and-sweep reconciles them. **No path silently loses a -delete.** - -### Relevance filtering becomes product code (the cost that does not shrink) - -This is the one place the rewrite *adds* work, and it must be counted honestly. The committed e2e -audit policy ([test/e2e/cluster/audit/policy.yaml](../../test/e2e/cluster/audit/policy.yaml)) is not a -passthrough — it is a three-tier relevance filter that drops `*/status`, HPA `*/scale`, leases, -events, node heartbeats, and keeps human-meaningful create/update/patch/delete. **Watch has no such -policy.** It delivers every persisted `MODIFIED`, including all the churn the policy dropped at the -source before it ever reached the webhook. - -Watch-first mode must reproduce that filter in product code, on the hot path: - -1. **No-op suppression.** A `*/status` write bumps `resourceVersion` and produces a `MODIFIED` whose - sanitized desired-state projection equals the prior commit. The writer already diffs to no-op, but - now we pay per-event CPU on every status churn before discarding it. -2. **Followability encodes "controller-owned."** [`internal/typeset`](../../internal/typeset/) is the - home for "we do not mirror this type's churn" — the analogue of audit policy Rule 1. -3. **Sanitization is mandatory and on the hot path.** [`internal/sanitize`](../../internal/sanitize/) - strips status, managedFields, and volatile metadata before diffing so runtime churn never - masquerades as a desired-state change. - -We are not removing a filter; we are *moving* it from the cluster's audit policy into the product. That -is more honest and version-portable, but the event-volume cost is real and continuous for status-heavy -clusters. The filter must be observable (see metrics), so a mis-tuned filter is visible rather than -silently dropping intent. - -### History granularity changes (accept and document) - -Watch carries only the versions it observes. While connected it sees each `MODIFIED`; across a replay -after `410`, a compaction, or process downtime, it **collapses every intermediate version into current -state**. Those become one replay/committer commit (or none, if the net diff is empty), not the N user -commits audit would have produced. - -Honest guarantee: **watch-first delivers every persisted mutation observed while watching, and -collapses to current state across gaps.** It is a *state mirror with opportunistic per-mutation -history*, not a guaranteed per-mutation change log. This must be explicit in user docs. - -## Per-GitTarget ownership and HA - -A GitTarget owns the watches for its claimed types. This is the deliberate design choice and it buys -the HA story almost for free: - -- **Sharding is per GitTarget.** Assign whole GitTargets to replicas (leader-elected ownership or a - lease per GitTarget). A replica runs the watch sets for the GitTargets it owns. There is no shared - per-type pipeline to coordinate, no per-type handoff to lose. -- **Failover resumes from the cursor.** A replica taking over a GitTarget re-opens each watch with - `sendInitialEvents=true`, floored at the `(GVR, scope) → RV` cursor in Redis when one exists, or from - current state when the type has no cursor yet. Either way the replay-as-ADD reconstructs state - idempotently against Git; no durable object checkpoint is needed. -- **Correctness rests on replay, not on Redis durability.** Redis is a required dependency, but it - stores only two small things — the resume cursors and the attribution facts — and neither is a - *correctness* input. A missing or stale cursor never produces wrong state: the failover just degrades - to a full Mode-B replay (replay-as-ADD + mark-and-sweep) instead of an exact-RV delta resume. This is - what lets a GitTarget move between replicas freely. - -**The dedup tradeoff — accepted, and why.** Per-GitTarget watches mean that if two GitTargets both -claim `ConfigMaps` in the same scope, two ConfigMap watches run — duplicate API watch load and a -duplicate connection, ×N for a widely-claimed or wildcard type. We accept that inefficiency, and not -only for simplicity: **the replay makes per-GitTarget the natural unit.** Resume state is inherently -per-GitTarget — each watch's Mode-A cursor and its Mode-B mark-and-sweep are tied to *one* GitTarget's -Git tree, and two GitTargets sharing a `(GVR, scope)` can be in different resume modes at once (one -resuming from an exact RV, the other cold and needing a full replay). A single shared watch cannot be -both, so it would have to decouple the transport from the per-GitTarget replay/sweep and run N -independent mark-and-sweeps off one stream — exactly the fragility we are avoiding. So the watch -identity is `(GitTarget, GVR, scope)`, and each replay stays clean. - -**Future enhancement (explicit).** When overlap proves costly, share one watch per `(GVR, scope)` within -a replica and fan its events out to every owned GitTarget event stream that claims it (scope-filtered) — -the event-router fan-out seam already exists. The hard constraint is that it must preserve per-GitTarget -resume cursors and per-tree mark-and-sweep; it may deduplicate only the transport, never the replay -bookkeeping. Deferred until measured. - -## Attribution: an optional lookup table with slack - -Audit ingestion becomes an optional fact extractor that runs **only when audit attribution is enabled**. -It stores the smallest facts needed to name an author, not an authoritative object log. - -Fact shape (minimized): - -| Field | Purpose | -|---|---| -| `auditID` | diagnostics / dedupe | -| `user` / `impersonatedUser` | author candidate (human *or* service account) | -| `verb`, `subresource` | explain the write | -| `responseStatus.code`, `dryRun` | reject failures and non-persistent requests | -| GVR, namespace, name, UID (when available) | exact join keys | -| response object RV (when available) | exact watch-event match | -| request/stage timestamps | bounded time matching | - -The index is keyed for the join, strongest first: - -- `(group, resource, namespace, name, uid, responseRV)` — exact; -- `(group, resource, namespace, name, uid)` — for `deletecollection`, expand the response List into one - candidate per item and join the watch `DELETED` by UID, **without** RV equality; -- `(group, resource, namespace, name, responseRV)` — exact when UID absent; -- `(group, resource, namespace, name, time-bucket)` — weak, last resort. - -Retention is bounded by max audit delay + the attribution grace period — minutes, not hours. Old facts -are never needed for correctness because watch owns state. - -**The "slack."** A watch event waits a **bounded grace window** for a matching fact to arrive in the -index, then ships regardless. This is the one place watch-first still waits on audit, and it is what -makes "a late audit arrival must not rewrite a shipped commit" enforceable: we wait briefly *before* -shipping rather than rewrite afterwards. It is per-event, bounded, never a barrier, and it expires to -committer-authored rather than blocking state. - -### Confidence policy (strict — a wrong author is worse than no author) - -Because audit captures service-account activity too, a matched non-human actor is a *named* -attribution, not "unknown." Naming `system:serviceaccount:flux-system:kustomize-controller` is useful. - -| Confidence | When | Author | -|---|---|---| -| Exact (human) | watch UID/name/GVR/RV matches audit response; success; non-dry-run; real user | real user | -| Exact (service account) | same match strength; actor is an SA/controller | named SA (its own username) | -| Strong causal | scale subresource with matching parent objectRef + response RV (the parent watch event lands at exactly that RV) | real user / SA | -| Deletecollection | a `deletecollection` audit fact whose `responseObject` is a List is expanded into per-object candidates; each watch `DELETED` joins by **GVR + namespace + name + UID** — **not** RV (audit lists the pre-delete RV, watch the later delete RV; UID is the stable key) | real user / SA (reason `exact-deletecollection-item`) | -| Terminal delete (finalizer / cascade) | finalizer-removal or owner-ref cascade: the watch `DELETED` lands at a later RV and the actor that removed the object is a controller (corpus Rows 8/10) | committer, unless an *exact* match to the removing actor's audit fact exists (then that named SA) | -| Weak | same GVR/name/time only, missing RV/body, or multiple candidates | committer | -| Conflict | failure, dry-run, mismatched UID/RV, multiple users, stale | committer, with metric | -| Absent | attribution disabled, no audit fact, policy dropped the write, or no match | committer | - -`deletecollection` is the one delete case strong enough to attribute: implement it as a fact *expander* -— one collection fact becomes N per-object candidates keyed by GVR/namespace/name/UID — and join each -watch `DELETED` by UID. This holds only when audit captures the `responseObject` List; if the policy -gives request bodies only, or an aggregated API omits the response, it falls back to weak selector/time -matching or committer. Do **not** apply the expander to finalizer-delayed terminal deletes. - -The finalizer/cascade downgrade is deliberate, and it is exactly the case the strict rule exists for: -attributing a terminal `DELETED` to the human who *asked* for the delete looks right but commits the -file removal under the wrong actor. The human's `delete` only stamped `deletionTimestamp` (an earlier -event at an earlier RV); a controller removed the finalizer and caused the actual removal. Per "a wrong -author is worse than no author," that commits as committer unless we match the finalizer-removing patch -exactly — which usually names a service account, not the human. - -A matched actor — human **or** service account — is always named by its own username; there is no knob -to collapse a service account to the committer. The one thing the product exposes is a machine-readable -**reason code** on every outcome (`exact-user`, `exact-sa`, `exact-deletecollection-item`, `weak-no-rv`, -`conflict-multi-user`, `absent-attribution-disabled`, `absent-policy-dropped`, `expired`) so unknown -rates are explainable, not mysterious. - -## Commit windows and authors - -The current `BranchWorker` window accepts one `(author, GitTarget)` pair at a time -([open_window.go](../../internal/git/open_window.go), keyed on `event.UserInfo.Username`). Watch-first -needs only small adjustments: - -- exact-attributed events (human or named SA) use that actor as the author bucket — unchanged; -- everything else uses the configured committer identity; -- the bounded grace window may delay routing a watch event while a fact arrives; on expiry it routes as - committer and is **not** rewritten later; -- replay/reconcile changes use the committer identity; -- committer and attributed events must not be grouped into one real user's commit (the existing - no-blended-authors safety property is preserved). - -This means watch-first can produce *more* commits than audit-first when attribution is mixed, and -sometimes *fewer* when bursts/downtime collapse to current state. Both are acceptable and documented. - -## CommitRequest implications - -`CommitRequest` currently **fails closed** if it cannot attribute the requester -([commitrequest_controller.go](../../internal/controller/commitrequest_controller.go), -`attributionFailedMessage`). That does not fit a configured-author install. - -New behavior: - -- the controller-runtime watch on the `CommitRequest` object still triggers finalization; -- attribution is optional: with attribution enabled + a matching fact, the requester is named; otherwise - the request **finalizes as committer** with a status note that finalization happened without end-user - attribution; -- the request must never fail solely because attribution is disabled or no audit fact matched. - -If preserving the CommitRequest submitter matters without audit, that needs an explicit user field or a -signed request — Kubernetes object state alone does not identify the human who created it. - -## Operating modes - -| Mode | State source | Redis / audit | Author fidelity | Use | -|---|---|---|---|---| -| **Configured-author** | watch | Redis optional (no audit webhook); cold-replay without it | committer identity | simplest install, state mirror | -| **attributed-author** | watch | Redis + audit webhook | named user/SA on strong match; committer otherwise | recommended target | - -The product must not imply configured-author equals attributed-author mode, nor that attributed-author mode recovers -every author (the audit policy still bounds which writes carry a fact, and the CommitRequest submitter -is not recoverable from state alone). Both modes deliver a continuously updated Git mirror of desired -cluster state. - -## Keep / cut / reshape inventory - -What the big-bang touches, by package. Roughly **~5–6k LOC deleted**, replaced by **~1–1.5k** of -raw-watch ingestion plus a small attribution index and RV-resume cursor. - -| Package | Verdict | Notes | -|---|---|---| -| [internal/git](../../internal/git/) | **KEEP** | The whole downstream writer is source-agnostic. Only `open_window.go` (committer/SA/unknown buckets + grace) and `commit_request_attach*.go` (attribution optional) change. | -| [internal/manifestanalyzer](../../internal/manifestanalyzer/), [internal/manifestreport](../../internal/manifestreport/) | **KEEP** | Runtime manifest planning/diff/report. Source-agnostic. | -| [internal/typeset](../../internal/typeset/) | **KEEP, promote** | Followability/registry becomes the home of the relevance filter ("controller-owned → don't mirror"). | -| [internal/sanitize](../../internal/sanitize/) | **KEEP, promote** | Now mandatory on the hot path before every diff. | -| [internal/controller](../../internal/controller/) | **KEEP, reshape** | GitTarget reconcile opens watches instead of consuming a splice; CommitRequest flips fail-closed → finalize-as-committer. | -| [internal/reconcile](../../internal/reconcile/) | **KEEP** | `GitTargetEventStream` is the per-GitTarget seam the design leans on. | -| [internal/watch](../../internal/watch/) | **KEEP core, gut LIST/checkpoint half** | Keep discovery/catalog, `event_router`, `type_lifecycle`, `scope_resolve`, `watched_type_table`. Promote `watch_state.go`/`watch_compare.go` to the real path. Cut the `materialization.go` phase machine, `audit_tail.go`, `splice_snapshot.go`, `target_type_watermark.go`, and the heavy Redis objects-snapshot write in `type_objects_mirror.go` — but **keep that file's `sendInitialEvents` watch as the replay/resume path**, swapping the snapshot write for a tiny `(GVR,scope)→RV` cursor. | -| [internal/queue](../../internal/queue/) | **CUT ~90%** | Delete `redis_bytype_queue`, `redis_type_splice`, `redis_objects_snapshot`, `redis_audit_queue`, `redis_watch_splice`, `redis_watch_stream`, `subresource_translate`. Keep only the attribution-fact pieces (`commitrequest_author` folds into the new index). | -| internal/gate | **CUT** | Demand-gating existed to bound the per-type mirror. No mirror, no gate. | -| [internal/webhook](../../internal/webhook/) | **SHRINK hard** | `audit_joiner.go` (body-joining) → **gone**. `audit_handler.go` → shrinks to fact extraction → index. `admission_allow_handler.go` stays as the future-policy seam. | -| [internal/auditutil](../../internal/auditutil/) | **SHRINK** | Keep only what feeds attribution facts. | -| giteaclient, ssh, sshsig, telemetry, types, rulestore, mutationlab | **KEEP** | Provider/credential/signing/metrics/test infra, all source-agnostic. | - -## Change plan (big-bang, deletion-forward) - -Done as a single replacement on a branch; mostly deletion, so it should be quick relative to its -footprint. Validation gate is the full e2e suite green (`task lint` → `task test` → `task test-e2e`), -with the watch-vs-audit object-set diff used as the parity check during bring-up. - -**Stage 1 — Raw-watch ingestion as the GitTarget event source.** -Build a per-GitTarget watch manager: on GitTarget reconcile, resolve claimed ∩ followable -`(GVR, scope)` and open one raw watch per `(GVR, scope)` with `sendInitialEvents=true`. Persist -`(GVR, scope) → last RV` to Redis (when present) and resume from it; on `410`/disconnect, re-open with a -fresh replay. Wire each event through the relevance filter (sanitize → followability → no-op diff) into -the existing `GitTargetEventStream` → BranchWorker. **No timer sweep**; deletes are reconciled by the -Mode-A delta resume or the Mode-B mark-and-sweep on replay (see -[Resume and recovery](#resume-and-recovery)). This replaces the materialization phase machine + audit -tail + splice as the desired-set source. *Net: new code is small; it plugs into existing seams, and -`type_objects_mirror.go` already does the `sendInitialEvents` watch.* - -**Stage 2 — Optional attribution.** -When attribution is enabled: extract minimal audit facts in the (shrunk) audit handler → Redis index -with TTL; add the conservative resolver with the bounded grace window. When attribution is disabled: -skip the audit handler and the attribution index entirely (Redis still backs the resume cursors); -commits are committer-authored. Flip `CommitRequest` to finalize-as-committer without attribution. - -**Stage 3 — Demolition.** -Delete the audit-as-correctness machinery now that nothing imports it: `internal/queue` (minus the -moved attribution bits), `internal/gate`, `webhook/audit_joiner.go`, the materialization phase machine, -`audit_tail.go`, `splice_snapshot.go`, `target_type_watermark.go`, the objects snapshot/checkpoint, and -the late-lane/divert/nudge code. Shrink `audit_handler.go` to fact extraction. - -**Stage 4 — Config, wiring, docs.** -Note: the shipped behavior later relaxed this. Redis is now **optional** in `cmd/main.go` (an empty -`--redis-addr` is valid and runs configured-author with cold-replay; the readiness gate only applies -when Redis is configured). The audit webhook / attribution remains optional via -`--author-attribution=false`; when Redis *is* configured it still backs the resume cursors. Remove the dead audit-stream flags -(`--audit-redis-max-len`, `--audit-bytype-*`, `--watch-state-stream`, body TTL/wait, etc.). Update the -Helm chart and `config/` so the audit webhook is opt-out while Redis stays a hard dependency. Rewrite -`docs/architecture.md` to the watch-first model and document the two operating modes and the -history-granularity guarantee. - -## Reliability rules (non-negotiable) - -1. Audit attribution must never be required for object correctness, and Redis durability must never be - a correctness input — a missing or stale resume cursor degrades to a full Mode-B replay, never to - wrong state. (Redis is still a required dependency; it just isn't what guarantees correctness.) -2. A watch event with no confident audit match must still write state (as committer). -3. A failed, rejected, or dry-run audit fact must never create state. -4. Conflicting attribution facts produce committer, not "first wins." -5. A later audit arrival must not rewrite a commit already shipped as committer. -6. A resume replay's mark-and-sweep is authoritative only after it reaches `initial-events-end`; a - partial replay must not be treated as complete current state. Deletes missed while not watching are - reconciled by the Mode-A delta resume or the Mode-B mark-and-sweep — never silently dropped. -7. Every attribution outcome carries a machine-readable reason code. -8. Unknown-author rate is visible by GitTarget, GVR, verb/event type, and reason. -9. The relevance filter is observable: how many watch events were dropped as no-op/noise, by GVR. +# Watch-first ingestion architecture — retired design record + +> **Historical context only.** The current implementation contract is +> [`../architecture.md`](../architecture.md). This short record replaces the superseded rollout plan that +> previously occupied this path, so older links remain useful without presenting abandoned design work as live +> behaviour. + +The shipped model is watch-first: one raw watch per `(GitTarget, GVR, scope)` supplies object state. A +watch’s initial replay and any recovery replay establish the desired state; Git is the persisted mirror. +Audit events never create or repair object state. They are optional evidence used only to name a live +mutation’s Git author. + +## Current invariants + +- A branch worker serializes writes. A commit window belongs to one GitTarget and either one named actor or + an unnamed author state. +- With attribution disabled, and for replay/resync writes, Git uses the configured committer as author. +- With attribution enabled, a strong audit match names the authenticated actor. A weak, conflicting, late, + or missing fact produces `unknown (attribution unresolved) `. + A late fact never rewrites an already-created commit. +- Redis is optional in configured-author mode. It is required when audit-backed attribution is enabled and + can also retain watch-resume cursors. +- A `CommitRequest` captures its command submitter at validating admission when that optional Redis-backed + path is enabled. A request without a captured actor claims no actor; the matching window, not the request, + determines the final Git author. + +For the live details, use the [watch ingestion and reconcile](../architecture.md#watch-ingestion-and-reconcile), +[attribution](../architecture.md#attribution-audit-names-the-author-never-the-state), and +[CommitRequest finalize](../architecture.md#commitrequest-finalize) sections. ## Metrics -| Metric | Why | -|---|---| -| `gitopsreverser_watch_events_total{gvr,type,outcome}` | watch volume and drops | -| `gitopsreverser_watch_events_filtered_total{gvr,reason}` | relevance-filter behavior (no-op/status/runtime) | -| `gitopsreverser_watch_restarts_total{gvr,scope,reason}` | watch stability / `410 Gone` pressure (replay reconnects) | -| `gitopsreverser_watch_replay_seconds{gvr,scope}` | time to reach `initial-events-end` on (re)connect — resume cost | -| `gitopsreverser_attribution_total{result,reason,gvr}` | exact-user / exact-sa / weak / conflict / absent / expired | -| `gitopsreverser_attribution_wait_seconds{result}` | grace-window latency cost | - -## Decision table - -| Question | Answer | -|---|---| -| Source of object state? | WATCH with `sendInitialEvents` replay, per GitTarget. No separate LIST/checkpoint. | -| Is audit ever required for committing state? | No. It only changes the author. | -| Is Redis required? | Yes — a hard dependency in every mode. It holds each GitTarget's RV resume cursors (and, with attribution on, the audit facts) and underpins HA / the durable branch-worker queue. It is not a *correctness* input: a missing cursor just forces a full Mode-B replay. | -| Watch ownership granularity? | Per GitTarget. HA = assign GitTargets to replicas; failover resumes from the Redis RV cursor (or replays). | -| Periodic LIST / drift sweep? | No *timer* sweep. Deletes are reconciled by the Mode-A delta resume (exact RV) or a **mark-and-sweep on replay** (Mode B); the sweep fires on watch re-establishment, never hourly. | -| Dedup watches across GitTargets? | No — one watch per `(GitTarget, GVR, scope)`, because resume cursors and mark-and-sweep are per-GitTarget. Sharing the transport is an explicit future enhancement, deferred. | -| Guess an author from timing alone? | No. Weak matches commit as committer. | -| Does watch-first preserve per-mutation history? | No across gaps — it collapses to current state. State mirror with opportunistic history. | -| Migration shape? | Big-bang: remove the audit-as-correctness pipeline, don't keep it in parallel. | -| What gets harder? | The relevance filter (now product code on the hot path) and honest, deterministic attribution. | - -## Bottom line - -Watch is the better source for persisted object state, and the audit-as-correctness pipeline carries -the most incidental complexity (ordering, late-lane, demand-gating, the materialization phase machine). -Making watch the only state source per GitTarget deletes that complexity, makes the audit webhook -optional (Redis stays required for resume cursors), and turns shallow/aggregated audit events from -correctness problems into attribution limitations. - -Two prices are accepted in exchange: the audit *policy's* relevance filtering moves into product code -on the hot path, and authorship becomes probabilistic with per-observation (not per-mutation) history -granularity. The product stays honest about both — name a user or service account when the evidence is -strong, otherwise commit as the committer and say why, and filter controller churn deliberately rather -than relying on a cluster's audit policy to do it. +Metric names and operational interpretation are maintained in +[`../interpreting-metrics.md`](../interpreting-metrics.md). The telemetry design backlog is +[`../design/metrics-observability-plan.md`](../design/metrics-observability-plan.md); a planned metric is not +evidence that the runtime currently emits it. diff --git a/docs/future/design-commit-request-phase-2.md b/docs/future/design-commit-request-phase-2.md deleted file mode 100644 index cb69c0f5..00000000 --- a/docs/future/design-commit-request-phase-2.md +++ /dev/null @@ -1,99 +0,0 @@ -# Phase 2: `CommitRequest` — deferred work - -> Status: proposed. Follow-up to [design-commit-request-api.md](../spec/commitrequest-design.md). -> Date: 2026-05-18 - -The first cut of `CommitRequest` is intentionally minimal. This doc tracks what was deliberately -left out, with enough analysis that phase 2 does not start cold. The object-lifecycle question is -the largest gap and is treated first. - -## 1. Object lifecycle — when are `CommitRequest`s deleted? - -This is the biggest open question. Today **nothing deletes `CommitRequest` objects.** Every save -leaves a permanent object in the namespace. With `generateName` and a frontend that saves often, -a namespace accumulates them without bound. - -There are two distinct populations, and they need different answers. - -### a. Terminal objects (`Committed` / `NoOpenWindow` / `Failed`) - -These have done their job. They are useful only as a short-lived receipt the frontend reads once. -Options: - -- **TTL-after-finished controller**, modeled on Job's `ttlSecondsAfterFinished`. A controller - deletes terminal objects a configurable interval after `status.observedTime`. Owns the clock - server-side and survives a frontend that never comes back. Recommended. -- **Frontend deletes after reading status.** Pushes the clock to the client. Clean when the - client is well-behaved; leaks on every crashed or forgetful client. Not sufficient on its own. -- **Owner references.** GC would cascade if a save were owned by some parent object — but there - is no natural owner here, so this does not really apply. - -### b. Objects stuck in `WaitingForAuditEvent` - -This is the subtle one. If the audit pipeline is degraded or the audit event is dropped, the -object **never reaches a terminal phase** — so a TTL keyed on "terminal + `observedTime`" will -never collect it. These objects are both a leak *and* a silent failure: the user's save did -nothing and nothing says so. - -A TTL-after-finished controller must therefore be paired with a **max-age sweep**: any -`CommitRequest` older than some bound that is still `WaitingForAuditEvent` should be transitioned -to `Failed` ("audit event never observed") and then become eligible for normal terminal cleanup. -The bound must be safely longer than worst-case audit-pipeline lag. - -### Decisions phase 2 must make - -- Who owns the delete (a TTL controller — recommended) and the default TTL. -- The separate max-age for stuck `WaitingForAuditEvent`, and whether it flips the object to - `Failed` or just deletes it. Flipping to `Failed` is preferable — it stays observable. -- Whether the TTL is global (a Helm value) or per-object (a `spec.ttlSecondsAfterFinished`-style - field), or both. -- Whether the controller emits a metric/event when it reaps a stuck `WaitingForAuditEvent` object, - so the underlying audit-pipeline problem is visible (see §4). - -## 2. Retry for transient finalize failures - -Today any finalize error — including the transient `ErrFinalizeQueueFull` — becomes a terminal -`Failed`, and the caller's custom message is lost (the silence timer still commits the edits later -with a generated message). For a saturated-queue case specifically, a bounded retry before giving -up would preserve the custom message. - -This needs care: the audit message is ACKed exactly once with no redelivery, so a retry must -happen within the consumer's processing of that one event, or via an explicit requeue mechanism. -Worth doing only if queue-full turns out to be common in practice — gate it on the metrics from -§4. - -## 3. Multi-target "save everything I edited" - -`spec.gitTargetRef` is required precisely so `status` stays flat (one SHA, no list). A bare -"finalize every open window I have in this namespace" form is a plausible UX, but it makes -`status` a list and the matching logic non-trivial. Deferred until there is a concrete need. - -## 4. Observability - -There are no metrics specific to `CommitRequest` yet. Phase 2 should add at least: - -- a counter of outcomes by phase (`committed` / `no_open_window` / `failed`); -- a counter (or gauge) of objects reaped while still `WaitingForAuditEvent` — the - audit-pipeline-degraded signal from §1b; -- finalize latency (object creation → terminal phase). - -## 5. Helm: ship an audit-policy fragment - -The consumer only needs `Metadata`-level audit on `commitrequests` `create`. The repo's e2e -policy captures this via a catch-all `create` rule, but operators applying their own restrictive -audit policy have to derive the rule themselves. The chart should ship a recommended fragment, or -document it prominently, so `CommitRequest` works on a cluster with a narrow audit policy. - -## 6. Smaller items - -- **Cluster-scoped use.** `CommitRequest` is namespaced only. No current need for a cluster-scoped - variant; noted for completeness. - -## Suggested order - -1. **Lifecycle** — the TTL-after-finished controller **and** the stuck-`WaitingForAuditEvent` - max-age sweep (§1). This is the only item that is a correctness / operational problem rather - than a nicety. -2. **Outcome metrics** (§4) — needed to even know whether the retry work in §2 is worth doing. -3. **Audit-policy fragment** in the Helm chart (§5). -4. Revisit **retry** (§2) and **multi-target** (§3) only if real usage asks for them. diff --git a/docs/future/idea-unify-pending-write-kinds.md b/docs/future/idea-unify-pending-write-kinds.md index e362bbd1..1d20374a 100644 --- a/docs/future/idea-unify-pending-write-kinds.md +++ b/docs/future/idea-unify-pending-write-kinds.md @@ -23,12 +23,11 @@ The four use cases in [commit-window-refactor.md](../spec/commit-window-refactor 1. **Burst collapse.** Per-event-only behavior (atomic is one commit by definition). Unification neither helps nor breaks this. -2. **Honest authorship.** Reconcile snapshots are operator-authored; audit-driven - commits carry the audit user. Today this is enforced by *type*: the atomic path - never reads `event.UserInfo`. Under unification it becomes a *convention*: the - git layer falls back to committer-as-author when `event.UserInfo.Username == ""`, - so producers must not populate `UserInfo` for snapshot batches. Same observable - result, weaker compile-time guarantee. +2. **Honest authorship.** Reconcile snapshots are configured-author; live commits carry + either a resolved audit actor or the explicit unresolved identity. Today this is enforced by + *type*: the atomic path is `AttributionNotAttempted` and never reads `event.UserInfo`. Under + unification it becomes a *convention*: producers must retain the correct + `AttributionOutcome` for snapshot batches. Same observable result, weaker compile-time guarantee. 3. **Target isolation.** Single-target on both paths today. Unaffected. 4. **Safe replay.** Both paths carry resolved metadata. Unaffected. @@ -40,7 +39,7 @@ that one is comfortable being a convention rather than a type-system invariant. | Concern | Per-event / grouped | Atomic | Survives unification? | |---|---|---|---| -| Authorship source | `event.UserInfo.Username` | Operator (request-level, ignores `UserInfo`) | Yes, via "empty `UserInfo` → committer-as-author" fall-through at the git layer | +| Authorship source | resolved actor or explicit outcome | Configured author (`AttributionNotAttempted`) | Yes, by preserving `AttributionOutcome` rather than relying on an empty username | | Commit message | Template render | Caller-provided string | Yes, via `CommitMessage` non-empty as the dispatch signal | | Finalize timing | Author/target change, silence, byte cap, shutdown | Request boundary | Yes, by adding "request has `CommitMessage` set" as a finalize trigger | | Byte-cap behavior | Mid-stream finalize is fine | Must not split one snapshot across commits | Yes, via the rule "finalize the *next* organic window early; don't actively split a request" | diff --git a/docs/interpreting-metrics.md b/docs/interpreting-metrics.md index 496fdcd2..a0eb34b8 100644 --- a/docs/interpreting-metrics.md +++ b/docs/interpreting-metrics.md @@ -87,8 +87,16 @@ signals. Background: [architecture.md → Git Write Architecture](architecture.m `commits_total` carries the **`BranchWorker`'s** `{provider_namespace, provider_name, branch, author_kind}` identity, not a GitTarget: one worker can serve several GitTargets sharing a provider+branch, coalescing their writes into one commit batch, so -the worker is the honest attribution unit. `author_kind` is `user`, `serviceaccount`, or `committer`; -reconcile/resync commits and unattributed events use `committer`. The namespace/name keys are +the worker is the honest attribution unit. `author_kind` is `user`, `serviceaccount`, `committer`, or `unresolved`; +reconcile/resync commits and configured-author mode use `committer`. + +**`unresolved` is the one to watch.** It means attribution RAN and did not name an actor, so +the commit carries the `unknown (attribution unresolved)` author instead of a person. It is +deliberately not folded into `user` (which would make a lost actor look like a named one, so a +degrading attribution path would read as an improving one) nor into `committer` (which would +hide it among legitimately unattributed reconcile writes). For a live mutation that should be +attributable, a non-zero and growing `unresolved` share means the audit-attribution configuration +or delivery path needs investigation. The namespace/name keys are **prefixed on purpose** — a Prometheus pod scrape with `honor_labels=false` overwrites a bare `namespace` attribute with the scraping pod's namespace, so a per-provider `namespace` selector would silently match nothing. The same reasoning applies to @@ -242,10 +250,10 @@ histogram_quantile(0.95, ``` **Does audit actually attribute watch events?** Match coverage is the share of resolutions that named an -actor (human or service account) rather than falling back to the committer: +actor (human or service account) rather than producing an explicit unresolved author: ```promql -sum(rate(gitopsreverser_attribution_resolutions_total{result=~"exact_.*"}[5m])) +sum(rate(gitopsreverser_attribution_resolutions_total{result=~"exact_.*|weak"}[5m])) / sum(rate(gitopsreverser_attribution_resolutions_total[5m])) ``` @@ -257,21 +265,20 @@ sum(rate(gitopsreverser_attribution_resolutions_total[5m])) | `exact_user` | Exact UID+resourceVersion match for a human user. | | `exact_serviceaccount` | Exact UID+resourceVersion match for a named service account. | | `weak` | Non-exact match, such as UID-only or RV-only. | -| `conflict` | Multiple authors wrote facts for the selected join key. | -| `expired` | A fact existed but aged out before the watch event joined it. | -| `absent` | No fact or expiry evidence matched. | +| `absent` | No usable fact matched before the grace window elapsed. The resulting live commit is authored as `unknown (attribution unresolved)`. | **Is the grace window paying for itself?** Misses waiting near the configured grace window mean the operator is delaying commits without finding facts: ```promql histogram_quantile(0.95, - sum by (le, result) ( - rate(gitopsreverser_attribution_resolution_wait_seconds_bucket{result=~"absent|expired"}[5m]))) + sum by (le) ( + rate(gitopsreverser_attribution_resolution_wait_seconds_bucket{result="absent"}[5m]))) ``` -**Is the Redis fact index healthy?** Facts should be written and later matched; a high `late` or -`expired_unmatched` rate points at timing mismatch between audit and watch: +**Is the Redis fact index healthy?** Facts should be written and later matched; a high rate of +`op="written"` alongside `result="absent"` points at a timing, key, or source-identity mismatch +between audit and watch: ```promql sum by (op) (rate(gitopsreverser_attribution_fact_events_total[5m])) diff --git a/docs/spec/README.md b/docs/spec/README.md index 58ec3b38..cef1ba89 100644 --- a/docs/spec/README.md +++ b/docs/spec/README.md @@ -37,6 +37,6 @@ If you change one of these behaviours, change the document in the same commit. | [`sops-single-file-no-multidoc.md`](sops-single-file-no-multidoc.md) | one encrypted file is one document | | [`scale-subresource-audit-rehydration.md`](scale-subresource-audit-rehydration.md) | `/scale` → bounded field patch; every other subresource ignored | | [`commit-window-refactor.md`](commit-window-refactor.md) | one grouped commit = one (author, GitTarget) | -| [`commitrequest-multi-finalize-design.md`](commitrequest-multi-finalize-design.md) | why `MaxConcurrentReconciles=1` on the CommitRequest controller | +| [`commitrequest-design.md`](commitrequest-design.md) | how a request binds to a same-actor commit window and reports its outcome | | [`gittarget-isolation-on-rule-change.md`](gittarget-isolation-on-rule-change.md) | a rule change on target A never touches target B | | [`audit-readiness-probe-plan.md`](audit-readiness-probe-plan.md) | why liveness must never depend on Redis | diff --git a/docs/spec/commitrequest-admission-authorship.md b/docs/spec/commitrequest-admission-authorship.md index 838a8f25..64ed1039 100644 --- a/docs/spec/commitrequest-admission-authorship.md +++ b/docs/spec/commitrequest-admission-authorship.md @@ -1,613 +1,66 @@ ---- -title: CommitRequest authorship from admission — a command, captured at the source -status: implemented (investigate branch) -date: 2026-06-29 -related: - - ../finished/design-commit-request-api.md - - ../finished/commitrequest-attribution-divert-reliability.md - - watch-first-ingestion-architecture.md - - watch-event-ordering-and-attribution-grace.md - - reconcile-triggering.md - - redis-key-schema-v3.md ---- - # CommitRequest authorship from admission -> **spec** — current behaviour. The code depends on this document; change one, change the other. Index: [`../INDEX.md`](../INDEX.md) - -> **Thesis.** A `CommitRequest` is a **command** ("save now, as me"), not a piece of -> mirrored state. Its author should be captured where the command is issued — a -> **validating admission webhook** dedicated to our own command kinds (the -> *validate-operator-types* webhook) — into its **own Redis corner**, and read back by the -> controller with **no wait**. This is the opposite of how we attribute mirrored -> resources (audit facts joined to watch events), and deliberately so: the two have -> different provenance and must not share a store. This design **replaces** the -> audit-sourced CommitRequest attribution rather than layering on top of it. - -This is the forward-facing companion to -[commitrequest-attribution-divert-reliability.md](commitrequest-admission-authorship.md), -which adopted "Option C" (capture the author as a keyed fact at audit ingestion). -Option C was the right *shape* — a divert-immune keyed fact, not an ordered-stream -scan — but it kept the **wrong source** (the audit event) and therefore kept a -**wait**. Watch-first has since severed the only other thing the audit event did for -a CommitRequest (timing — now the close-delay collect window), so the audit event's -last remaining job here is to carry one string. We can get that string earlier, more -reliably, and without the audit webhook at all. +> **spec** — current behaviour. The code depends on this document; change one, change the other. +> Index: [`../INDEX.md`](../INDEX.md) ---- +A `CommitRequest` is a command to close an existing GitTarget commit window. Its submitter is captured +by the `/validate-operator-types` validating admission webhook, while mirrored-resource attribution is +separately derived from kube-apiserver audit facts. The two paths answer different questions and must not +be conflated. -## 1. Two kinds of authorship, two provenances +## Contract -We answer two different questions with two different mechanisms. Keeping them -separate is the whole point. - -| | **Mirrored-resource attribution** | **Command authorship** | +| Concern | CommitRequest submitter | Mirrored-resource author | |---|---|---| -| Question | "Who performed mutation X that *persisted* at RV N?" | "Who *issued* this command?" | -| Source | Audit event (`ResponseComplete`, 2xx, RV changed) | Admission request `userInfo` | -| Timing vs persist | **After** persist (proof of persistence) | **Before** persist (an assertion) | -| What makes it trustworthy | The audit accept gate ([`classifyAuditIngress`](../../internal/webhook/audit_handler.go)) | The controller only finalizes **persisted** objects (the informer never sees an un-persisted command) | -| Join | identity **+ resourceVersion**, matched within a grace window | identity 1:1 (`uid`), no join, no window | -| Store | `AttributionIndex` (`AuthorFact`, conflict markers, RV/auditID) | dedicated `CommandAuthorStore` under `author:v1:command` (this doc) | -| Lifetime | the watch-event join grace | admission → finalize (effectively immediate, §2) | - -The reliability argument the project built the audit path on — *admission sees -attempts, not persistence* ([mutation-capture-lab-design.md](mutation-capture-lab-design.md), -rows 11–13) — **still holds for mirrored state and is unchanged.** It does not bite -a command, because we never write the command object into git. We extract one thing -from it (the author), and whether that extraction is ever *used* is gated downstream -by persistence: the controller reconciles off the informer, which only ever delivers -objects that actually landed in etcd. The persistence guarantee is not lost — it is -**relocated** from the audit event to the controller's watch. - -### Why not reuse the attribution index (the "don't pretend it came from audit" rule) - -Folding the admission assertion into the audit `AuthorFact` -([attribution_index.go:88-102](../../internal/queue/attribution_index.go#L88-L102)) -would silently inherit semantics that are wrong for a command: - -- **Conflict detection.** `storeFactKey` writes a `Conflict` marker when two users - hit the same key ([attribution_index.go:457-473](../../internal/queue/attribution_index.go#L457-L473)). - A command has exactly one creator; that machinery is dead weight at best and a - corruption vector at worst. -- **RV / auditID / stageTimestamp.** All empty for an admission capture — the fields - exist to disambiguate a *post-persist join*, which we do not do. -- **TTL semantics.** The fact TTL is tuned for the watch-join grace; the command - record's TTL is purely for cleanup (§3). -- **Provenance blur.** A reader could no longer tell "proven persisted, by audit" - from "asserted at admission, persistence enforced by the watch." Those are - different trust models and should be legible as different. - -So: a **separate store, separate author subnamespace, separate record type.** It is -not cosmetic separation — it keeps each path's invariant clean. The Redis naming plan -is `author:v1:audit` for persisted-resource facts and `author:v1:command` for -admission-captured commands ([redis-key-schema-v3.md](../finished/redis-key-schema-v3.md)). - ---- - -## 2. The authorship invariant (why there is no wait) - -> **By the time a command object is visible to anything — `kubectl get`, the informer, -> the controller — its author has already been recorded.** - -This falls out of admission ordering: - -1. A client `CREATE`s a `CommitRequest`. -2. The apiserver resolves `generateName → name`, assigns `metadata.uid`, runs - mutating admission, then runs **validating** admission — our validate-operator-types handler. -3. Our handler **synchronously writes** `{uid → author}` to Redis and returns - `Allowed`. The write completes *before* `Handle` returns. -4. *Only then* does the apiserver persist the object to etcd and return to the client. -5. *Only after persist* does the informer deliver it and the controller reconcile. - -Step 3 strictly precedes steps 4–5. So when the controller first sees the object, the -record is **present-or-never**: present if the webhook ran, never if it did not (the -validate-operator-types webhook is not configured, or a Redis write failed under -`failurePolicy: Ignore`). **Waiting cannot help** — there is no asynchronous arrival -to wait for, unlike an audit event that trickles in seconds after persist. This is the -precise reason the 60 s `commitRequestAttributionTimeout` wait -([commitrequest_controller.go:77-82](../../internal/controller/commitrequest_controller.go#L77-L82)) -disappears: it was an artifact of the audit event's *post-persist* timing, and that -timing is gone. - -Contrast with audit, where the order is inverted (persist → audit event → maybe -seconds later → fact in Redis), which is exactly why the audit-sourced controller -*had* to poll and fall back on a timeout. - -**This does not weaken the "prior edits already landed" guarantee.** That was an -audit-stream-ordering property in the audit-first era; in watch-first it is the -**close-delay collect window** (`finalizeAt = receipt + CloseDelaySeconds`, -[commit_request_attach.go:87-90](../../internal/git/commit_request_attach.go#L87-L90)), -which is independent of where the author string comes from. Moving authorship to -admission touches the *who*, never the *when*. - ---- - -## 3. The dedicated Redis corner - -A small store hung off the always-present `RedisStore` -([redis_store.go](../../internal/queue/redis_store.go)), mirroring how -`AttributionIndex` and the watch-cursor store share the one connection but own -disjoint key namespaces. It lives under the same top-level `author` Redis domain as -audit-sourced authorship, but in the separate `command` subfamily because its -provenance and lookup semantics are different. It is generic over command kinds -(keyed by `uid` alone, which is globally unique) so a future command CRD reuses it -unchanged. - -```go -// internal/queue/command_author_store.go - -// commandAuthorKeySuffix namespaces the keys this store owns. It shares the top-level -// author domain with audit-sourced resource facts, but the command subfamily has a -// different provenance and no grace-window join. -const commandAuthorKeySuffix = ":author:v1:command:" - -// commandAuthorRecordTTL bounds a captured authorship record. It is NOT tuned to cover -// any wait: by the authorship invariant (§2) the record is written before the object -// exists, and the controller reads it on the first reconcile after persist — typically -// sub-second. The TTL exists ONLY so an orphan record (a command object deleted before -// its reconcile) self-cleans. It is a fixed internal constant, deliberately not a flag: -// there is nothing to tune. An hour is generous headroom for a slow reconcile backlog -// or a leader failover. -const commandAuthorRecordTTL = time.Hour - -// CommandAuthor is the minimal authorship captured at admission for one command object. -// It carries only what a git commit author needs — no RV, no auditID, no conflict bit: -// this is a 1:1 command capture, not a post-persist join. -type CommandAuthor struct { - Author string `json:"author"` - DisplayName string `json:"displayName,omitempty"` - Email string `json:"email,omitempty"` - RequestedAt string `json:"requestedAt,omitempty"` // RFC3339Nano, for lag metrics/debug -} - -// CommandAuthorStore records and reads command authorship. It shares RedisStore's -// connection but is wired whenever the validate-operator-types webhook is enabled — -// independent of --author-attribution, which only governs mirrored-resource attribution. -type CommandAuthorStore struct { - client *redis.Client - ttl time.Duration -} - -// RecordCommandAuthor is the admission-side write: capture the authenticated submitter -// the instant a command CREATE is admitted, before it persists. Last-write-wins (a -// CREATE fires admission once; a retried admission re-asserts the same user). -func (s *CommandAuthorStore) RecordCommandAuthor( - ctx context.Context, uid types.UID, author CommandAuthor, -) error { - raw, err := json.Marshal(author) - if err != nil { - return fmt.Errorf("marshal command author: %w", err) - } - return s.client.Set(ctx, s.key(uid), raw, s.ttl).Err() -} - -// LookupCommandAuthor is the controller-side read, keyed by the persisted object's UID. -// ok=false means no record was captured — the validate-operator-types webhook is not -// configured (or a best-effort write missed) — and the controller finalizes as the -// committer, immediately, with AuthorAttributed=False (§5). -func (s *CommandAuthorStore) LookupCommandAuthor( - ctx context.Context, uid types.UID, -) (CommandAuthor, bool) { - raw, err := s.client.Get(ctx, s.key(uid)).Bytes() - if err != nil { - return CommandAuthor{}, false - } - var a CommandAuthor - if json.Unmarshal(raw, &a) != nil || a.Author == "" { - return CommandAuthor{}, false - } - return a, true -} - -// key identifies the command by UID alone — globally unique, like the watch cursor key, -// so namespace/name/kind would be redundant (kept out for a tight key). -func (s *CommandAuthorStore) key(uid types.UID) string { - return keyPrefix + commandAuthorKeySuffix + escapeKeyField(string(uid)) -} -``` - -And the builder on `RedisStore`, alongside `AttributionIndex(...)`: - -```go -// CommandAuthorStore builds the command-authorship store on this connection. Wire it -// when the validate-operator-types webhook is enabled; it does not depend on attribution. -func (s *RedisStore) CommandAuthorStore() *CommandAuthorStore { - return &CommandAuthorStore{client: s.client, ttl: commandAuthorRecordTTL} -} -``` - -> **Keying by UID only.** The controller reads by the persisted object's `uid`, which -> admission already assigned. `generateName` is a non-issue: the server-assigned name -> *and* uid are both present in `request.Object` by the time validating admission runs, -> so we never have to recover a name from a response body the way the audit path did. - ---- - -## 4. The validate-operator-types admission handler - -One handler on its own path, `/validate-operator-types`, that recognizes **our own command -kinds** and captures the submitter of each. It is not CommitRequest-specific by -construction — it dispatches on a small registry of command resources, so a future -command CRD is one table entry plus one webhook rule, not a new handler. It replaces -the no-op [`AdmissionAllowHandler`](../../internal/webhook/admission_allow_handler.go) -*for this purpose*; the catch-all `*` observer in -[validating-webhook.yaml](../../config/webhook/validating-webhook.yaml) is a separate, -future policy extension point and stays as-is. - -```go -// internal/webhook/internal_commands_handler.go - -const ValidateOperatorTypesPath = "/validate-operator-types" - -// commandKinds is the registry of our own command CRDs whose submitter we capture at -// admission. Adding a command kind is one entry here plus one rule in the webhook -// config — the handler body does not change. -var commandKinds = map[metav1.GroupResource]struct{}{ - {Group: "configbutler.ai", Resource: "commitrequests"}: {}, - // future: {Group: "configbutler.ai", Resource: ""}: {}, -} - -// ValidateOperatorTypesHandler captures the authenticated submitter of one of our command -// objects into the CommandAuthorStore and always allows. It is pure observation with a -// single side effect (a Redis upsert); it never rejects. -type ValidateOperatorTypesHandler struct { - Store *queue.CommandAuthorStore - Decoder admission.Decoder -} - -func (h *ValidateOperatorTypesHandler) Handle(ctx context.Context, req admission.Request) admission.Response { - gr := metav1.GroupResource{Group: req.Resource.Group, Resource: req.Resource.Resource} - if _, ok := commandKinds[gr]; !ok { - return admission.Allowed("not a command kind") // belt-and-suspenders; rules already scope us - } - // Dry-run never persists, so the controller will never read this record — and we - // declare sideEffects: NoneOnDryRun, so we must honor it. - if req.DryRun != nil && *req.DryRun { - return admission.Allowed("dry-run: not recorded") - } - - // metadata.uid identifies the object (not req.UID, which identifies the review). - obj := &metav1.PartialObjectMetadata{} - if err := h.Decoder.Decode(req, obj); err != nil { - // Cannot key the record without a uid; allow the command, fall back to committer. - log.FromContext(ctx).Error(err, "decode command object for authorship", "name", req.Name) - return admission.Allowed("undecodable: not recorded") - } - - author := queue.CommandAuthor{ - Author: req.UserInfo.Username, // effective (impersonated) user — apiserver resolved it - DisplayName: firstExtraValue(req.UserInfo.Extra, displayNameExtraKey), - Email: firstExtraValue(req.UserInfo.Extra, emailExtraKey), - RequestedAt: time.Now().UTC().Format(time.RFC3339Nano), - } - if author.Author == "" { - return admission.Allowed("no user to attribute") // anonymous never reaches a persisted CREATE - } - - // Synchronous, best-effort: the write completes before we return, so it is present - // before the object is visible (the authorship invariant, §2). A failure degrades to - // the committer — never block the command. - if err := h.Store.RecordCommandAuthor(ctx, obj.UID, author); err != nil { - log.FromContext(ctx).Error(err, "record command author; finalizing as committer", - "namespace", req.Namespace, "name", req.Name) - } - return admission.Allowed("author recorded") -} -``` +| Source | validating admission request `userInfo` | post-persist audit fact joined to watch event | +| Key | CommitRequest UID | source provider, GVR, UID, and resourceVersion | +| Timing | written before the object persists | waits up to `--author-attribution-grace` after the watch event | +| Miss | request claims no actor | watch commit has the explicit unresolved author | -The matching `ValidatingWebhookConfiguration` rule — **narrow, `Ignore`, and packaged -in the Helm chart** (this is product behavior now, not the e2e-only `*` observer): +The admission handler records the authenticated submitter in the Redis command-author store and always +allows the request. It is deliberately best-effort: a Redis failure or an unavailable webhook must not +reject a user's command. The controller reads the record when the persisted `CommitRequest` is first +reconciled. The result is **present-or-never**; waiting cannot create a record that admission did not write. -```yaml -webhooks: - - name: validate-operator-types.configbutler.ai - clientConfig: - service: { name: gitops-reverser-webhook, namespace: , path: /validate-operator-types, port: 9443 } - rules: - - operations: [CREATE] - apiGroups: [configbutler.ai] - apiVersions: [v1alpha3] - resources: [commitrequests] # one line per command kind - failurePolicy: Ignore # authorship is best-effort; never block the command - sideEffects: NoneOnDryRun # Redis write on real requests; nothing on dry-run - matchPolicy: Equivalent - timeoutSeconds: 2 - admissionReviewVersions: [v1] -``` +## Attach and final Git author -`failurePolicy: Ignore` here is correct for the opposite reason it is in the e2e -observer: not to avoid a bootstrap deadlock, but because a missed author capture must -degrade to the committer, never reject a user's command. The handler also returns -`Allowed` even when the Redis write errors, so the command's success never depends on -the failure policy — `Ignore` only covers the webhook being entirely unreachable. +`AuthorAttributed=True` means admission captured the command submitter. The request can attach only to an +open window with that same named actor and GitTarget. ---- +`AuthorAttributed=False` (`CommitterFallback`) means capture ran but no admission record was available; +`AuthorCaptureDisabled` means capture was not configured. Neither is a failure and neither means the +eventual Git author is necessarily the configured committer. The request claims no actor and can attach only +to an unnamed window. That window determines the actual Git author: -## 5. The controller, before and after +- configured-author mode or a replay/resync write: the configured committer; +- live attribution that ran but found no usable audit fact: + `unknown (attribution unresolved) `. -The state machine loses its first wait entirely. Author resolution becomes a single -synchronous lookup with two outcomes, both immediate and both settled. +The worker never closes another actor's window. If no matching window appears before the close delay +expires, the request ends successfully with `Ready=True`, `Pushed=False`, and `NoWindowInGrace` or +`WindowMismatch`. -**Before** ([commitrequest_controller.go:164-233](../../internal/controller/commitrequest_controller.go#L164-L233)): +## Conditions -``` -first sight ─▶ Attributed=Unknown, WaitingForAuditEvent - │ - ▼ attributeAuthor: poll LookupCommitRequestAuthor every 2s - ├─ hit ───────────────▶ attributionResolved - ├─ within 60s, miss ──▶ attributionPending → requeue 2s (THE WAIT) - └─ past 60s, miss ────▶ attributionTimedOut (Attributed=False) - ▼ - ATTACH + POLL (close-delay) ─▶ terminal -``` +`AuthorAttributed` is binary and settled on the first reconcile; it has no audit-wait or `Unknown` state. -**After:** - -``` -first sight ─▶ attributeAuthor: LookupCommandAuthor (one read) - ├─ hit ─▶ author named, AuthorAttributed=True (AttributedFromAdmission) - └─ miss ─▶ committer, AuthorAttributed=False (CommitterFallback) - ▼ (no Unknown phase, no requeue-for-author) - WaitingForCloseDelay ─▶ ATTACH + POLL ─▶ terminal -``` - -Concretely, `attributeAuthor` collapses to: - -```go -func (r *CommitRequestReconciler) attributeAuthor( - ctx context.Context, cr *configbutleraiv1alpha3.CommitRequest, -) (queue.CommandAuthor, commitRequestAttribution) { - if r.AuthorLookup == nil { // validate-operator-types webhook disabled → configured-author - return queue.CommandAuthor{}, attributionCommitter - } - if a, ok := r.AuthorLookup.LookupCommandAuthor(ctx, cr.UID); ok { - return a, attributionFromAdmission - } - return queue.CommandAuthor{}, attributionCommitter // present-or-never: a miss is committer, now -} -``` - -The reconcile body drops the `waitForAudit` branch and its -`RequeueAfter: commitRequestAttributionRetryDelay` -([commitrequest_controller.go:168-171](../../internal/controller/commitrequest_controller.go#L168-L171)); -first-sight stamping goes straight to `WaitingForCloseDelay`. - -### Status: the `AuthorAttributed` condition - -Rename the domain condition `Attributed` → **`AuthorAttributed`** -([constants.go:59-63](../../internal/controller/constants.go#L59-L63)) and give it a -clean **binary, immediately-settled** meaning — no `Unknown`, no timeout state: - -| `AuthorAttributed` | Reason | Meaning | +| Condition | True | False | |---|---|---| -| `True` | `AttributedFromAdmission` | The submitter was captured at admission and named as the commit author. | -| `False` | `CommitterFallback` | No admission author record — the validate-operator-types webhook is not configured (or did not record one) — so the commit is authored by the configured committer. | - -`AuthorAttributed=False` is **not a failure** and does not affect `Ready`: the command -still commits successfully, just as the committer. The pairing `Ready=True` + -`AuthorAttributed=False` is the precise, honest signal a user reads as *"saved — but as -the bot, because author attribution isn't wired up here."* This is the status surface -for the edge case the whole design hinges on (record absent ⇒ webhook not configured ⇒ -committer), and it is why the absent case earns a real condition rather than being -hidden. - -Removed reasons from -[commitrequest_finalize.go:36-50](../../internal/controller/commitrequest_finalize.go#L36-L50): -`WaitingForAuditEvent` (and its `Attributed=Unknown` first-sight state), -`AttributedFromAuditEvent` (→ `AttributedFromAdmission`), `AuditEventNotObserved`, and -`AttributionNotRequired` (→ `CommitterFallback`). The `commitRequestAttribution` enum -drops `attributionPending`/`attributionTimedOut`, keeping `attributionFromAdmission` -and `attributionCommitter`. e2e specs asserting the old reasons or the -`WaitingForAuditEvent` phase must be updated — call this out in the PR. (v1alpha3 is -alpha, so the status-contract change is acceptable.) - ---- - -## 6. Wiring (`cmd/main.go`) - -Authorship capture is gated by the **validate-operator-types webhook**, not -`--author-attribution`: - -```go -// Command authorship is captured at admission and lives in its own Redis corner, -// independent of --author-attribution (which governs mirrored-resource attribution). -var commandAuthorStore *queue.CommandAuthorStore -if cfg.internalCommandsWebhookEnabled { - commandAuthorStore = redisStore.CommandAuthorStore() - mgr.GetWebhookServer().Register( - webhookhandler.ValidateOperatorTypesPath, - &ctrladmission.Webhook{Handler: &webhookhandler.ValidateOperatorTypesHandler{ - Store: commandAuthorStore, - Decoder: admission.NewDecoder(scheme), - }}, - ) -} -// ... -if err := (&controller.CommitRequestReconciler{ - // ... - AuthorLookup: commandAuthorStore, // nil when the webhook is disabled → configured-author, immediate -}).SetupWithManager(mgr); err != nil { /* ... */ } -``` - -The `AuthorLookup == nil` contract is preserved and *strengthened*: nil → configured-author; -non-nil + miss → configured-author **for that request** — both immediate, neither waits, -both surface `AuthorAttributed=False`. - -**HA.** Admission can hit any replica (it is a `Service`); the controller leader reads -from the shared Redis corner. The synchronous-write-before-persist invariant holds per -request regardless of which replica served admission, so a leader failover between -admission and finalize still finds the record. - -> **Flag naming.** The existing `--admission-webhook` flag and its `admission-webhook-*` -> cert flags ([cmd/main.go](../../cmd/main.go)) gate the e2e `*` observer today. Decide -> whether the validate-operator-types webhook rides the same admission server (one flag, one -> cert, one port — likely yes) or gets its own gate. Recommended: **one admission -> server**, both handlers registered on it, gated by the existing `--admission-webhook`; -> the validate-operator-types handler is always registered when that server is on. - ---- - -## 7. TLS — same story as the other certs - -The validate-operator-types webhook needs a serving cert exactly like the metrics endpoint -and the audit webhook. Follow the **established pattern**: full configuration for those -who need it, and a cert-manager easy-route shipped in the chart for everyone else. - -- **Full configuration (BYO).** Keep the existing serving-cert flags - (`--admission-webhook-cert-path` / `-cert-name` / `-cert-key`, - [cmd/main.go](../../cmd/main.go)) so an operator can mount any cert (their own PKI, a - different issuer, a rotated secret). The cert-watcher already hot-reloads on rotation, - as the audit and metrics certs do. -- **Easy route (chart + config).** The Helm chart and the `config/` kustomize overlay - ship the cert-manager resources that wire it up automatically, the same way the other - endpoints are templated: - - an `Issuer`/`Certificate` minting the admission serving cert into a Secret the - Deployment mounts; - - the `cert-manager.io/inject-ca-from` annotation on the - `ValidatingWebhookConfiguration` so the CA bundle is injected for you — the - config overlay already uses exactly this annotation - ([validating-webhook.yaml](../../config/webhook/validating-webhook.yaml)). - Gate the whole block on a chart value (e.g. `internalCommands.enabled`, - `internalCommands.certManager.enabled`) so a non-cert-manager user can opt into BYO - certs instead. See [audit-webhook-tls-design.md](../finished/audit-webhook-tls-design.md) for the - long-lived-CA + short-lived-serving-cert rotation model to mirror. - -This is the one genuinely new packaging surface (the chart "ships no webhook/cert -templates" today, per the header in -[validating-webhook.yaml](../../config/webhook/validating-webhook.yaml)). It is real, -but it is the standard operator-managed-webhook pattern and far less than the -apiserver-side wiring the audit webhook demands (§9). - ---- - -## 8. Edge cases and degradation - -| Case | Behavior | -|---|---| -| **Internal-commands webhook not configured** | `AuthorLookup == nil` (or record absent) → committer, immediate, **`AuthorAttributed=False (CommitterFallback)`**. The headline edge case: no record means the webhook isn't wired up; fall back to the committer and *say so* in status. | -| Redis write fails at admission | Handler logs, returns `Allowed`; controller finds no record → committer, `AuthorAttributed=False`. | -| Webhook pod unreachable | `failurePolicy: Ignore` admits the command; no record → committer, `AuthorAttributed=False`. | -| `generateName` | Name + uid present in `request.Object` at validating admission; recorded by uid. No response-body recovery (the audit path's `generateName` headache is gone). | -| Impersonation (`--as`) | `req.UserInfo` is the effective (impersonated) user — parity with the audit path's `resolveUserInfo`. | -| Service-account submitter | Recorded by its `system:serviceaccount:…` username like any other; `AuthorAttributed=True`. | -| Object deleted before reconcile | Informer never reconciles it; the orphan record self-cleans via the fixed TTL. Harmless. | -| Recreate same name (new uid) | Different uid → different key; no stale inheritance. | -| Webhook on but record missing | A real anomaly (write miss). `AuthorAttributed=False` like any committer fallback; surface it via a `command_author_lookup{result=miss}` metric so it is observable rather than silent. | - ---- - -## 9. Note for end users — why your *commands* are attributed but your *edits* may not be - -There is an honest asymmetry worth stating plainly, because users will notice it: - -> *"You can name me as the author of a CommitRequest I create, but my normal edits to a -> ConfigMap commit as the bot unless I run the audit webhook. Why?"* - -Because the two are not the same kind of thing, and crediting them the same way would -be wrong: - -- A **command** (a CommitRequest) is something you hand us directly, and we only ever - *act* on it once it has actually persisted. If your command never persists (rejected - by another webhook, a conflict), we simply never act — so attributing it at admission - can never put a wrong author on a real commit. We can safely sign for the parcel you - handed us. -- A **normal edit** attributed at admission could credit you with a change that **never - landed** — a later webhook rejects it, an optimistic-concurrency conflict drops it, a - dry-run was only a rehearsal. If we mirrored that into git, the repository would claim - a change the cluster never accepted. We will not sign for a delivery we only saw - *attempted*. Proof-of-persistence for an arbitrary edit lives only in the **audit - event**, which is why faithful attribution of normal activity needs the audit webhook. - -So the asymmetry is a **feature, not a gap**: gitops-reverser gives you the most -attribution each environment can *correctly* support. On a managed cluster where you -cannot enable an apiserver audit webhook, your commands are still attributed (admission -is enough for them), while your normal edits commit as the configured committer until -audit is available — and the `AuthorAttributed` condition tells you, per command, -exactly which path you got. Nothing is ever attributed on a guess. - -This is the same principle as the rest of the design: **command authorship is "who -asked"; state attribution is "what actually happened, and who."** We never conflate them. - ---- - -## 10. What this removes (forward-facing, not kept in parallel) - -Per the "don't keep the old audit ingestion around for this" directive: - -- **`AttributionIndex.LookupCommitRequestAuthor`** - ([attribution_index.go:410-423](../../internal/queue/attribution_index.go#L410-L423)) - and the `CommitRequestAuthorLookup` interface - ([commitrequest_controller.go:54-59](../../internal/controller/commitrequest_controller.go#L54-L59)) - — deleted; the controller depends on `CommandAuthorStore` instead. -- The **wait machinery**: `commitRequestAttributionTimeout`, - `commitRequestAttributionRetryDelay`, the `waitForAudit` return, the - `WaitingForAuditEvent` progress state, and the `Attributed=Unknown/False` audit - states (§5). -- `commitRequestResolveTimeout` shrinks by its `+60s` attribution component - ([commitrequest_controller.go:92-98](../../internal/controller/commitrequest_controller.go#L92-L98)). -- The audit handler still records facts for every mutating resource generically, so - `commitrequests` facts are *written* but **no longer read** for authorship. - Optionally skip recording the `commitrequests` resource in `RecordFact` to avoid dead - keys (minor). - -The audit webhook itself is **untouched and still required for mirrored-resource -attribution** — this change is scoped to the command path only. - -### Breaking change (alpha) - -CommitRequest authorship now comes **only** from the admission webhook -(`--admission-webhook` / the validate-operator-types webhook). An operator who ran -**`--author-attribution=true` with `--admission-webhook=false`** and relied on -audit-sourced CommitRequest attribution will, after this change, get -`AuthorAttributed=False (CommitterFallback)` — the commit is authored by the configured -committer. To keep CommitRequest authorship, enable the admission webhook. Mitigation: -the `config/` SUT and the Helm chart (`servers.admission.enabled`) both default it **on**, -so a fresh install is unaffected; only an audit-on/admission-off configuration regresses. -Mirrored-resource attribution (normal edits) is unchanged. v1alpha3 is alpha, so the -status-contract and behavior change are acceptable. - ---- - -## 11. Resolved decisions & remaining open questions - -**Resolved (this iteration):** +| `AuthorAttributed` | `AttributedFromAdmission`: the command submitter was captured | `CommitterFallback`: capture ran but no command-author record; `AuthorCaptureDisabled`: capture is off; request claims no actor | +| `Pushed` | the attached window was committed and pushed | a benign no-commit or finalize failure | +| `Ready` | a pushed commit or benign no-commit | progress or a finalize failure | -- **Record cleanup:** TTL, fixed internal constant (`commandAuthorRecordTTL`), **no - flag** — the timing is deterministic, so there is nothing to tune (§3). -- **Handler shape:** a dedicated, extensible **validate-operator-types** handler/path, not a - CommitRequest-only one — future command kinds are a table entry plus a webhook rule (§4). -- **Absent record:** fall back to committer and surface it as - **`AuthorAttributed=False (CommitterFallback)`** (§5, §8). -- **TLS:** the established pattern — full BYO-cert configuration plus cert-manager - resources shipped in the chart/`config` for the easy route (§7). +`Reconciling=True` with `WaitingForCloseDelay` is the only normal in-progress state. It covers the +optional collect delay followed by worker finalization and push. -**Decided during implementation:** +## Wiring -- **One admission server.** The validate-operator-types handler rides the existing - `--admission-webhook` server: when that flag is on, `cmd/main.go` builds the - `CommandAuthorStore` and registers both the always-allow observer and the - validate-operator-types handler (`setupAdmissionWebhooks`). The controller's `AuthorLookup` - is that store (nil when the flag is off → configured-author, immediate). -- **Chart value surface.** A new `internalCommands` block in `values.yaml` - (`enabled`, `bindAddress`/`port`, `tls.*`, `certManager.enabled`, `timeoutSeconds`), - defaulting `enabled: true` to mirror `attribution.enabled`. New templates - `validate-operator-types-webhook.yaml` (the `ValidatingWebhookConfiguration` with - `cert-manager.io/inject-ca-from`) and `validate-operator-types-certificates.yaml` (the - serving cert), plus the admission flags/cert mount in `deployment.yaml` and a - `webhook` port on the controller Service. -- **Cleanup:** TTL alone (`commandAuthorRecordTTL`, 1h). No terminal-status delete. +The command-author store is wired when the admission webhook is enabled and Redis is configured. It is +independent of `--author-attribution`, which controls audit-backed authorship for live watched resources. +An installation may therefore have either, both, or neither source of author information. -**Still open (deferred, not blocking):** +Related live contracts: -- **Capture-lag / `command_author_lookup{result}` metric (§8).** Not yet wired — the - `AuthorAttributed=False` condition already gives per-object observability of the - fallback; the aggregate counter/gauge is a follow-up. -- **Window-match coupling.** Command authorship is admission-sourced, but the attach - still matches the open window *by author string* (`openWindow.Author`, set from the - edit's audit attribution). So a named admission author only finalizes a window that - was *also* attributed to the same user — i.e. with audit on. In configured-author mode - the window author is empty and a named command author would not match; the - CommitRequest e2e specs that assert a named author therefore still skip - configured-author mode. Worth a follow-up if command authorship should stand fully - alone. +- [architecture: CommitRequest Finalize](../architecture.md#commitrequest-finalize) +- [configuration: CommitRequest conditions](../configuration.md#commitrequest-finalize) +- [status conditions guide](status-conditions-guide.md) diff --git a/docs/spec/commitrequest-design.md b/docs/spec/commitrequest-design.md index fea2cc08..8cec1d28 100644 --- a/docs/spec/commitrequest-design.md +++ b/docs/spec/commitrequest-design.md @@ -1,757 +1,70 @@ -# CommitRequest: design around two use cases +# CommitRequest window finalization -> **Status & field correction (2026-06-28):** this note predates two renames. -> (1) CommitRequest now reports progress and outcome through kstatus-compatible **conditions** — `Ready` -> (summary), `Reconciling`/`Stalled` (kstatus), and the domain conditions `Attributed` and `Pushed` — with -> no `phase` string. Where the text below says `WaitingForAuditEvent`, read "in-progress (`Reconciling=True`, -> `Attributed` not yet `True`)"; the post-attribution wait the text calls "in grace" / "Finalizing" is now -> the `Reconciling` reason **`WaitingForCloseDelay`**; `Committed`/`Rejected`/`Failed` map to the -> `Ready`/`Pushed`/`Stalled` conditions and the `Ready` reason. -> (2) The spec field `delaySeconds` is now **`closeDelaySeconds`** (it delays closing the open window). -> Read every `delaySeconds` below as `closeDelaySeconds`. See -> [status-conditions-guide.md](status-conditions-guide.md) §CommitRequest and -> `api/v1alpha3/commitrequest_types.go` for the contract. +> **spec** — current behaviour. The code depends on this document; change one, change the other. +> Index: [`../INDEX.md`](../INDEX.md) -> Status: living design note, 2026-06-12. -> Supersedes the speculative redesign in -> [github-e2e-per-type-tail-failure-investigation.md](../finished/signing-snapshot-tail-replay-failure-investigation.md) §11–§15. -> Related: [commitrequest-barrier-timeout-decision.md](commitrequest-design.md) -> (Option A), [commitrequest-multi-finalize-design.md](commitrequest-multi-finalize-design.md) -> (single-worker serialization), [canonical-stream-retirement.md](../architecture.md) -> (the per-type-stream refactor that motivates this note). +A `CommitRequest` is a one-shot “save now” command for one `GitTarget`. It does not mirror the +`CommitRequest` object. Instead, it can attach a message to a matching open commit window and asks the +worker to close that window after the requested collect delay. -## 1. What a CommitRequest is for +## Request and window contract -A CommitRequest is a one-shot **"save my changes now"** signal. Instead of waiting -for a GitTarget's silence timer (the commit window) to expire, a user creates a -CommitRequest naming the GitTarget; the operator finalizes that user's open commit -window into a real Git commit and reports the branch and SHA back in status. +The request identifies the target in `spec.targetRef.name`, may provide `spec.message`, and sets +`spec.closeDelaySeconds` (0–300 seconds). It is handled by the target’s single branch worker, so resource +events and the attach request share one FIFO. -The promise we want to keep is small and precise: +The worker attaches a request only when all of these match an open window: -> The edits **this author** made **before** asking to save are finalized into a -> commit **attributed to that author**, carrying **the author's message**, and the -> commit SHA is reported back. +1. GitTarget name and namespace; +2. the named actor, if either side has one; and +3. whether each side names an actor (`AttributionOutcome.NamesActor`). -This note designs the feature around the **two use cases that must work really -well**, states the simplifying assumptions that make them tractable, and is -explicit about what we cannot promise. +The third rule keeps independently configured command admission and live-resource attribution from being +coupled. A request with no named submitter can attach to either a configured-author or an unresolved live +window, but never to a named actor’s window. A request with a named submitter can attach only to that +actor’s named window. Therefore one user’s request never finalizes another user’s work. -## 2. The two use cases +On its first attach, the worker sets the deadline to receipt plus `closeDelaySeconds`. Repeated reconciles +are idempotent and keep that first deadline. The delay lets a `kubectl apply` bundle whose `CommitRequest` +arrives before its resource mutations collect into the same window. -### UC1 — "Save" button +## Authorship -A user makes a handful of changes, then enters a commit message and presses -**Save**. The UI emits a CommitRequest with `spec.message` set. There is human time -(seconds) between the edits and the save. +The validating admission webhook captures a command submitter before the `CommitRequest` persists. The +controller performs one best-effort, present-or-never lookup: -> **Expected:** the edits made before Save are committed as one commit with the -> typed message; the SHA comes back. Until Save, nothing is pushed (the commit -> window is long). - -### UC2 — `kubectl apply` of a bundle that includes a CommitRequest - -A user runs `kubectl apply -f bundle.yaml`, where the bundle contains several -resources **and** a CommitRequest carrying the intended message. The API server -processes the documents close together but in some order, and they fan out across -**different per-type audit streams**. - -> **Expected:** all the bundle's resources and the CommitRequest land in **one -> commit** with the CommitRequest's message. - -UC2 is the hard one, for one reason: **the CommitRequest may be processed first, in -the middle, or last.** If it is processed first, its "save" intent arrives before -the resources it is meant to save even exist in the pipeline. So the CommitRequest -must not shut the window the instant it is attributed — it needs a **grace period** -during which the rest of the same-author bundle can arrive and join the window. - -## 3. Simplifying assumptions - -This design is written for the two use cases above under two explicit assumptions. -Both are realistic for the use cases and keep the problem tractable. - -- **A1 — The commit window is large.** The GitProvider's silence timer is long - (minutes), so it never rolls the window out from under us during a save or an - apply. Once the author's window opens, it stays open until *we* finalize it. -- **A2 — No concurrent authors.** Only one author is writing to this GitTarget - during the action. We already decided we cannot merge independent authors into - one commit (§4), so we assume that situation away here rather than pretend to - handle it. - -Relaxing either assumption lands you in the "cannot promise" territory of §4 — that -is exactly where it belongs. - -## 4. What we honestly cannot promise - -Stating the limits is half the design. These are consequences of having **one -commit to make** and of giving up the single total order in the per-type refactor. -The design's job is to **bound and report** them, not to pretend they don't exist. - -- **One window, one commit — authors cannot be merged.** A branch worker holds at - most one open commit window. If ten different authors each make a change in one - second, we audit each one and produce **ten commits**, one per author, in arrival - order. We will not, and should not, fold two authors' edits into a single commit: - that would require conflict resolution between independent writers, which is the - complexity a single-window model exists to avoid. A CommitRequest from author X - only ever finalizes author X's window; another author's window is left untouched - (status `Rejected`/`WindowMismatch`, §6.7). **That strictness is a feature.** -- **Cross-type order is already approximate.** Because each type has its own audit - stream and the streams drain independently, even those ten one-event-each commits - may not be in the exact order the API server saw them. For a - **single author writing one bundle (UC2) this does not matter** — all the edits - land in the *same* commit regardless of intra-bundle order. It only surfaces - across commit boundaries, which we accept. -- **An internal writer can still steal the window.** Even with A1+A2, a per-type - **resync** finalizes the open live window before it applies (to preserve arrival - order). If one fires between an edit and the CommitRequest's finalize, the - author's window is gone. We cannot stop this — but §6.4 makes it *graceful* - instead of lossy: once the message is attached to the window, the resync's commit - carries the user's message and the CommitRequest reports `Committed`. The only - irreducible loss is an interrupt that lands *before* attribution, when there is no - message to attach yet. The worker must also ensure the stolen window's commit is - still **pushed** so the data is never lost. (This is the failure analyzed in the - investigation note; the push fix is worker-side, and §6.4 is the semantics that - preserve intent.) -- **"Before the CommitRequest" is bounded by grace, not guaranteed.** An edit made - moments before the save, on a type whose stream lags, may arrive after the grace - window and land in the **next** commit. We make the grace configurable and say so - in status; we do not wait unboundedly. - -## 5. How it works today - -### 5.1 The CRD - -`CommitRequestSpec` ([api/v1alpha3/commitrequest_types.go](../../api/v1alpha3/commitrequest_types.go)): - -- `gitTargetRef.name` — the GitTarget whose window to finalize (same namespace). -- `message` — optional verbatim commit message (1–1024 chars, no control chars). -- `delaySeconds` — the collect-grace window, `0`–`300`, default `0`. -- The spec is **immutable** after creation (CEL `self == oldSelf`). - -`CommitRequestStatus` reports `phase` (`WaitingForAuditEvent`, `Committed`, -`NoOpenWindow`, `Failed`), an optional `message`, and `branch` / `sha` on success. -*(The design renames the `NoOpenWindow` phase to `Rejected` and adds a structured -`reason` — see §6.7.)* - -### 5.2 The controller state machine - -[`CommitRequestReconciler.Reconcile`](../../internal/controller/commitrequest_controller.go) -runs with `MaxConcurrentReconciles=1` — the single worker *is* the -multi-CommitRequest ordering design, so no two finalizes interleave. One object is -advanced through: - -1. **Stamp** `WaitingForAuditEvent` (uncached re-reads guard against a stale cache - echo re-running a finalize that already terminated). -2. **Attribute** — `LookupCommitRequestAuthor` polls the `commitrequests` per-type - audit stream for *this object's own create event* (matched by namespace, name, - UID) and takes the author from it. Not seen yet → requeue every 2s. Past the 60s - attribution timeout → **fail closed** (`Failed`). The author is never guessed. -3. **Delay** — if `delaySeconds > 0`, requeue until `creationTimestamp + - delaySeconds`. Anchored at the object's creation timestamp. *(The design re-anchors - this at the attribution moment — see §6.4.4 — because creation-anchoring is fragile - under a delayed ingestion pipeline.)* -4. **Finalize** — enqueue the author-bound finalize signal and write terminal - status. (Today this step is preceded by the watermark barrier of §5.4, which - §6 argues to remove.) - -### 5.3 Attribution is the keystone - -Step 2 does two jobs at once, and this is the most important property of the design: - -- **Identity** — the author comes from a real audit event, never a guess. -- **Ordering anchor** — every mutation, including this CommitRequest, enters through - the same audit path. The author's earlier edits entered it *before* their - CommitRequest, so by the time we observe the CommitRequest's own create event, - the author's earlier work is — under comparable per-type ingestion delay — already - ingested and enqueued. Attribution buys most of the ordering we want, for free. - This is the basis for both use cases working with very little extra machinery. - -### 5.4 The finalize signal and the open window - -`FinalizeGitTargetWindow` enqueues a `FinalizeSignal` -(finalize_signal.go) onto the branch -worker's **FIFO event queue** — the same queue resource events ride — so by arrival -order it is processed after every earlier write for that worker. -`handleFinalizeSignal`: - -- no open window → `NoOpenWindow`; -- an open window whose **author + GitTarget + namespace** do not match → leave it - open, report `WindowMismatch` (cannot happen under A2 for the author's own work); -- a matching window → finalize into a commit; record branch + SHA. - -### 5.5 The watermark barrier (the part we are removing) - -Today, before finalizing, the controller takes a per-type watermark snapshot and -drains the tails to it (commit_barrier.go), -bounded by `FinalizeBarrierTimeout = 15s`, degrading visibly on timeout (Option A). -Three facts decide its fate: - -- It is **already best-effort** — the 15s timeout means it never was an invariant. -- It is taken **audit-anchored**, in step 4 *after* attribution and delay - ([commitrequest_controller.go:175](../../internal/controller/commitrequest_controller.go#L175)), - despite the stale "creation time" comment in the code. -- It has a **target-local gap**: the tail advances one cursor *per type* even when a - particular GitTarget was skipped - (audit_tail.go:177 vs the skip at - audit_tail.go:200), so it can report - `barrierReached=true` for a target that did not actually receive every protected - entry. A best-effort mechanism that can return a **false positive** launders "we - don't know" into a green status. - -## 6. The design that makes UC1 and UC2 work - -The core is small and, except for removing the barrier, already built: - -1. **Attribute** from the CommitRequest's own create audit event, or fail closed. -2. **Wait** `delaySeconds` from creation — the collect-grace window. -3. **Finalize** the open window bound to the attributed author + GitTarget. - -No snapshot, no drain loop. Here is why that is enough for each use case. - -### 6.1 UC1 walks straight through (delaySeconds = 0) - -The user edits, then — seconds later — presses Save. By the time the CommitRequest's -audit event is observed, the edits (made earlier, on the human timescale) are long -since mirrored into the open window (A1 keeps it open; A2 means it is the author's). -Attribution completes, `delaySeconds: 0` means finalize immediately, the window -matches → **one commit, the typed message, the SHA returned.** The barrier adds -nothing here: the human gap already guarantees the edits are present. - -### 6.2 UC2 needs the collect-grace, and the collect-grace is enough (delaySeconds > 0) - -Lay out the three orderings, all under A1+A2: - -- **CommitRequest last.** Resources ingest first and open/extend the window; the - CommitRequest's event arrives last; finalize after the grace → all in one commit. -- **CommitRequest in the middle.** Some resources are already in the window at - attribution; the rest arrive during the grace and join the same (still-open) - window; finalize at the end of the grace → all in one commit. -- **CommitRequest first.** This is the case that breaks a naive "finalize on - attribution." At attribution there may be **no window yet**. But the finalize is - deferred by the grace (`attribution + delaySeconds`, §6.4.4); during that grace the - bundle's resources arrive, open a window, and fill it; the deferred finalize then - finds the now-open window → all in one commit. - -So the **single mechanism — defer the finalize by `delaySeconds` and then finalize -whatever same-author window is open — covers all three orderings.** "A little bit of -grace before the CommitRequest is really shut down" *is* `delaySeconds`. UC2's -contract is therefore: set `delaySeconds` to comfortably exceed the bundle's -ingestion spread (a few seconds), and all of it lands in one commit. - -The honest boundary (§4): if a resource's stream lags past the grace, that resource -opens a *fresh* window after the finalize and waits for the next close. We do not -wait unboundedly, and we say so. - -### 6.3 Why the barrier goes - -The barrier helped neither use case: UC1 is covered by the human gap, UC2 by the -grace. It defended only a narrow, low-harm, already-degradable case (a same-author -edit on a lagging stream), while doing nothing for the cases that actually bite -(§4), and it could report false-positive success. **Remove it** — and retire the -Option-A timeout wording with it. We do not replace it with a blanket caveat on every -commit: §6.5 makes `Committed` mean "pushed to the remote", which is the honest claim -that matters. The one residual — an edit on a lagging type stream may land in the next -commit — is a documented property of the model (§4), not a per-request status note, -because once the barrier is gone there is nothing left that detects it. - -Keep the *idea* of an audit-anchored picture documented as the first thing to reach -for **if** real workloads ever prove harmful per-type skew — but reintroduce it then -as a **target-local applied watermark**, never the global cursor. - -### 6.4 Window-attach with eager message binding (the recommended target) - -§6.2 makes the **happy paths** of UC1 and UC2 work, but it has one weakness on the -**unhappy path**: between attribution and the finalize deadline, the CommitRequest's -message lives only on the controller side. If anything finalizes the window during -that interval — a resync (§4), a buffer-limit flush, anything — the resulting commit -carries the *generated* message and the CommitRequest later finds no window and -is `Rejected` (`NoWindowInGrace`, §6.7). The work is committed; the user's **intent** -(their message, and a `Committed` result) is lost. - -Window-attach fixes exactly that, and it is the model this feature should grow into: - -> **Bind the CommitRequest's message to the author's open window as early as -> possible. Once bound, whichever path finalizes the window first uses that -> message and resolves the CommitRequest as `Committed`.** - -The grace period stops being a "hold before we act" and becomes a "hold during which -the intent is already safe." That is the durability the feature is really about. - -#### 6.4.1 Worker model: the window carries the message - -There are **two state machines**, at two layers, and they hand off at attribution: - -- **Controller-side lifecycle** (the CRD phase): `Created → WaitingForAuditEvent - (attribution) → [attributed] → attach → poll → terminal`. The attribution wait is a - first-class state — it is where a **delayed ingestion pipeline is absorbed**, and it - is the point the grace is anchored *after* (§6.4.4). Nothing below happens until the - controller has observed the CommitRequest's own create audit event and bound the - author. (So this section's states are *not* missing the "waiting for the audit - event" moment — that moment lives one layer up, in §5.2 step 2 and the §7 table.) -- **Worker-side states** (below), which begin only *after* attribution, when the - controller sends the attach. - -Today the message travels on the one-shot `FinalizeSignal` and is applied only at the -finalize call. Window-attach moves it onto the window itself: - -``` -openWindow: - ... existing fields (author, target, events) ... - pendingMessage string // the CommitRequest's message, once attached - pendingCR *commitRequestRef // namespace/name/uid + result bookkeeping -``` - -A small per-worker table tracks CommitRequests not yet attached and the outcomes of -those already resolved: - -``` -pendingByAuthorTarget map[authorTargetKey][]commitRequestRef // waiting for a window -commitRequestOutcomes map[commitRequestID]Outcome // resolved: phase, sha, branch -``` - -A pending CommitRequest moves through four worker-local states: - -- **WaitingForWindow** — no matching (author + GitTarget + namespace) window is open - yet; the request is parked, with a finalize deadline. -- **Attached** — a matching window is open and now carries this request's - `pendingMessage`/`pendingCR`; a finalize is armed for the deadline. -- **AwaitingPush** — the window was finalized by *some* path; the `pendingCR` moved - onto the resulting `PendingWrite`, which is retained until its push lands (§6.5). -- **Resolved** — recorded in `commitRequestOutcomes`: `Committed` with the **pushed** - SHA once the PendingWrite reaches the remote, or `Rejected` with a reason — the - deadline passed with no window, the window was finalized but produced no diff, or it - belonged to someone else (§6.7). - -Transitions, all on the single worker loop goroutine (so no locking beyond the table): - -- attach request dequeued, matching window open and unclaimed → **Attached** - (set `pendingMessage`, arm finalize timer at the deadline); -- attach request dequeued, no matching window → **WaitingForWindow** (park, arm a - deadline timer); -- a same-author window opens while WaitingForWindow → **Attached**; -- finalize timer fires while Attached → finalize the window with the attached - message → **AwaitingPush**; -- **any other finalize path** runs on a window that is Attached (resync-before-apply, - buffer-limit, author switch, silence timer, shutdown) → it uses the attached - message → **AwaitingPush**; -- the `PendingWrite` carrying the `pendingCR` pushes successfully → **Resolved** - (`Committed` + pushed SHA, §6.5); -- a finalize produces no diff (the change is already present) → **Resolved** - (`Rejected`/`AlreadyPresent`, §6.7) — it does *not* enter AwaitingPush; -- deadline passes while still WaitingForWindow → **Resolved** (`Rejected`/`NoWindowInGrace`). - -#### 6.4.2 One chokepoint makes every cut-off honor the message - -All finalize paths already funnel through a single function -([`finalizeOpenWindowWithMessage`](../../internal/git/branch_worker.go)). That is -the leverage point. Change it to: - -1. use the explicit override message if the caller passed one, **else the window's - `pendingMessage`,** else the generated grouped message; and -2. if the window carries a `pendingCR`, **carry it onto the resulting `PendingWrite`** - alongside the message. Resolution does not happen here — it happens when that - `PendingWrite` pushes (§6.5). - -Because resync-before-apply, the commit-window timer, the buffer-limit flush, the -author-switch close, and shutdown **all** call this one function, every one of them -automatically carries the attached message *and* the `pendingCR` onto the -`PendingWrite` — with no per-path code. This is what makes "if something cuts it off, -the message is still there" true by construction rather than by enumerating closers. - -The companion worker fix from the investigation note still applies and is now a -hard prerequisite: a finalize that closes a window into a pending write **must -schedule its push** (`maybeSchedulePush`), so an attached-then-cut commit actually -reaches the remote. Attach makes the *result* correct; the push fix makes the *data* -land. - -#### 6.4.3 Controller ↔ worker protocol (attach, then poll) - -The synchronous request/reply of today does not fit, because the finalize may now -happen seconds later or be triggered by an unrelated path. Make it asynchronous, in -the same poll-via-requeue shape the attribution step already uses: - -1. **Controller**, the instant it attributes, sends one **AttachCommitRequest**`{crID, - author, target, message, delaySeconds}` onto the worker FIFO — no controller-side - delay. The worker stamps `finalizeAt = receipt + delaySeconds` on first registration - (§6.4.4). Re-sends across requeues are idempotent: the worker keys pending requests - by `crID` and keeps the first deadline. -2. **Worker** attaches (or parks) per §6.4.1; on the push that carries the - request's `PendingWrite`, it records the outcome (§6.5). -3. **Controller** requeues and polls `LookupCommitRequestOutcome(crID)`: - - not resolved and within a safety bound (covers the push cooldown + retries) → - requeue, surfacing any push-retry error in `status.message`; - - resolved → write the terminal status from the recorded outcome; - - no worker exists for the target → `Rejected` (`NoWindowInGrace`), as today. - -Polling (not a held result channel) keeps the controller non-blocking, survives a -reconcile running on a different goroutine, and is restart-tolerant. The worker GCs -`commitRequestOutcomes` entries by age. The controller's existing terminal-status -guards (uncached re-read, UID check) still make the status write at-most-once. - -#### 6.4.4 Timing: anchor the grace at attribution, not at object creation - -`delaySeconds` is a **collect window**, so it must be measured from the moment the -save is *observed*, not from when the object was created. Anchor it at attribution: - -> `finalizeAt = (attribution observed) + delaySeconds`, with `delaySeconds` bounded to -> ≤ 300s. - -This is a deliberate change from today, which anchors at the object's -`creationTimestamp` (§5.2). Object-creation anchoring is fragile under a delayed -ingestion pipeline: if ingestion takes longer than `delaySeconds`, then -`creation + delaySeconds` is already in the past by the time we attribute — the grace -has been entirely eaten by ingestion latency, the window may still be empty, and we -collect nothing (worst case `Rejected`/`NoWindowInGrace`). Creation-anchoring forces -`delaySeconds` to cover the *absolute* pipeline latency; attribution-anchoring lets it -cover only the inter-stream *spread*, which is all UC2 actually needs. Under a slow -pipeline it degrades gracefully — wait for ingestion, *then* collect — instead of -failing to collect. - -Concretely, no new status field is needed: the controller sends the attach the -instant it attributes, and the **worker** stamps `finalizeAt = receipt + delaySeconds` -when it first registers the request (receipt ≈ the attribution moment). Idempotent -re-sends keep the first deadline. - -- **UC1, `delaySeconds: 0`** — window already open at attribution; attach and - finalize immediately → `Committed`, the typed message. -- **UC2, `delaySeconds: N`** — attach the moment the first same-author edit opens the - window (before *or* after attribution, depending on bundle ordering); keep - collecting same-author edits; finalize at `finalizeAt`. - -An optional idle variant — finalize a few seconds after the *last* same-author edit, -capped at `finalizeAt` — would batch a burst more tightly; it adds a knob and the risk -of a chatty author deferring the commit, so treat it as a later tweak, not the -baseline. - -#### 6.4.5 What this buys, and the one residual - -Eager attach shrinks the window of vulnerability from the whole grace period down to -**just the attribution latency**: - -- Before attribution, there is no message to attach; if a resync cuts the window in - that gap, the commit uses the generated message and the CommitRequest finds no - window → `Rejected` (`NoWindowInGrace`). This is irreducible — we cannot attach an - intent we have not yet attributed. -- After attribution, the message is on the window. Any cut-off — resync or - otherwise — produces a commit **with the user's message**, and the CommitRequest - resolves as `Committed` once that commit is pushed (§6.5). The §4 "internal writer - steals the window" case stops being a lost-intent failure and becomes a graceful, - slightly-early commit. - -So we still cannot promise *one* commit in the face of a mid-flight resync (late -same-author edits that arrive after the cut open a fresh window and land later), but -we can promise the thing the user cares about most: **the message and the -work-so-far are committed under the author's name, and the request succeeds.** That -is the honest, defensible contract. - -#### 6.4.6 Edge cases - -- **Two CommitRequests for the same author/target.** A window carries at most one - `pendingCR`. A second request arriving while the window is Attached is parked - WaitingForWindow and binds to the *next* window (the edits after the first - finalize). Each CommitRequest still maps to exactly one commit with its own message. -- **No window ever opens.** Deadline passes WaitingForWindow → `Rejected` - (`NoWindowInGrace`) — pressing save with nothing pending, or a bundle with no - watched resources. -- **Restart.** Explicitly out of scope; see §6.6. - -## 6.5 Resolving the request: on push, from the `PendingWrite` - -The `PendingWrite` is already the system's nicest construct here: it carries the -commit message verbatim ([commit_executor.go:73](../../internal/git/commit_executor.go#L73)), -it is retained across the cooldown wait, and it survives the push-conflict -rebase-replay ([branch_worker.go:1091](../../internal/git/branch_worker.go#L1091)) -because the replay re-commits straight from it. So make it carry the CommitRequest's -result handle too, and resolve the request **when, and only when, its `PendingWrite` -is pushed**: - -``` -PendingWrite: - ... existing fields (events, message, signer, target) ... - CommitRequest *commitRequestRef // who to resolve once this write is pushed - CommitSHA plumbing.Hash // the hash of the commit this write created -``` - -- `executePendingWrite` already calls `worktree.Commit`, which returns the hash; - thread it back onto `CommitSHA`. On a rebase-replay the write is re-executed, so - `CommitSHA` is naturally **refreshed to the post-rebase hash** — no stale SHA. -- On a successful push ([`pushPending`](../../internal/git/branch_worker.go#L910)), - walk the pushed writes and, for each that carries a `CommitRequest`, record - `commitRequestOutcomes[cr] = {Committed, CommitSHA, branch}`. - -This is cheap — a couple of fields and a loop on the success path — and it finishes -the contract: - -- **`Committed` becomes honest.** The request is not resolved at local-commit time; - it is resolved when the commit is genuinely on the remote. A "save" therefore - confirms the push, which is exactly what a save should mean. The cost is latency - bounded by the push cooldown plus retries — seconds, and the right kind of wait. -- **The SHA is the real one.** It is read from the pushed write's own commit - (per-write, not branch HEAD, since a batched push may stack a later commit on top), - *after* any rebase-replay. The SHA-churn problem from the push-honesty discussion - disappears. -- **Push failure is honest too.** A failed push retains the `PendingWrite` - ([branch_worker.go:916-922](../../internal/git/branch_worker.go#L916-L922)) and - does not advance `lastPushAt`, so the request simply stays non-terminal while the - worker retries, with the push error surfaced in `status.message`. It flips to - `Committed` when a retry lands. It genuinely is not saved until then, and the status - says so — no new phase required (the CRD already documents `WaitingForAuditEvent` as - "the finalize it gates has not completed"). - -## 6.6 Restart and replay: explicitly out of scope - -We do **not** engineer durable recovery of an in-flight CommitRequest across an -operator restart. The message itself is never lost — it lives in `spec.message`, a -durable Kubernetes object — and on restart the controller re-reconciles any -non-terminal request, re-attributes from the durable audit event, and re-attaches. -That heals the common cases for free. - -The one case we knowingly accept is: a request whose commit was already **pushed** -but whose terminal status was not yet written when the operator restarted. The -in-memory `commitRequestOutcomes` entry is gone, the re-driven attach finds the work -already mirrored (so the re-finalize is a no-op), and the request resolves -`Rejected` (`AlreadyPresent`, §6.7) even though its commit — with its message — is on -the remote. The data and the message are safe; only the request's own status is -slightly off — and `AlreadyPresent` is at least an honest hint that the change is -already there. We judge this rare and low-harm and choose **not** to add a durable -record (e.g. a commit trailer) to -close it. If that ever changes, the durable system of record would be the commit -itself, stamped with the request identity; until then, restart recovery is best-effort. - -## 6.7 When nothing is committed: the `Rejected` outcome - -Not every CommitRequest produces a commit, and "no commit" is not one thing. Today -these collapse onto a single `NoOpenWindow` phase (with a mismatch *message* for the -foreign case). Rename that terminal phase to **`Rejected`** — one honest "the request -was handled correctly but produced no commit" state — and distinguish the reasons with -a structured `status.reason` plus a human `status.message`: - -| `reason` | When | Note | +| Admission result | `AuthorAttributed` | Request claim | |---|---|---| -| `NoWindowInGrace` | The deadline passed with no matching same-author window. | Benign: nothing was pending to save. | -| `WindowMismatch` | A window was open but belonged to a different author/GitTarget. | Benign and strict (§4): the foreign window is left untouched. | -| `AlreadyPresent` | A matching window was finalized, but its events produced **no diff** — the change already matches the remote, so the commit was dropped (loop prevention). | The theoretical edge this section exists for. | - -`Rejected` is deliberately **not** `Failed`. In all three cases the system behaved -correctly; there was simply no commit to make. `Failed` stays reserved for genuine -faults — attribution failed, the finalize/commit errored, or a push that can never -land — so a dashboard can treat `Failed` as "look at this" and `Rejected` as -informational. (At the worker level the existing `FinalizeNoOpenWindow` / -`FinalizeWindowMismatch` outcomes, plus a new no-change signal, simply roll up to -`Rejected` + the matching reason.) - -Concretely, the CRD gains a typed, validated `reason` enum and the field on status: - -```go -// CommitRequestRejectReason explains a Rejected CommitRequest: the request was -// handled correctly but produced no commit. It is set only when phase is Rejected. -// +kubebuilder:validation:Enum=NoWindowInGrace;WindowMismatch;AlreadyPresent -type CommitRequestRejectReason string - -const ( - // RejectNoWindowInGrace: the grace period elapsed with no matching same-author - // window — nothing was pending to save. - RejectNoWindowInGrace CommitRequestRejectReason = "NoWindowInGrace" - // RejectWindowMismatch: an open window existed but belonged to a different author - // or GitTarget, so it was deliberately left untouched. - RejectWindowMismatch CommitRequestRejectReason = "WindowMismatch" - // RejectAlreadyPresent: a matching window was finalized but produced no diff — the - // change already matches the remote, so the commit was dropped (loop prevention). - RejectAlreadyPresent CommitRequestRejectReason = "AlreadyPresent" -) - -// in CommitRequestStatus: -// // Reason explains a Rejected phase. Empty for non-Rejected phases. -// // +optional -// Reason CommitRequestRejectReason `json:"reason,omitempty"` -``` - -The human `message` still carries the prose; `reason` is the stable, machine-readable -discriminator that status consumers and tests assert on. - -### `AlreadyPresent` must not hang in `AwaitingPush` - -This is the case that interacts with §6.5, and it is why it needs explicit handling. A -finalized window normally creates a commit whose `PendingWrite` resolves the request -on push. But [`executePendingWrite`](../../internal/git/commit_executor.go#L132-L138) -returns `(0, nil)` when the events produce no change — **no commit, nothing to push.** -A request waiting for a push that never comes would sit in `AwaitingPush` until the -controller's safety bound and then mis-resolve. - -So the no-op is detected at finalize time and resolved immediately: - -- when a `PendingWrite` carrying a `pendingCR` executes and reports no change - (`anyChanges == false`), the worker records `commitRequestOutcomes[cr] = {Rejected, - AlreadyPresent}` right there — it never enters `AwaitingPush`; -- only a `PendingWrite` that actually created a commit (a real `CommitSHA`) takes the - `AwaitingPush` → `Committed`-on-push path of §6.5. - -In the §6.4.1 state machine this is one extra terminal edge off the finalize step: -*Attached → finalize produces no diff → Resolved(`Rejected`/`AlreadyPresent`).* - -### Worth a unit test - -Cheap, deterministic, and it guards a path that "never happens": finalize a window -whose event re-asserts state already present in the worktree (so -`applyPendingWriteEvents` returns `false`), then assert the carried request resolves to -`Rejected` with `reason == AlreadyPresent` — **and** that it resolves promptly rather -than blocking on a push. Assert on the structured `reason`, never on message text. - -## 7. State model - -The CRD keeps a small phase set; the richer "moments" are internal. - -| Internal moment | Meaning | User-visible phase | -|---|---|---| -| Created, waiting for audit event | Its own create audit event has not reached the `commitrequests` stream yet — this is where a delayed ingestion pipeline is absorbed. | `WaitingForAuditEvent` | -| Attributed, in grace | Author known; in the `delaySeconds` collect window anchored at attribution (§6.4.4). | `WaitingForAuditEvent` | -| Window matched, collecting | A matching same-author window exists and stays collectable until grace ends. | `WaitingForAuditEvent` | -| Finalized, awaiting push | The window was committed locally and carries the message; its `PendingWrite` is retained until it pushes (push retries surface in `status.message`). | `WaitingForAuditEvent` | -| Pushed | The carrying commit reached the remote; branch + pushed SHA recorded (§6.5). | `Committed` | -| No commit produced | No window in grace, a foreign window, or the change already matched the remote — distinguished by `reason` (§6.7). | `Rejected` (+ `reason`) | -| Attribution failed | The own audit event never arrived within 60s. | `Failed` | - -## 8. End-to-end tests (both use cases must be covered) - -Both use cases get a dedicated e2e spec. They share the existing harness shape -([test/e2e/commit_request_e2e_test.go](../../test/e2e/commit_request_e2e_test.go)): -a dedicated Gitea repo, GitProvider with a **large `commitWindow` (300s)** so the -silence timer can never be what produces the commit, a GitTarget, and a WatchRule. -A large window plus the dedicated repo realize A1; the dedicated namespace/repo -realizes A2. - -### 8.1 E2E-1 — UC1 "Save" (delaySeconds = 0) - -This is essentially the **existing** first spec ("finalizes the open commit window -on demand and reports the resulting SHA") and is the regression anchor for UC1: - -1. Apply a Deployment (opens the window). -2. `Consistently` assert for 10s that `main` does not yet exist — the edit is held. -3. Apply a CommitRequest (`delaySeconds: 0`) with an explicit `message`. -4. `Eventually` assert `status.phase == Committed`, `status.sha` non-empty, - `status.branch == main`. -5. Assert the commit subject equals `spec.message` verbatim, the Deployment file is - present, and `HEAD` equals the reported SHA. - -Strengthen it with one assertion: **exactly one new commit** was produced (HEAD -advanced by one), proving the save did not also trigger a stray second commit. - -### 8.2 E2E-2 — UC2 `kubectl apply` bundle (delaySeconds > 0, CommitRequest first) - -A new spec proving the bundle lands in one commit even when the CommitRequest is -applied **first** (the hard ordering): - -1. Build a single multi-document manifest whose **first** document is a - CommitRequest (`delaySeconds: 8`, explicit `message`) followed by **two or three - Deployments** in the watched namespace. -2. Apply the whole bundle in one `kubectl apply -f -`. -3. `Eventually` assert `status.phase == Committed` and `status.sha` non-empty. -4. Assert **exactly one commit** was produced for the bundle (HEAD advanced by - exactly one), its subject equals the CommitRequest's `message`, and **every** - Deployment in the bundle is present in that single commit. -5. (Stronger variant, optional) Watch a second type so the bundle spans two per-type - streams (e.g. add a ConfigMap to the bundle and a ConfigMap WatchRule), proving - the one-commit guarantee holds across independent streams — mind the pre-existing - `kube-root-ca.crt` ConfigMap noted in the existing suite when asserting commit - counts. - -The `delaySeconds: 8` value must comfortably exceed the bundle's ingestion spread so -the test is deterministic; the assertion is on the **outcome** (one commit, all -files, the message), not on internal ordering. Putting the CommitRequest first in -the file is the deliberately-hard arrangement: it exercises "save intent arrives -before the work" and proves the collect-grace (§6.2) closes that gap. - -### 8.3 Intent durability — a cut-off still carries the message (§6.4) - -The property §6.4 exists for is best pinned by a **deterministic worker/integration -test**, because forcing a resync to interleave is racy in full e2e: - -1. Open an author window (one event), then deliver an `AttachCommitRequest` with a - distinctive `message` and a non-zero `finalizeAt` so the window is **Attached** - but not yet finalized. -2. Before the deadline, invoke a window-closing path directly — the - `resync-before-apply` finalize is the canonical one. -3. Assert the resulting commit's message equals the CommitRequest's `message` (not - the generated grouped message), and that **after the carrying `PendingWrite` - pushes** (§6.5) `LookupCommitRequestOutcome` reports `Committed` with the **pushed** - commit's SHA — and that the reported SHA matches what is actually on the remote. - -Two complementary assertions: (a) `finalizeOpenWindowWithMessage` with no override on -a window carrying a `pendingMessage`/`pendingCR` uses the pending message and moves -the `pendingCR` onto the `PendingWrite`; (b) a push that hits a conflict and -rebase-replays still resolves the request to the *post-replay* SHA (no stale/orphaned -SHA). Add a full e2e only if a reliable resync trigger exists; the integration test is -the dependable pin. +| submitter record found | `True` / `AttributedFromAdmission` | that named actor | +| capture ran but no record | `False` / `CommitterFallback` | no actor | +| webhook disabled or Redis unavailable | `False` / `AuthorCaptureDisabled` | no actor | -### 8.4 Already-present — a finalize that produces no diff (§6.7) +`AuthorAttributed=False` is not a statement about the eventual Git author. The matched live window decides +that: -The theoretical edge, pinned by a deterministic unit test (see §6.7): finalize a -window whose event re-asserts already-present state so `applyPendingWriteEvents` -returns `false`, and assert the carried request resolves to `Rejected` with -`reason == AlreadyPresent` **promptly** — it must not block waiting for a push that -never comes. Assert on the structured `reason`, not on message text. +- configured-author, replay, and resync windows use the configured committer; +- a resolved live attribution fact uses the authenticated actor; and +- a live attribution miss uses `unknown (attribution unresolved) `. -## 9. Migration / order of work +See [CommitRequest admission authorship](commitrequest-admission-authorship.md) for capture provenance and +[architecture: author identity](../architecture.md#author-and-committer-identity-in-git) for the three Git +author states. -1. Add **E2E-2** (UC2), strengthen **E2E-1** (UC1), and add the **§8.3 intent- - durability** integration test — they pin the behavior before any refactor. -2. Remove the snapshot + drain step from `Reconcile`; keep attribution, delay, and - the author-bound finalize unchanged. -3. Delete (or park behind a clearly-labelled "future, target-local" stub) - `TakeTypeSnapshot` / `DrainTailsToSnapshot` and the per-type tail **cursor** - bookkeeping used only by the barrier. The tail itself, its checkpoint anchor, and - the late-event nudge stay — they are the ingestion path, not the barrier. -4. Retire the Option-A barrier-timeout status wording along with the barrier. Leave - the happy-path status clean: §6.5 makes `Committed` mean "pushed", which is the - honest claim. The lagging-stream caveat (§4) lives in the docs, not on every - request. -5. Worker: ensure **any** finalize that closes a window schedules its push (the - stranded-write fix) — prerequisite for §6.4 to be safe under a cut-off. -6. Build **§6.4 eager attach**: move the message onto `openWindow`, make - `finalizeOpenWindowWithMessage` honor the attached message and carry the - `pendingCR` onto the resulting `PendingWrite`, evolve the `FinalizeSignal` into an - idempotent `AttachCommitRequest`, switch the controller to attach-then-poll, and - **re-anchor the grace at attribution** — the controller drops its own delay and the - worker stamps `finalizeAt = receipt + delaySeconds` (§6.4.4), replacing today's - creation-anchored delay. -7. Build **§6.5 resolve-on-push**: add `CommitRequest` + `CommitSHA` to `PendingWrite`, - capture the commit hash from `executePendingWrite` (refresh it on rebase-replay), - and resolve the request from the push success path with the pushed SHA. This makes - `Committed` mean "on the remote" and removes the stale-SHA case. -8. Adopt **§6.7 `Rejected`**: rename the `NoOpenWindow` phase to `Rejected`, add the - structured `status.reason` (`NoWindowInGrace` / `WindowMismatch` / `AlreadyPresent`), - and resolve a no-diff finalize (`anyChanges == false` carrying a `pendingCR`) - immediately as `Rejected`/`AlreadyPresent` rather than letting it wait on a push. - Update the existing e2e assertions that expect `NoOpenWindow`. (CRD enum change is - acceptable in `v1alpha1`.) -9. Document restart recovery as out of scope (§6.6) — no durable record built for now. -10. Future, only if proven needed: the idle-reset grace variant (§6.4.4), and — if - cross-type skew is ever shown harmful — a **target-local** audit-anchored - watermark, never the global cursor. +## Lifecycle and outcomes -## 10. Bottom line +The controller stamps its conditions on first reconcile, immediately attaches the request, and polls the +worker. There is no audit wait for a `CommitRequest`; a delayed admission record cannot arrive after the +object is visible. -The refactor's one true cost is the loss of a single total order, and we tried to -buy it back with a per-type watermark barrier that is best-effort, partly incorrect, -and aimed at a case neither use case needs. Strip it. The promise users care about — -**save this author's recent work, attributed correctly, with their message, and tell -me the SHA** — is kept by two things we already have: **attribution from the -CommitRequest's own audit event** and a **configurable collect-grace**. UC1 needs -only the first; UC2 needs both, with `delaySeconds` sized to the bundle. Be honest in -status about what "before the CommitRequest" can mean across independent streams, pin -both use cases with e2e tests, and resist rebuilding the total order until something -real proves we need it. +| Outcome | Conditions | +|---|---| +| Commit pushed | `Ready=True`, `Pushed=True`, reason `Committed`; `status.sha` and `status.branch` are set | +| No same window before deadline | `Ready=True`, `Pushed=False`, reason `NoWindowInGrace` or `WindowMismatch` | +| Window produced no diff | `Ready=True`, `Pushed=False`, reason `AlreadyPresent` | +| Finalize or push error | `Ready=False`, `Pushed=False`, `Stalled=True`, reason `FinalizeFailed` | -And grow toward **eager message attachment** (§6.4): bind the author's message to -their open window the instant we can, so the grace period protects intent rather than -merely deferring action. Then the worst realistic interruption — a resync cutting the -window — yields a commit *with the user's message*, and the only thing we cannot save -is an interrupt that beats attribution itself. +`Reconciling=True` with reason `WaitingForCloseDelay` is the normal in-progress state. The controller fails +with `FinalizeFailed` only if the worker does not resolve the request within its bounded safety window; it +never polls indefinitely. -Finally, let the `PendingWrite` carry the request to its end (§6.5): it already holds -the message verbatim and survives every replay, so it is the natural place to also -hold the result handle and the commit SHA. Resolve the request **on push**, with the -pushed SHA — and `Committed` stops meaning "written locally" and starts meaning "on -the remote," with no stale SHA and at near-zero cost. Restart recovery we knowingly -leave best-effort (§6.6). That is a contract worth being proud of: simple at the core, -honest at the edges, and durable exactly where it counts. +The complete status vocabulary is in the [status conditions guide](status-conditions-guide.md). diff --git a/docs/spec/commitrequest-multi-finalize-design.md b/docs/spec/commitrequest-multi-finalize-design.md deleted file mode 100644 index 792a85e5..00000000 --- a/docs/spec/commitrequest-multi-finalize-design.md +++ /dev/null @@ -1,405 +0,0 @@ -# C-B2: per-GitTarget finalize coordinator (Option A design) - -> **spec** — current behaviour. The code depends on this document; change one, change the other. Index: [`../INDEX.md`](../INDEX.md) - -> Status: **superseded in part — C-B2 LANDED 2026-06-11 in a simpler form; see §12.** -> The per-type snapshot analysis (§3) and the ordering invariant (§1) are the parts that -> shipped; the coordinator goroutine, the annotation persistence, and the requeue-poll -> state machine did not — `MaxConcurrentReconciles=1` plus a fresh snapshot per attempt -> deliver the same guarantees with less machinery. -> Context: [canonical-stream-retirement.md](../architecture.md) §8 C-B2 row -> + the multi-CommitRequest ordering hazard raised 2026-06-11 - -## 1. The problem the design must solve - -The planned C-B2 puts finalize logic into the `CommitRequestReconciler`. The immediate -question is: **what happens when two CommitRequests for the same GitTarget exist at the -same time?** - -The root cause is that controller-runtime reconciles independent objects concurrently. -Two reconcile goroutines — one for CR-1 (rv=100), one for CR-2 (rv=110) — can both -pass their watermark barrier and then race to call `FinalizeGitTargetWindow`. The -BranchWorker's FIFO determines ordering; whichever goroutine enqueues first wins. If -CR-2 wins the race: - -``` -BranchWorker FIFO (wrong order): - [upserts@<100] [upserts@100-110] [finalize_CR2] [finalize_CR1] - ← CR-2's barrier ← CR-1's barrier, loses race -``` - -Result: **one commit attributing everything to CR-2**, then CR-1 gets `NoOpenWindow`. -The author who created CR-1 first gets no commit, even though every mutation was -present. The correctness guarantee from §6 of the retirement doc is violated. - -**The invariant that must hold:** -> For any two CommitRequests CR-i and CR-j targeting the same GitTarget where -> `rv_i < rv_j`, the sequence `barrier(CR-i) → enqueue(finalize_i)` must complete -> **before** `barrier(CR-j) → enqueue(finalize_j)` begins. - ---- - -## 2. Solution: per-GitTarget FinalizeCoordinator - -A single goroutine per active GitTarget owns the (barrier → finalize-enqueue) pair. -CommitRequest reconcilers hand off their work via a **buffered channel** and poll for -the result. The goroutine serializes naturally: it processes one finalize request at a -time, in FIFO order. Because it is the only one calling `FinalizeGitTargetWindow` for -its GitTarget, the invariant above holds by construction. - -```mermaid -graph TD - subgraph controller-runtime - CRC1["CommitRequestReconciler\ngoroutine A (CR-1)"] - CRC2["CommitRequestReconciler\ngoroutine B (CR-2)"] - end - - subgraph EventRouter - FC["FinalizeCoordinator\n(one goroutine per GitTarget)\n\nchan finalizeRequest ← buffered"] - FAW["FinalizeAtWatermark\n(DrainTailsToWatermark + Enqueue)"] - end - - subgraph Manager - TC["auditTailCursors\n(per-GVR, updated after apply)"] - end - - BW["BranchWorker\n(FIFO per GitTarget)"] - - CRC1 -- "Enqueue(rv_C=100, types_snapshot, ...)" --> FC - CRC2 -- "Enqueue(rv_C=110, types_snapshot, ...)" --> FC - FC -- "one at a time, in order" --> FAW - FAW -- "DrainTailsToWatermark(rv)" --> TC - FAW -- "EnqueueFinalize(signal)" --> BW - FC -- "result written to k8s status" --> API["kube-apiserver\n(CommitRequest status)"] - BW -- "flushed commit" --> Git["Git remote"] -``` - ---- - -## 3. What a finalizeRequest carries - -The request is assembled by the reconciler at the moment it first sees the -CommitRequest (`phase == ""`), **before** the status stamp bumps the object's -ResourceVersion: - -```go -type finalizeRequest struct { - // typeSnapshot maps each claimed GVR to the current top of its stream, - // read via TakeTypeSnapshot at CR creation time. This is the per-type - // watermark the barrier will wait on — one independent comparison per type, - // no cross-type RV ordering assumed (see §4). - typeSnapshot map[schema.GroupVersionResource]string - - // rv_C is the CommitRequest's create ResourceVersion, stored only for the - // restart-safety annotation (see §8). It is NOT used as a barrier watermark. - rv_C string - - // commit identity - author string - message string - - // crRef identifies the CommitRequest to write status back to. - crNamespace string - crName string - crUID types.UID // guards against delete+recreate between enqueue and writeback -} -``` - -### Why per-type watermarks, not a single rv_C? - -Kubernetes only guarantees ResourceVersion monotonicity **within a single resource -type**. In practice, core types all share the same etcd global counter, but: - -1. Aggregated-API types (served by a separate APIService) have RVs in an entirely - different counter. A CRD type might have entries at rv `8, 9, 10` while the - CommitRequest has rv `200` (core etcd). Comparing `cursor_T` (agg RV=10) against - `rv_C` (core RV=200) gives 10 < 200 — the barrier would stall until the 15 s - timeout. - -2. Even for core types, a type claimed **after** rv_C (say at checkpoint rv=150) gets - its tail cursor anchored at `"150-maxuint64"`. If rv_C=100, the barrier would wait - for that cursor to reach 100, which it has already passed — but could also stall if - the tail hasn't started yet. - -The per-type snapshot avoids both problems: for each type T the GitTarget was already -watching when the CR was created, record the current top of T's stream. The barrier -waits until `cursor_T >= snapshot[T]` — a comparison within T's own RV space. -Correctness argument: any mutation the author made to type T before creating the CR -was already in T's stream when the snapshot was taken, so `snapshot[T] >= rv(mutation)`, -and the barrier guarantees the tail consumed it. - -**The snapshot is built by `EventRouter.TakeTypeSnapshot(ctx, gitTargetName, ns)`** -which calls `Manager.TakeTypeSnapshot` → `TypeAuditHighWater` (per claimed type via the -`auditHighWaterReader` optional capability on the reader). The barrier -(`DrainTailsToSnapshot`) then polls only the in-memory cursor map — no Redis calls -during the wait loop. - ---- - -## 4. Full sequence: single CommitRequest - -```mermaid -sequenceDiagram - autonumber - participant Author - participant API as kube-apiserver - participant WH as webhook - participant Streams as :audit:stream (×N types) - participant Tails as per-type tails - participant BW as BranchWorker (FIFO) - participant CRC as CommitRequestReconciler - participant FC as FinalizeCoordinator - participant Git as Git remote - - Author->>API: PUT cm/a (rv=101), PUT deploy/b (rv=102) - API-->>WH: audit events - WH-->>Streams: mirror cm@101-*, deploy@102-* - Streams-->>Tails: deliver entries - Tails->>BW: enqueue upsert(cm/a), upsert(deploy/b) - Tails->>Tails: cursor[configmaps]=101-0, cursor[deployments]=102-0 - - Author->>API: CREATE CommitRequest (rv_C = 105) - API-->>CRC: watch event - - Note over CRC: phase == "", first reconcile - CRC->>CRC: capture rv_C = 105 (object.ResourceVersion) - CRC->>CRC: snapshot typesAtRV = [configmaps, deployments] - CRC->>CRC: store rv_C in annotation (restart safety) - CRC->>API: stamp WaitingForAuditEvent (RV now 106) - CRC->>FC: Enqueue(rv_C=105, types=[cm,deploy], author, message) - CRC->>CRC: requeueAfter(200ms) to poll for result - - Note over FC: goroutine owns the barrier+enqueue pair - FC->>FC: DrainTailsToWatermark(rv_C=105, types=[cm,deploy]) - Note over FC: cursor[cm]=101≥100? ✓ cursor[deploy]=102≥100? ✓ → passes immediately - FC->>BW: EnqueueFinalize(author, message) ← after all upserts in FIFO - BW->>Git: flush open window → commit with cm/a, deploy/b - BW-->>FC: FinalizeResult{Committed, sha=abc123} - - FC->>API: write status Committed / sha=abc123 - - CRC->>API: re-read CommitRequest (poll on requeue) - Note over CRC: phase=Committed → terminal, nothing more to do -``` - ---- - -## 5. Full sequence: two concurrent CommitRequests - -This is the critical scenario. Both CRs target the same GitTarget. - -```mermaid -sequenceDiagram - autonumber - participant CRC1 as Reconciler (CR-1, rv=100) - participant CRC2 as Reconciler (CR-2, rv=110) - participant FC as FinalizeCoordinator\n(single goroutine) - participant Tails - participant BW as BranchWorker (FIFO) - - CRC1->>FC: Enqueue({rv_C=100, types, author1, msg1}) - CRC2->>FC: Enqueue({rv_C=110, types, author2, msg2}) - Note over FC: channel is FIFO, CR-1 was enqueued first - - Note over FC: START processing CR-1 - FC->>Tails: DrainTailsToWatermark(rv_C=100) - Tails-->>FC: cursor[cm]=101≥100 ✓, cursor[deploy]=102≥100 ✓ → pass - FC->>BW: EnqueueFinalize(author1, msg1, rv_C=100) - BW->>BW: flush window → commit-1 (cm/a, deploy/b at rv<100) - BW-->>FC: Committed, sha=aaa - FC->>API: write CR-1 status = Committed - - Note over FC: START processing CR-2 (only now) - FC->>Tails: DrainTailsToWatermark(rv_C=110) - Note over Tails: tails continue delivering entries 100-110 while FC waited - Tails-->>FC: cursor[cm]=108≥110? maybe not yet → FC waits - Tails->>BW: enqueue more upserts (changes 100-110) - Tails-->>FC: cursor[cm]=111≥110 ✓ → pass - FC->>BW: EnqueueFinalize(author2, msg2) - BW->>BW: flush window → commit-2 (changes between commit-1 and now) - BW-->>FC: Committed, sha=bbb - FC->>API: write CR-2 status = Committed -``` - -The coordinator is the single point that prevents the race: CR-2's barrier cannot -start until CR-1's barrier + enqueue is complete. The BranchWorker FIFO then -naturally holds `[upserts@<100] [finalize_1] [upserts@100-110] [finalize_2]`. - ---- - -## 6. Reconciler state machine - -```mermaid -stateDiagram-v2 - [*] --> FirstReconcile : watch event (phase == "") - - FirstReconcile --> WaitingForResult : call TakeTypeSnapshot (per-type stream tops),\nstore rv_C + snapshot in annotation,\nstamp WaitingForAuditEvent,\nEnqueue(typeSnapshot, rv_C, author, msg) to coordinator,\nrequeueAfter(200ms) - - WaitingForResult --> WaitingForResult : poll: status still WaitingForAuditEvent\n→ requeueAfter(200ms) - - WaitingForResult --> Terminal : coordinator wrote terminal status\n→ nothing more to do - - FirstReconcile --> WaitingForResult : restart — phase=WaitingForAuditEvent,\nread snapshot+rv_C from annotation,\nre-Enqueue to coordinator - - Terminal --> [*] -``` - -The reconciler itself **never blocks** for more than a single API call. All blocking -(the ≤15 s barrier wait) happens in the coordinator goroutine, which calls -`DrainTailsToSnapshot` — a pure in-memory poll of cursor values, no Redis calls during -the wait loop. - ---- - -## 7. Coordinator lifecycle - -| Event | Action | -|---|---| -| First `Enqueue` call for a GitTarget | Start coordinator goroutine, create buffered channel (capacity=8) | -| GitTarget deleted (`UnregisterGitTargetEventStream`) | Close channel → goroutine exits after draining | -| Process restart | All coordinators are gone. Reconciler re-fires for each WaitingForAuditEvent CR and re-enqueues (rv_C re-read from annotation). | -| A CR is deleted before coordinator processes it | Coordinator reads the CR before writing status; `IsNotFound` → skip. | -| UID mismatch (delete + recreate with same name) | Coordinator guards with `crUID` check before writing status → skip stale result. | - ---- - -## 8. Restart safety: persisting the snapshot across pod restarts - -**The problem:** The reconciler calls `TakeTypeSnapshot` and captures -`rv_C = commitRequest.ResourceVersion` on the first reconcile, *before* stamping -status. Once it stamps `WaitingForAuditEvent`, the object's `ResourceVersion` changes -(to e.g. 106). If the pod restarts, the reconciler sees `phase=WaitingForAuditEvent` -but both `rv_C` and the snapshot are gone. - -**Solution: store rv_C and snapshot in an annotation before the status stamp.** - -``` -Annotation key: configbutler.ai/finalize-snapshot -Value: JSON: {"rv":"105","snapshot":{"":"secrets:101","apps":"deployments:50"}} -``` - -Reconciler on restart: -``` -if phase == WaitingForAuditEvent: - rv_C, snapshot = parseAnnotation("configbutler.ai/finalize-snapshot") - if annotation missing: call TakeTypeSnapshot again (stale, waits for a bit more but safe) - re-Enqueue(snapshot, rv_C, ...) to coordinator -``` - -**On restart without the annotation** (annotation somehow lost): the reconciler calls -`TakeTypeSnapshot` again. The new snapshot will have stream tops at the restart time, -which are ≥ the original snapshot values — the barrier waits for a superset of what the -original snapshot required. This is safe (correct, slightly more conservative) and -bounded by the 15 s timeout. - -**Why an annotation and not a status field?** Adding a status field requires a CRD -schema change and regeneration. An annotation is a metadata write that can be applied -in the same `Patch` as the status stamp — no schema bump needed. - -**Order of operations on first reconcile:** - -``` -1. TakeTypeSnapshot → snapshot (one Redis call per claimed type) -2. Patch: set annotation with rv_C + snapshot (metadata write) -3. Status().Update: phase = WaitingForAuditEvent (status write) -4. Enqueue(snapshot, rv_C, author, message) to coordinator -5. requeueAfter(200ms) -``` - -If the pod dies between steps 2 and 3: on restart, annotation is present but phase is -still `""` → first-reconcile path, reads annotation to avoid a second `TakeTypeSnapshot`. - -If the pod dies between steps 3 and 4: on restart, phase=WaitingForAuditEvent, reads -annotation, re-enqueues. The coordinator is idempotent for the same CR+UID. - ---- - -## 9. Where does the coordinator live? - -**`EventRouter`** — it already owns: -- `WatchManager` (→ `DrainTailsToWatermark`) -- `WorkerManager` (→ `FinalizeGitTargetWindow`) -- `Client` (→ status writeback) -- `gitTargetStreams` map with the same per-GitTarget keying pattern - -It adds a `finalizeCoordinators map[types.ResourceReference]chan finalizeRequest` with -the same mutex discipline. The coordinator goroutine calls -`r.FinalizeAtWatermark(...)` (already implemented, C-B1) and then writes status -directly using `r.Client`. - -The `CommitRequestReconciler` gets an `EventRouter` reference (it already has none -today; it would be wired in `cmd/main.go` where the reconciler is registered). - ---- - -## 10. What changes in the audit consumer - -*(Landed with C-B2 — `commit_request.go` was deleted outright; `applyFinalizeResultToStatus` -and friends moved to `internal/controller/commitrequest_finalize.go`.)* - -After C-B2 lands: - -- `isCommitRequestCreate` — **deleted** -- `handleCommitRequest` / `writeCommitRequestStatus` — **deleted** -- `applyFinalizeResultToStatus` — **moved** to `internal/watch/` (or kept in - `internal/queue/` and imported by the coordinator) - -The consumer's only remaining job before C-C is: route `AuditConsumer.routeAuditEvent` -— which today already returns early for everything except scale (already deleted in -C-A2). After C-B2 it returns early for everything. C-C then deletes the file entirely. - ---- - -## 11. Open questions before implementation - -| # | Question | Impact | -|---|---|---| -| Q1 | Should the coordinator goroutine also be responsible for the `barrierTimedOut` metric (`commitrequest_barrier_timeouts_total`) or should it be in `DrainTailsToWatermark`? | Logging/metric placement | -| Q2 | Channel capacity: 8 per GitTarget feels generous. Should it be bounded differently? | Backpressure vs. blocking reconciler on `Enqueue` | -| Q3 | Should the coordinator skip the barrier entirely if rv_C is missing from the annotation (restart + annotation lost)? Or finalize with `barrierReached=false`? | Restart degrade behavior | -| Q4 | The annotation `configbutler.ai/finalize-watermark-rv` — is a separate `metadata` Patch acceptable, or should rv_C go into the CRD status (requires schema bump)? | CRD versioning | - -The answers to Q3 and Q4 are the most load-bearing before writing the implementation. - ---- - -## 12. As landed (C-B2, 2026-06-11): the simplifications - -The implementation kept this doc's per-type snapshot (§3) and ordering invariant (§1) -but dropped three pieces of machinery. The reasoning mirrors the Option-A timeout memo's -own rejection of its option C: CommitRequests are rare, so blocking one dedicated -reconcile worker for ≤15 s is fine, and everything built to avoid that blocking was -weight without benefit. - -1. **No FinalizeCoordinator.** The reconciler runs barrier → finalize → status **inline** - in one `Reconcile` invocation, and the controller is registered with - `MaxConcurrentReconciles: 1`. A single worker serializes concurrent CommitRequests in - workqueue (≈ creation) order — exactly the FIFO the coordinator channel would have - provided, because the channel was filled in reconcile order anyway. The §1 invariant - holds by construction: no two barrier/finalize pairs ever interleave. -2. **No annotation, no persisted rv_C.** Every attempt (first reconcile, conflict retry, - post-restart redelivery) calls `TakeTypeSnapshot` fresh. A later snapshot waits on a - **superset** of the create-time entries — conservative, still bounded by the 15 s - timeout, and correct without any cross-restart state. (§8's own fallback analysis - already proved this safe; landing made the fallback the only path.) -3. **The author comes from the CR's own audit event — attribution as a state.** §3's - `finalizeRequest.author` field never named its source; the landed design makes it - explicit: the reconciler polls `…commitrequests:audit:stream` (which `mirrorByType` - already fills) for the CR's create event, takes `resolveUserInfo(event).Username`, - and only then finalizes with the strict author match. Unattributable within 60s → - `Failed`, fail closed. The same wait re-anchors the trigger on the audit ingest path, - closing the watch-event-outruns-ingest race the first e2e run exposed — see the DEC-B - "as landed" note in [canonical-stream-retirement.md](../architecture.md) §4. - A new `spec.delaySeconds` (0–300) optionally holds the finalize as an extra collect - window after creation. - -The stale-cache hazard that motivated the coordinator's UID bookkeeping is handled by an -**uncached APIReader re-read** before any work: a watch echo whose cached object still -says `WaitingForAuditEvent` after the terminal phase was written short-circuits on the -live read. A crash between finalize and status write re-runs the idempotent flow on -redelivery and lands `NoOpenWindow` — the same behaviour the consumer's XAUTOCLAIM -redelivery had. - -The §11 questions resolve as: **Q1** — the metric increments in -`EventRouter.FinalizeAtWatermark` next to the degrade log (one place, covers any future -caller); **Q2/Q3/Q4** — moot (no channel, no annotation, no persisted watermark). diff --git a/docs/spec/deletecollection-attribution-expander.md b/docs/spec/deletecollection-attribution-expander.md index cee3d8f4..6d70bdc8 100644 --- a/docs/spec/deletecollection-attribution-expander.md +++ b/docs/spec/deletecollection-attribution-expander.md @@ -27,8 +27,8 @@ When someone runs `kubectl delete configmaps --all -n team-a`, the API server em with no special code. Two things are *not* free: - **Attribution.** Each per-object removal runs the resolver, finds no fact (the collection event is name-less, so - it stores nothing today — §4), and ships as **committer**. "Alice deleted these 12 configmaps" is recorded as - the operator. The **expander** (§5) closes this. + it stores nothing today — §4), and ships with the explicit **unresolved** author. "Alice deleted these 12 + configmaps" is visibly unresolved rather than silently credited to the operator. The **expander** (§5) closes this. - **Finalizers — and this is where the design got interesting.** An object with a finalizer is *not* removed by the delete; it gets a `deletionTimestamp` and lingers in `Terminating` until a controller clears the finalizer. An earlier revision of this doc tried to defer attribution to that eventual removal and fight the resulting @@ -149,9 +149,9 @@ variants are skipped when no RV is supplied, which is precisely right since the has a name, so `RecordFact` already writes its uid-only fact; with §2's intent rule, the finalizer single-delete is now removed and attributed at intent time too — for free, no expander needed. The expander exists **only** for the name-less collection case. -- **The conservative resolver fails closed** — multiple authors on one key → `AttributionConflict` → committer - ([storeFactKey](../../internal/queue/attribution_index.go#L403)). Governing rule: **a wrong author is worse than - no author.** +- **The conservative resolver fails closed** — multiple authors on one key → no usable attribution fact → the + explicit unresolved author ([storeFactKey](../../internal/queue/attribution_index.go#L403)). Governing rule: + **a wrong author is worse than no author.** - **The grace window** absorbs a watch event that arrives before its audit fact ([author_resolver.go:40](../../internal/watch/author_resolver.go#L40)). @@ -206,19 +206,19 @@ list**. We know the actor, type, maybe namespace, maybe a label selector, and ro objects**. The owner's trap: *in a few seconds you could see more than one `deletecollection` that "fits."* - **Option A — keep it around and guess by scope.** Reject. Two independent mis-attribution modes: (1) two - actors deleting in the same `(type, namespace)` window → honest answer is conflict→committer, so it degrades to - committer under exactly the load that makes the case interesting; (2) even a single collection delete would + actors deleting in the same `(type, namespace)` window → honest answer is conflict→unresolved, so it degrades to + the explicit unresolved author under exactly the load that makes the case interesting; (2) even a single collection delete would capture an *unrelated* plain `kubectl delete configmap x` in the same window. Selector re-matching narrows but doesn't remove it (selectors overlap; empty selector matches all). Violates "a wrong author is worse than no author." -- **Option B — `Co-authored-by` floor.** Honest credit under ambiguity (multiple trailers), git author stays - committer. But in watch-first the deletes are driven by watch events, not the audit cause, so it needs real +- **Option B — `Co-authored-by` floor.** Honest credit under ambiguity (multiple trailers), Git author stays + unresolved. But in watch-first the deletes are driven by watch events, not the audit cause, so it needs real plumbing to carry a scope-cause into the commit-window builder. A deliberate fast-follow. -- **Option C — commit as committer, document the limit.** v1. +- **Option C — commit as the explicit unresolved author, document the limit.** v1. **v1: Option C.** The hard case is a narrow intersection (aggregated/hollow body ∧ supports `deletecollection` ∧ -watched ∧ attributed-author mode), and the failure is *degraded attribution*, not wrong state or wrong author — which is -the correct conservative outcome. **Ship §2 + §5 now; Option B is the named fast-follow; Option A is rejected.** +watched ∧ attributed-author mode), and the failure is *degraded attribution*, not wrong state or a guessed author — which +is the correct conservative outcome. **Ship §2 + §5 now; Option B is the named fast-follow; Option A is rejected.** ## 7. Recommendation at a glance @@ -227,14 +227,14 @@ the correct conservative outcome. **Ship §2 + §5 now; Option B is the named fa | Any delete (single or collection member) | **Remove file at intent time; never commit `deletionTimestamp`** (§2) | Git is intent; reversible invariant; manifests stay re-appliable. | | Finalizer object | **Removed immediately, attributed to the delete-requester**; finalizer cleanup is runtime no-op in Git | Reframe dissolves the old delay/conflict; controllers still finalize in-cluster. | | Collection delete, body present | **Per-UID expander (§5)** credits the actor on each removal | API server states "these exact objects, by this user." | -| Collection delete, hollow body | **Committer (Option C, §6)** | Narrow; degraded attribution beats a wrong author. | +| Collection delete, hollow body | **Explicit unresolved author (Option C, §6)** | Narrow; degraded attribution beats a wrong author. | | Stuck `Terminating` | **Operational status/metric** (§2.5), file already absent | Don't pollute intent with runtime state. | ## 8. Observability & diagnostics - **`AttributionExactDeleteCollectionItem`** flows onto `AttributionResolutionsTotal{result=…}` via `recordAttributionResolution` ([author_resolver.go:169](../../internal/watch/author_resolver.go#L169)) — a - dashboard can show collection-member precise attributions vs. committer fallbacks. + dashboard can show collection-member precise attributions vs. unresolved outcomes. - **Expander write counter** (`op="deletecollection_expanded"`) on `AttributionFactEventsTotal` via the existing `recordFactEvent` hook ([attribution_index.go:433](../../internal/queue/attribution_index.go#L433)). - **Stuck-finalizer / terminating diagnostics** (§2.5): surface long-`Terminating` watched objects whose files we diff --git a/docs/spec/e2e-bi-directional-corner.md b/docs/spec/e2e-bi-directional-corner.md index 356dd780..a3fb5ef3 100644 --- a/docs/spec/e2e-bi-directional-corner.md +++ b/docs/spec/e2e-bi-directional-corner.md @@ -211,7 +211,7 @@ parallel with both. `BI_DIRECTIONAL_GINKGO_PROCS` is therefore `3`. Filter changes so the default suite stops paying for it: -- `test/e2e/Taskfile.yml`: `E2E_LABEL_FILTER` default `!image-refresh` +- `test/e2e/Taskfile-e2e.yml`: `E2E_LABEL_FILTER` default `!image-refresh` → `!image-refresh && !bi-directional` - `.github/workflows/ci.yml`, `full-core` leg: `!manager && !image-refresh` → `!manager && !image-refresh && !bi-directional` @@ -234,7 +234,7 @@ chart**. `_argocd-installed` `helm pull`s the pinned chart into `.stamps/cluster//argocd/` (retryable network step) and `helm upgrade --install`s it with `values.yaml` and `--wait`, then gates on CRD establishment. -- `ARGOCD_CHART_VERSION` (e.g. `10.1.3`) lives in `test/e2e/Taskfile.yml` and pins +- `ARGOCD_CHART_VERSION` (e.g. `10.1.3`) lives in `test/e2e/Taskfile-e2e.yml` and pins the chart, which carries the Argo CD release the specs pin behaviour against (its appVersion, e.g. `v3.4.5`). Unlike `FLUX_VERSION` it is not a devcontainer env var: nothing outside the cluster needs an `argocd` binary — the specs drive Argo diff --git a/docs/spec/status-conditions-guide.md b/docs/spec/status-conditions-guide.md index 2f6273a8..6586e787 100644 --- a/docs/spec/status-conditions-guide.md +++ b/docs/spec/status-conditions-guide.md @@ -78,28 +78,30 @@ trio; the bounded "did the work actually happen" signal lives on the `Pushed` do ```go const ( - TypeReady = "Ready" // True at a non-error terminal outcome (committed, or a benign no-commit) - TypeReconciling = "Reconciling" // True through both progress waits (reason names which) - TypeStalled = "Stalled" // True when the finalize failed (kstatus Failed) - TypeAttributed = "Attributed" // True once the author is settled (immediately True when not required) - TypePushed = "Pushed" // True once the commit is in the remote repository + TypeReady = "Ready" // True at a non-error terminal outcome + TypeReconciling = "Reconciling" // True while the close delay/finalize is in progress + TypeStalled = "Stalled" // True when the finalize failed (kstatus Failed) + TypeAuthorAttributed = "AuthorAttributed" // Whether admission captured the command submitter + TypePushed = "Pushed" // True once the commit is in the remote repository ) ``` -A request passes through two distinct, observable progress waits, both `Reconciling=True` but told apart -by the `Reconciling`/`Ready` reason and the `Attributed` condition. +A request has one progress wait: its optional `closeDelaySeconds` collect window, followed by finalization +and push. `AuthorAttributed` settles at first sight because command authorship is captured synchronously +at admission; it never has an `Unknown` or audit-wait state. Canonical reads: -* waiting for the create audit event (attributed-author mode only): `Reconciling=True` reason - `WaitingForAuditEvent`, `Attributed=Unknown` → kstatus InProgress -* waiting for the close delay (author settled, attached — the `closeDelaySeconds` collect window, then - commit and push): `Reconciling=True` reason `WaitingForCloseDelay`, `Attributed` settled, `Pushed=False` - → kstatus InProgress +* waiting for the close delay: `Reconciling=True` reason `WaitingForCloseDelay`, `AuthorAttributed` + settled, `Pushed=Unknown` → kstatus InProgress * committed: `Ready=True`, `Pushed=True`, `Stalled=False`, reason `Committed` → kstatus Current * benign no-commit (nothing to save / already present / foreign open window): `Ready=True`, `Pushed=False`, `Stalled=False`, with the specific reason on `Ready` → kstatus Current (a correct, non-error outcome) * failed finalize: `Ready=False`, `Stalled=True`, reason `FinalizeFailed` → kstatus Failed -* configured-author mode sets `Attributed=True` (`AttributionNotRequired`) immediately; attributed-author mode leaves - it `Unknown` until the create audit event names the author, or `False` (`AuditEventNotObserved`) if it - never arrives and the commit is authored by the configured committer + +`AuthorAttributed=True` (`AttributedFromAdmission`) means the command submitter was captured. `False` +(`CommitterFallback`) means capture ran but no admission author record exists; `False` +(`AuthorCaptureDisabled`) means capture is disabled. Both claim no actor and can attach only to an unnamed +window. That condition does not itself determine the Git author: the attached watch window is +configured-author (the committer) when watch attribution is disabled, or explicitly unresolved when watch +attribution ran but could not name an actor. diff --git a/docs/tasks-overview.md b/docs/tasks-overview.md index 5550a0f4..c8275b8f 100644 --- a/docs/tasks-overview.md +++ b/docs/tasks-overview.md @@ -34,10 +34,22 @@ Task keeps the behavior easier to follow: `flatten: true` so everything shares one flat command surface, and sets `run: once` so the e2e DAG can fan out through `deps:` without a shared node starting twice. - [Taskfile-build.yml](../Taskfile-build.yml): build, codegen, lint, and unit-test tasks. -- [test/e2e/Taskfile.yml](../test/e2e/Taskfile.yml): the e2e bring-up DAG. +- [test/e2e/Taskfile-e2e.yml](../test/e2e/Taskfile-e2e.yml): the e2e bring-up DAG. Run `task` (or `task help`) to list everything. +> **Why `Taskfile-e2e.yml` and not `Taskfile.yml`?** Deliberate — please don't rename it +> back. Every path in that file is relative to the **repo root** (`.stamps/cluster/…`, +> `test/e2e/cluster/audit`, `find cmd/mutation-capture-lab …`), because the root Taskfile +> includes it with `dir: .`. If it were named `Taskfile.yml`, `task` run from inside +> `test/e2e/` would stop its upward search there and execute with `test/e2e/` as the working +> directory — every one of those paths would then silently resolve to a non-existent +> location. `task clean-cluster` would delete no stamps while still deleting the cluster, +> leaving a stale `ready` stamp that makes the next `prepare-e2e` skip cluster creation and +> fail with `No nodes found for given cluster`. With the current name there is no +> `Taskfile.yml` in `test/e2e/` to stop the search, so `task` walks up to the repo root and +> runs with the correct working directory no matter where you invoke it from. + ## Most important tasks | Task | What it does | When to run it | diff --git a/docs/tasks/bounded-queue.md b/docs/tasks/bounded-queue.md deleted file mode 100644 index 2180c800..00000000 --- a/docs/tasks/bounded-queue.md +++ /dev/null @@ -1,221 +0,0 @@ -# gitops-reverser feature request - -## Title - -Bound and monitor the durable audit queue by default - -## Problem - -The durable audit Redis/Valkey stream can grow without bound. On the demo -cluster this produced a multi-GB Valkey dataset and repeated Valkey restarts, -which temporarily broke audit ingestion. - -Live observation: - -```text -namespace: gitops-reverser -pod: valkey-867df97d8b-cckmm -restartCount: 9 -lastState.terminated.exitCode: 137 -lastState.terminated.reason: Error -``` - -Valkey startup log after one restart: - -```text -RDB memory usage when created 6124.68 Mb -Done loading RDB, keys loaded: 13359, keys expired: 645. -DB loaded from disk: 14.964 seconds -Ready to accept connections tcp -``` - -During that loading window, `gitops-reverser` could not enqueue or consume -audit events: - -```text -XREADGROUP failed: dial tcp 10.96.98.248:6379: connect: connection refused -failed to claim audit decision ... LOADING Valkey is loading the dataset in memory -``` - -This is operationally risky because the audit queue is the handoff between the -Kubernetes API server audit webhook and the controller. A bloated queue can -turn normal demo traffic into a queue outage, and queue outages can lead to -missed or delayed Git commits. - -## Current behavior - -The chart default is unbounded: - -```yaml -queue: - redis: - maxLen: 0 -``` - -This renders: - -```text ---audit-redis-max-len=0 -``` - -`0` disables stream trimming. The debug stream also defaults to unbounded when -enabled: - -```yaml -webhook: - audit: - debugStream: - maxLen: 0 -``` - -For a long-running cluster or a noisy audit policy, this allows unbounded data -growth in Valkey. - -## Expected behavior - -`gitops-reverser` should have a bounded, production-safe default for audit -stream storage. A default install should not be able to grow Valkey to several -GB without the operator explicitly opting into that behavior. - -The queue should provide enough retention for normal controller catch-up and -short outages, but it should not act as an infinite audit archive. Kubernetes -audit logs or object storage are better places for long-term raw audit -retention. - -## Suggested implementation - -### 1. Change chart defaults - -Set a bounded default for the hot audit stream, for example: - -```yaml -queue: - redis: - maxLen: 10000 -``` - -Treat the default as an operational starting point rather than a universal -capacity answer. Sizing should be based on expected audit events per second, -the outage/catch-up window the queue should tolerate, and a safety factor. - -If exact retention is not required, keep using approximate Redis stream -trimming for performance: - -```text -XADD ... MAXLEN ~ 10000 -``` - -This is a deliberate tradeoff: bounded streams protect Valkey memory and reload -time, but they can trim entries before a slow or stopped consumer has processed -them. The implementation should make that tradeoff visible through metrics and -documentation. - -Keep `0` available as an explicit opt-out for users who really want unbounded -retention: - -```yaml -queue: - redis: - maxLen: 0 # explicit unbounded mode -``` - -### 2. Bound the debug stream too - -If `webhook.audit.debugStream.enabled=true`, the debug stream should also have -a bounded default. For example: - -```yaml -webhook: - audit: - debugStream: - enabled: false - maxLen: 10000 -``` - -The debug stream is especially dangerous because it stores every decoded audit -event before filtering, joining, or dropping. - -### 3. Surface a warning for unbounded mode - -At startup, log a clear warning when either stream is unbounded, whether the -configuration came from Helm values or direct binary flags: - -```text -audit redis stream max length is 0; queue retention is unbounded -``` - -This makes accidental unbounded production installs visible in logs. - -### 4. Add metrics - -Expose queue size and trimming-relevant metrics, for example: - -- `gitops_reverser_audit_queue_stream_length` -- `gitops_reverser_audit_queue_consumer_lag` -- `gitops_reverser_audit_queue_pending_entries` -- `gitops_reverser_audit_queue_oldest_entry_age_seconds` -- `gitops_reverser_audit_queue_oldest_pending_age_seconds` -- `gitops_reverser_audit_debug_stream_length` - -These should be low-cardinality and scoped to the configured stream names. -`pending_entries` alone is not enough: Redis pending entries are claimed but -unacked messages, while consumer-group lag captures unread backlog that may be -trimmed before consumption. - -### 5. Document retention semantics - -Document that: - -- the Redis/Valkey stream is a durable work queue, not the long-term audit log; -- `maxLen` is an approximate upper bound if Redis `MAXLEN ~` is used; -- too-low values can drop events before slow consumers catch up; -- too-high values can create memory pressure and long restart/reload times. - -## Configuration recommendation for current deployments - -Until defaults change, set this explicitly: - -```yaml -queue: - redis: - maxLen: 10000 -``` - -For demo/dev deployments with debug stream enabled: - -```yaml -webhook: - audit: - debugStream: - enabled: true - maxLen: 10000 -``` - -## Acceptance criteria - -- A fresh chart install renders a non-zero `--audit-redis-max-len` by default. -- Debug stream, when enabled, renders a non-zero - `--audit-debug-redis-max-len` by default. -- Setting either value to `0` still works, but logs a startup warning. -- Unit tests cover bounded `MAXLEN ~` behavior and explicit unbounded `0` - behavior for the canonical and debug streams. -- Chart tests or rendered-manifest assertions cover the bounded defaults and - explicit `0` opt-out. -- Queue stream length, consumer-group lag, pending entries, and oldest - entry/pending ages are observable via metrics. -- Helm README and project documentation explain the retention tradeoff and - recommend sizing guidance. -- A long-running demo cluster under normal audit traffic does not grow Valkey - into multi-GB RDB files without explicit unbounded configuration. - -## Related bug - -This was discovered while investigating generated-name `CommitRequest` objects -stuck in `WaitingForAuditEvent`. The queue growth was not the root cause of -that bug, but it made the failure mode worse by causing audit ingestion outages. - -See: - -```text -docs/tasks/generated-name-support.md -``` diff --git a/hack/attribution-diagnostics.sh b/hack/attribution-diagnostics.sh new file mode 100755 index 00000000..15f20303 --- /dev/null +++ b/hack/attribution-diagnostics.sh @@ -0,0 +1,106 @@ +#!/usr/bin/env bash +# Post-run diagnostics for the author-attribution path. +# +# When no audit fact matches within the grace window (watch.DefaultAttributionGraceWindow, +# 3s), the commit is visibly authored as `unknown (attribution unresolved)` rather than as +# the configured committer. For a live change that should be attributable, that author is a +# concrete signal to check audit-attribution configuration and delivery. +# +# It answers the one question that splits the diagnosis in two: +# +# Is the fact ABSENT from Redis (the apiserver never delivered it — batching, queue +# overflow, a routing error), or is it PRESENT but was written too late for the 3s +# grace window (the resolver gave up first)? +# +# Fact keys carry a 10-minute TTL, so the remaining TTL back-computes when each fact +# landed: age = ttl_total - ttl_remaining. Comparing that against the fact's own +# stageTimestamp gives the apiserver→Redis delivery lag directly. +# +# Usage: hack/attribution-diagnostics.sh [name-substring] +# Run it IMMEDIATELY after an e2e run — facts expire after 10 minutes. +set -euo pipefail + +CTX="${CTX:-k3d-gitops-reverser-test-e2e}" +VALKEY_NS="${VALKEY_NS:-valkey-e2e}" +PROM_NS="${PROM_NS:-prometheus-operator}" +FACT_TTL_SECONDS="${FACT_TTL_SECONDS:-600}" +FILTER="${1:-}" + +kc() { kubectl --context "${CTX}" "$@"; } + +echo "== attribution resolution outcomes (Prometheus) ==" +echo " absent = Git author was unknown (attribution unresolved); no actor was named." +prom_pod_port=9097 +kc -n "${PROM_NS}" port-forward svc/prometheus-operated ${prom_pod_port}:9090 >/dev/null 2>&1 & +pf=$! +trap 'kill ${pf} 2>/dev/null || true' EXIT +sleep 5 +curl -sG "http://localhost:${prom_pod_port}/api/v1/query" \ + --data-urlencode 'query=sum by (result) (gitopsreverser_attribution_resolutions_total)' | + python3 -c ' +import json,sys +d=json.load(sys.stdin) +rows=[(r["metric"].get("result","?"), float(r["value"][1])) for r in d.get("data",{}).get("result",[])] +total=sum(v for _,v in rows) or 1 +for k,v in sorted(rows, key=lambda x:-x[1]): + print(f" {k:32s} {v:8.0f} ({100*v/total:5.1f}%)") +total_label="TOTAL" +print(f" {total_label:32s} {total:8.0f}") +absent=dict(rows).get("absent",0) +if absent: + print(f"\n >> {absent:.0f} resolution(s) produced the explicit unresolved Git author.") + print( " >> For a change that should be attributable, inspect the audit policy, route,") + print( " >> source identity, and Redis delivery below.") +' || echo " (query failed)" + +echo +echo "== apiserver-side audit delivery ==" +curl -sG "http://localhost:${prom_pod_port}/api/v1/query" \ + --data-urlencode 'query=sum(apiserver_audit_requests_rejected_total) or vector(0)' | + python3 -c 'import json,sys; d=json.load(sys.stdin); r=d.get("data",{}).get("result",[]); print(" apiserver_audit_requests_rejected_total =", r[0]["value"][1] if r else "n/a (not scraped)")' \ + || echo " (query failed)" +curl -sG "http://localhost:${prom_pod_port}/api/v1/query" \ + --data-urlencode 'query=sum(gitopsreverser_audit_eventlist_events_total) or vector(0)' | + python3 -c 'import json,sys; d=json.load(sys.stdin); r=d.get("data",{}).get("result",[]); print(" audit events INGESTED by us =", r[0]["value"][1] if r else "0")' \ + || echo " (query failed)" +kill ${pf} 2>/dev/null || true + +echo +echo "== attribution facts present in Valkey ==" +pw="$(kc -n "${VALKEY_NS}" get secret valkey-auth -o jsonpath='{.data.*}' | head -1 | base64 -d)" +pod="$(kc -n "${VALKEY_NS}" get pods -o jsonpath='{.items[0].metadata.name}')" +vc() { kc -n "${VALKEY_NS}" exec "${pod}" -c valkey -- valkey-cli -a "${pw}" --no-auth-warning "$@" 2>/dev/null; } + +keys="$(vc --scan --pattern '*author:v1:audit:*' | { [ -n "${FILTER}" ] && grep -F "${FILTER}" || cat; } | head -40)" +if [ -z "${keys}" ]; then + echo " no facts matching ${FILTER:-}" + echo " >> If a commit has the unresolved author AND no fact exists, check audit delivery:" + echo " >> check audit-webhook-batch-max-wait/-size and the /audit-webhook/ route." + exit 0 +fi + +echo " (fact age is back-computed from the remaining TTL against a ${FACT_TTL_SECONDS}s TTL)" +for k in ${keys}; do + val="$(vc GET "${k}")" + ttl="$(vc TTL "${k}")" + python3 - "${k}" "${val}" "${ttl}" "${FACT_TTL_SECONDS}" <<'PY' +import json,sys +key,val,ttl,total = sys.argv[1],sys.argv[2],sys.argv[3],int(sys.argv[4]) +try: + f=json.loads(val) +except Exception: + print(f" {key} "); raise SystemExit +try: + age=total-int(ttl) +except ValueError: + age="?" +ns=f.get("namespace",""); name=f.get("name","") +print(f" {ns}/{name} verb={f.get('verb','?')} author={f.get('author','?')}") +print(f" stageTimestamp={f.get('stageTimestamp','?')} written_to_redis≈{age}s ago") +PY +done + +echo +echo " Reading this: a fact PRESENT here for an object whose commit has the unresolved author means" +echo " the fact arrived, just not within the 3s grace — the grace window (or the audit" +echo " batching delay ahead of it) is the problem, not fact delivery." diff --git a/internal/controller/commitrequest_attribution_agreement_test.go b/internal/controller/commitrequest_attribution_agreement_test.go new file mode 100644 index 00000000..59f6bdfa --- /dev/null +++ b/internal/controller/commitrequest_attribution_agreement_test.go @@ -0,0 +1,152 @@ +// SPDX-License-Identifier: Apache-2.0 + +package controller + +import ( + "context" + "testing" + + "github.com/go-logr/logr" + "github.com/stretchr/testify/assert" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + k8stypes "k8s.io/apimachinery/pkg/types" + + configv1alpha3 "github.com/ConfigButler/gitops-reverser/api/v1alpha3" + "github.com/ConfigButler/gitops-reverser/internal/git" + "github.com/ConfigButler/gitops-reverser/internal/queue" + "github.com/ConfigButler/gitops-reverser/internal/watch" +) + +// absentLookup is an attribution index that never has a fact — the production shape of +// "attribution is enabled but nothing matched within the grace". +type absentLookup struct{} + +func (absentLookup) LookupAuthorResolution( + _ context.Context, + _ string, + _ schema.GroupVersionResource, + _ k8stypes.UID, + _ string, + _ bool, +) queue.AuthorResolution { + return queue.AuthorResolution{Result: queue.AttributionAbsent} +} + +// windowOutcomeAttributionOff is the outcome a commit window carries in configured-author mode. +// It is read off a zero git.Event on purpose rather than written as a constant: that IS the +// production value, because watch.Manager.attachAuthor returns early without assigning +// Attribution when the Manager's AuthorResolver is nil. +func windowOutcomeAttributionOff() git.AttributionOutcome { + return git.Event{}.Attribution +} + +// windowOutcomeAttributionMissed drives the real resolver over a lookup that has no fact, which +// is what a live watch event gets when attribution is on and nothing matched in the grace. +func windowOutcomeAttributionMissed(t *testing.T) git.AttributionOutcome { + t.Helper() + _, outcome := watch.NewAuthorResolver(absentLookup{}, 0, logr.Discard()).ResolveAuthor( + context.Background(), + "default", + schema.GroupVersionResource{Version: "v1", Resource: "configmaps"}, + k8stypes.UID("uid-1"), + "101", + true, + ) + return outcome +} + +// TestCommitRequest_OutcomesAgree is the cross-subsystem half of the P0 regression test. +// +// The gap that let the blocker through was that every existing test set both sides of the +// comparison to the same literal, so nothing ever checked that the two INDEPENDENT producers +// emit compatible values. This drives each side from its real source — the controller's +// gitOutcome projection, and the watch package's actual resolver / actual zero event — and +// asserts they agree on the only thing matching may depend on across that boundary. +// +// The default deployment is the first row: --admission-webhook defaults to false, so the +// controller emits attributionNotAttempted, while --author-attribution=false leaves the window's +// outcome at the zero value. When those two were different strings, no CommitRequest could +// attach to any window and the user's commit message was silently dropped. +func TestCommitRequest_OutcomesAgree(t *testing.T) { + tests := []struct { + name string + requestSide commitRequestAttribution + windowSide git.AttributionOutcome + wantNamesActor bool + }{ + { + name: "default deployment: webhook off, attribution off", + requestSide: attributionNotAttempted, + windowSide: windowOutcomeAttributionOff(), + }, + { + name: "webhook off, attribution on but missed", + requestSide: attributionNotAttempted, + windowSide: windowOutcomeAttributionMissed(t), + }, + { + name: "webhook on but no record, attribution off", + requestSide: attributionCommitter, + windowSide: windowOutcomeAttributionOff(), + }, + { + name: "webhook on but no record, attribution on but missed", + requestSide: attributionCommitter, + windowSide: windowOutcomeAttributionMissed(t), + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + requestOutcome := tc.requestSide.gitOutcome() + assert.Equal(t, tc.wantNamesActor, requestOutcome.NamesActor()) + assert.Equal(t, requestOutcome.NamesActor(), tc.windowSide.NamesActor(), + "the CommitRequest side produced %q and the window side produced %q; they must "+ + "agree on whether an actor was named, or the request cannot attach and the "+ + "user's commit message is silently dropped", + requestOutcome, tc.windowSide) + }) + } +} + +// TestAttributionOutcome_OffLeavesZeroOutcome states the invariant the default deployment rests on, +// separately from the agreement table so a regression names itself precisely. +func TestAttributionOutcome_OffLeavesZeroOutcome(t *testing.T) { + assert.Equal(t, git.AttributionNotAttempted, windowOutcomeAttributionOff(), + "configured-author mode must leave the event's outcome at AttributionNotAttempted") + assert.Equal(t, git.AttributionNotAttempted, attributionNotAttempted.gitOutcome(), + "a CommitRequest with command-author capture off must carry AttributionNotAttempted") + assert.Equal(t, git.AttributionUnresolved, windowOutcomeAttributionMissed(t), + "attribution that ran and found nothing must be AttributionUnresolved, not the zero value") +} + +// TestCommitRequestAttribution_GitOutcome keeps every controller outcome's projection explicit, +// including the defensive zero value. +func TestCommitRequestAttribution_GitOutcome(t *testing.T) { + tests := []struct { + name string + input commitRequestAttribution + want git.AttributionOutcome + }{ + {name: "unset is not attempted", input: attributionUnset, want: git.AttributionNotAttempted}, + {name: "admission capture is resolved", input: attributionFromAdmission, want: git.AttributionResolved}, + {name: "admission miss is unresolved", input: attributionCommitter, want: git.AttributionUnresolved}, + {name: "capture disabled is not attempted", input: attributionNotAttempted, want: git.AttributionNotAttempted}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.want, tc.input.gitOutcome()) + }) + } +} + +// TestCommitRequestAttribution_UnsetStatusIsCaptureDisabled keeps an unset decision visible and safe. +func TestCommitRequestAttribution_UnsetStatusIsCaptureDisabled(t *testing.T) { + var request configv1alpha3.CommitRequest + + setCommitRequestAttributed(&request, attributionUnset) + + requireCondition(t, request, ConditionTypeAuthorAttributed, metav1.ConditionFalse, crReasonAuthorCaptureDisabled) +} diff --git a/internal/controller/commitrequest_controller.go b/internal/controller/commitrequest_controller.go index 522d61fd..37a643da 100644 --- a/internal/controller/commitrequest_controller.go +++ b/internal/controller/commitrequest_controller.go @@ -22,15 +22,15 @@ import ( ) // CommitRequestFinalizer is the EventRouter seam the reconciler drives, using the -// attach-then-poll protocol (docs/spec/commitrequest-design.md §6.4.3): +// attach-then-poll protocol (docs/spec/commitrequest-design.md): // ServiceCommitRequest registers the attach idempotently on the GitTarget's branch // worker (bind the message to the author's open window, finalize after the grace) // and returns the request's current outcome — resolved=false means keep polling. // watch.EventRouter satisfies it without adaptation. // -// There is no watermark barrier (§6.3): UC1 is covered by the human gap between the +// There is no watermark barrier: the interactive case is covered by the human gap between the // edit and the save, UC2 by the collect-grace. The grace is anchored at attribution -// — the worker stamps finalizeAt = receipt + closeDelaySeconds (§6.4.4) — so the +// — the worker stamps finalizeAt = receipt + closeDelaySeconds — so the // controller no longer holds the finalize itself. type CommitRequestFinalizer interface { ServiceCommitRequest(ctx context.Context, attach git.AttachCommitRequest) (git.FinalizeResult, bool, error) @@ -39,9 +39,9 @@ type CommitRequestFinalizer interface { // CommandAuthorLookup resolves the author of a CommitRequest from the submitter // captured at admission by the validate-operator-types webhook, keyed by the persisted // object's UID. *queue.CommandAuthorStore satisfies it without adaptation. The lookup -// is present-or-never (docs/spec/commitrequest-admission-authorship.md §2): a miss is +// is present-or-never (docs/spec/commitrequest-admission-authorship.md): a miss is // immediate and final — the webhook is not configured (or a best-effort write missed) — -// and the controller finalizes as the committer with no wait. +// and the request claims no actor with no wait. type CommandAuthorLookup interface { LookupCommandAuthor(ctx context.Context, uid types.UID) (queue.CommandAuthor, bool) } @@ -58,26 +58,26 @@ const resolveTimeoutMessage = "the CommitRequest finalize did not resolve within const ( // commitRequestPollInterval is the requeue cadence while polling the worker - // for the attached request's outcome (attach-then-poll, §6.4.3). + // for the attached request's outcome (attach-then-poll). commitRequestPollInterval = 2 * time.Second // commitRequestResolveTimeout bounds the attach-then-poll wait, measured from // object creation: it must cover the maximum collect-grace (closeDelaySeconds ≤ 300s, // anchored at attribution) and the push cooldown plus retries. Authorship is now - // settled synchronously at first sight (no attribution wait, §2), so the former + // settled synchronously at first sight (no attribution wait), so the former // +60s attribution component is gone. Past it, a request the worker never resolved // (e.g. a vanished worker) fails closed instead of polling forever. commitRequestResolveTimeout = 300*time.Second + 120*time.Second ) // CommitRequestReconciler drives a CommitRequest through its state machine -// (docs/spec/commitrequest-design.md §6.4 and -// docs/spec/commitrequest-admission-authorship.md §5): +// (docs/spec/commitrequest-design.md and +// docs/spec/commitrequest-admission-authorship.md): // // 1. ATTRIBUTE — a single synchronous read of the submitter captured at admission -// (present-or-never, §2). A hit names that submitter as the author -// (AuthorAttributed=True); a miss falls back to the configured committer -// immediately (AuthorAttributed=False). There is no wait and no requeue for the +// (present-or-never). A hit names that submitter as the author +// (AuthorAttributed=True); a miss claims no actor immediately +// (AuthorAttributed=False). There is no wait and no requeue for the // author: the record is written before the object is visible, so waiting cannot // help. // 2. ATTACH + POLL — the instant the author is settled, send the attach to the @@ -99,7 +99,8 @@ type CommitRequestReconciler struct { // Finalizer attaches the request to the author-bound open window and reports // its outcome; AuthorLookup resolves the submitter captured at admission. When // AuthorLookup is nil (the validate-operator-types webhook is disabled), requests - // finalize as the configured committer — immediately, with AuthorAttributed=False. + // claim no actor immediately, with AuthorAttributed=False. The attached window + // determines the eventual Git author. Finalizer CommitRequestFinalizer AuthorLookup CommandAuthorLookup } @@ -125,9 +126,9 @@ func (r *CommitRequestReconciler) Reconcile(ctx context.Context, req ctrl.Reques return ctrl.Result{}, nil } - // 1. ATTRIBUTE: settle the commit author synchronously (present-or-never, §2). - // A hit names the admission submitter; a miss is the configured committer. Either - // way the decision is final — there is no wait and no requeue for the author. + // 1. ATTRIBUTE: settle the request's actor synchronously (present-or-never). + // A hit names the admission submitter; a miss claims no actor. Either way the + // decision is final — there is no wait and no requeue for the author. author, attribution := r.attributeAuthor(ctx, commitRequest) // First sight: stamp the still-running conditions so the object reports its @@ -143,12 +144,13 @@ func (r *CommitRequestReconciler) Reconcile(ctx context.Context, req ctrl.Reques // 2. ATTACH + POLL: register the attach idempotently the instant we attribute // (no controller-side delay — the worker anchors the grace at attribution, - // §6.4.4) and poll the outcome. + // close-delay contract) and poll the outcome. result, resolved, serviceErr := r.Finalizer.ServiceCommitRequest(ctx, git.AttachCommitRequest{ Namespace: commitRequest.Namespace, Name: commitRequest.Name, UID: string(commitRequest.UID), Author: author.Author, + Attribution: attribution.gitOutcome(), GitTargetName: commitRequest.Spec.TargetRef.Name, GitTargetNamespace: commitRequest.Namespace, Message: capCommitRequestMessage(commitRequest.Spec.Message), @@ -181,30 +183,32 @@ func (r *CommitRequestReconciler) Reconcile(ctx context.Context, req ctrl.Reques } // attributeAuthor settles the commit author with a single synchronous lookup of the -// submitter captured at admission (present-or-never, §2). It never waits: a nil -// AuthorLookup (the validate-operator-types webhook is disabled) or a miss both resolve to -// the configured committer immediately. The miss case is final — the record is written -// before the object is visible, so there is no asynchronous arrival to wait for. +// submitter captured at admission (present-or-never). It never waits: a nil +// AuthorLookup (the validate-operator-types webhook is disabled) or a miss both claim no +// actor immediately. The miss case is final — the record is written before the object is +// visible, so there is no asynchronous arrival to wait for. // // The lookup result is logged at Info: it is the counterpart to the admission handler's // "recorded command author" line, so a hit/miss pair makes the whole capture→read path -// legible (the first thing to check when a CommitRequest commits as the committer). +// legible (the first thing to check when a request unexpectedly claims no actor). func (r *CommitRequestReconciler) attributeAuthor( ctx context.Context, commitRequest *configbutleraiv1alpha3.CommitRequest, ) (queue.CommandAuthor, commitRequestAttribution) { log := logf.FromContext(ctx).WithName("CommitRequestReconciler") if r.AuthorLookup == nil { - log.Info("command-author lookup disabled (validate-operator-types webhook off); committing as committer", + log.Info("command-author lookup disabled (validate-operator-types webhook off); request claims no actor", "name", client.ObjectKeyFromObject(commitRequest), "uid", commitRequest.UID) - return queue.CommandAuthor{}, attributionCommitter + // Capture is off, so nothing was attempted — distinct from a capture that ran and + // found no record, which is what attributionCommitter now means. + return queue.CommandAuthor{}, attributionNotAttempted } if author, ok := r.AuthorLookup.LookupCommandAuthor(ctx, commitRequest.UID); ok { log.Info("command author resolved from admission record", "name", client.ObjectKeyFromObject(commitRequest), "uid", commitRequest.UID, "author", author.Author) return author, attributionFromAdmission } - log.Info("no admission command-author record found; committing as committer", + log.Info("no admission command-author record found; request claims no actor", "name", client.ObjectKeyFromObject(commitRequest), "uid", commitRequest.UID) return queue.CommandAuthor{}, attributionCommitter } @@ -341,9 +345,9 @@ func (r *CommitRequestReconciler) writeTerminalStatus( // multi-CommitRequest ordering design — concurrent CommitRequests for the same // GitTarget are serialized exactly as a dedicated finalize-coordinator // goroutine would serialize them, without the extra moving parts (see -// docs/spec/commitrequest-multi-finalize-design.md). +// docs/spec/commitrequest-design.md). // -// Restart recovery is best-effort by design (commitrequest-design.md §6.6): the +// Restart recovery is best-effort by design: the // message is durable in spec.message, so on restart any non-terminal request is // re-reconciled — author-resolved from the admission cache when present and // re-attached — which heals the common cases. The one knowingly-accepted gap is a request whose diff --git a/internal/controller/commitrequest_controller_unit_test.go b/internal/controller/commitrequest_controller_unit_test.go index 490868ec..9b21fb58 100644 --- a/internal/controller/commitrequest_controller_unit_test.go +++ b/internal/controller/commitrequest_controller_unit_test.go @@ -258,11 +258,11 @@ func TestCommitRequestReconcile_FinalizeErrorBecomesFailed(t *testing.T) { requireCondition(t, got, ConditionTypePushed, metav1.ConditionFalse, "") } -// A lookup miss (the validate-operator-types webhook recorded no author for this object) -// resolves to the configured committer immediately — present-or-never, no wait. Even a -// freshly created CommitRequest attaches at once with a blank author and records -// AuthorAttributed=False (CommitterFallback). -func TestCommitRequestReconcile_LookupMissFallsBackToCommitter(t *testing.T) { +// TestCommitRequestReconcile_LookupMissClaimsNoActor verifies that a lookup miss claims no actor +// immediately — present-or-never, no wait. A freshly created CommitRequest attaches at once with a +// blank author and records AuthorAttributed=False (CommitterFallback). The matching window determines +// the Git author. +func TestCommitRequestReconcile_LookupMissClaimsNoActor(t *testing.T) { cr := newCommitRequest("save-fresh") cr.CreationTimestamp = metav1.Now() // fresh: the old path would have requeued here c := newCommitRequestClient(t, nil, cr) @@ -278,7 +278,7 @@ func TestCommitRequestReconcile_LookupMissFallsBackToCommitter(t *testing.T) { assert.Zero(t, res.RequeueAfter, "a miss is final; the request must not poll for an author") assert.Equal(t, 1, lookup.calls, "the author is looked up exactly once, synchronously") require.Len(t, f.calls, 1, "the attach is sent immediately on a miss") - assert.Empty(t, f.calls[0].Author, "a miss attaches with a blank author (the configured committer)") + assert.Empty(t, f.calls[0].Author, "a miss attaches with a blank, unnamed actor") got := fetchCommitRequest(t, c, "save-fresh") requireCondition(t, got, ConditionTypeReady, metav1.ConditionTrue, crReasonCommitted) @@ -380,8 +380,8 @@ func TestCommitRequestReconcile_StaleCacheEchoDoesNotRefinalize(t *testing.T) { } // Webhook-disabled mode (no AuthorLookup) never waits: a freshly created CommitRequest -// attaches immediately with a blank author and commits as the configured committer, -// recording AuthorAttributed=False (CommitterFallback). +// attaches immediately with a blank author and records AuthorAttributed=False +// (AuthorCaptureDisabled). The matching window determines the Git author. func TestCommitRequestReconcile_ConfiguredAuthorCommitsWithoutWaiting(t *testing.T) { cr := newCommitRequest("save-configured-author") cr.CreationTimestamp = metav1.Now() // fresh: a waiting path would requeue instead of commit @@ -398,13 +398,13 @@ func TestCommitRequestReconcile_ConfiguredAuthorCommitsWithoutWaiting(t *testing got := fetchCommitRequest(t, c, "save-configured-author") requireCondition(t, got, ConditionTypeReady, metav1.ConditionTrue, crReasonCommitted) requireCondition(t, got, ConditionTypePushed, metav1.ConditionTrue, crReasonPushed) - requireCondition(t, got, ConditionTypeAuthorAttributed, metav1.ConditionFalse, crReasonCommitterFallback) + requireCondition(t, got, ConditionTypeAuthorAttributed, metav1.ConditionFalse, crReasonAuthorCaptureDisabled) assert.Equal(t, "c0ffee", got.Status.SHA) require.Len(t, f.calls, 1, "the attach is sent immediately, with no attribution wait") assert.Empty(t, f.calls[0].Author, "webhook-disabled mode attaches with a blank author") } -// Webhook-disabled mode settles AuthorAttributed=False (CommitterFallback) immediately +// Webhook-disabled mode settles AuthorAttributed=False (AuthorCaptureDisabled) immediately // and never parks the request in a "waiting for the author" state: even while the // attach is still being polled, the request is in the WaitingForCloseDelay wait. func TestCommitRequestReconcile_ConfiguredAuthorAttributedImmediately(t *testing.T) { @@ -421,7 +421,7 @@ func TestCommitRequestReconcile_ConfiguredAuthorAttributedImmediately(t *testing assert.Empty(t, f.calls[0].Author) got := fetchCommitRequest(t, c, "save-committer-poll") - requireCondition(t, got, ConditionTypeAuthorAttributed, metav1.ConditionFalse, crReasonCommitterFallback) + requireCondition(t, got, ConditionTypeAuthorAttributed, metav1.ConditionFalse, crReasonAuthorCaptureDisabled) requireCondition(t, got, ConditionTypeReconciling, metav1.ConditionTrue, crReasonWaitingForCloseDelay) assert.False(t, commitRequestIsTerminal(&got), "in the close-delay wait, not terminal") } @@ -514,7 +514,7 @@ func TestApplyFinalizeResultToStatus(t *testing.T) { assert.Equal(t, "main", cr.Status.Branch) }) - t.Run("no window in grace is a benign ready (committer fallback)", func(t *testing.T) { + t.Run("no window in grace is a benign ready (no admission author)", func(t *testing.T) { var cr configv1alpha3.CommitRequest applyFinalizeResultToStatus(&cr, git.FinalizeResult{Outcome: git.FinalizeNoOpenWindow, Branch: "main"}, nil, attributionCommitter) diff --git a/internal/controller/commitrequest_finalize.go b/internal/controller/commitrequest_finalize.go index 7d197b9e..ac161d31 100644 --- a/internal/controller/commitrequest_finalize.go +++ b/internal/controller/commitrequest_finalize.go @@ -28,6 +28,7 @@ const ( crReasonUnexpectedOutcome = "UnexpectedOutcome" crReasonAttributedFromAdmission = "AttributedFromAdmission" crReasonCommitterFallback = "CommitterFallback" + crReasonAuthorCaptureDisabled = "AuthorCaptureDisabled" crReasonPushed = "Pushed" ) @@ -50,15 +51,39 @@ const notStalledMessage = "the CommitRequest is not stalled" type commitRequestAttribution int const ( + // attributionUnset is the defensive zero value. It must never report a resolved + // actor when a caller neglected to initialize the attribution decision. + attributionUnset commitRequestAttribution = iota // attributionFromAdmission means the submitter was captured at admission by the // validate-operator-types webhook and named as the commit author. - attributionFromAdmission commitRequestAttribution = iota - // attributionCommitter means no admission author record exists — the - // validate-operator-types webhook is not configured (or did not record one) — so the - // commit is authored by the configured committer. + attributionFromAdmission + // attributionCommitter means command-author capture ran but no admission author record + // exists, so the request does not claim an actor. attributionCommitter + // attributionNotAttempted means command-author capture is switched off entirely (the + // validate-operator-types webhook is not running), so no submitter was ever sought. It is + // distinct from attributionCommitter, where capture DID run and found nothing. + attributionNotAttempted ) +// gitOutcome projects the controller's attribution state onto the git layer's outcome, which +// is what the worker matches windows on. Keeping the projection here — rather than passing the +// controller's own enum down — means the git package never learns about admission records. +func (a commitRequestAttribution) gitOutcome() git.AttributionOutcome { + switch a { + case attributionUnset: + return git.AttributionNotAttempted + case attributionFromAdmission: + return git.AttributionResolved + case attributionCommitter: + return git.AttributionUnresolved + case attributionNotAttempted: + return git.AttributionNotAttempted + default: + return git.AttributionNotAttempted + } +} + // setCommitRequestCondition upserts a condition keyed by type, stamping it with // the request's generation so kstatus can tell a current status from a stale one. func setCommitRequestCondition( @@ -102,19 +127,32 @@ func markCommitRequestWaitingForCloseDelay(cr *configv1alpha3.CommitRequest, att } // setCommitRequestAttributed records the settled, binary author decision on the -// AuthorAttributed condition. False (CommitterFallback) is not a failure and does not -// affect Ready — it is the honest signal that no admission author record was found. +// AuthorAttributed condition. A False reason distinguishes an absent record +// (CommitterFallback) from capture disabled (AuthorCaptureDisabled). Neither is a failure or +// affects Ready; the request claims no actor. func setCommitRequestAttributed(cr *configv1alpha3.CommitRequest, attribution commitRequestAttribution) { switch attribution { + case attributionUnset: + setCommitRequestCondition(cr, ConditionTypeAuthorAttributed, metav1.ConditionFalse, + crReasonAuthorCaptureDisabled, + "command-author capture state was unset; the request can attach only to a window with no named actor") case attributionFromAdmission: setCommitRequestCondition(cr, ConditionTypeAuthorAttributed, metav1.ConditionTrue, crReasonAttributedFromAdmission, "the submitter was captured at admission and named as the commit author") case attributionCommitter: + // Capture RAN and found nothing. Distinct from attributionNotAttempted below, whose + // message this used to carry — reporting "the webhook is not configured" for a + // webhook that is configured and simply missed sends operators to the wrong place. setCommitRequestCondition(cr, ConditionTypeAuthorAttributed, metav1.ConditionFalse, crReasonCommitterFallback, - "no admission author record (the validate-operator-types webhook is not configured); "+ - "committed as the configured committer") + "no admission author record was found for this request; it can attach only to a "+ + "window with no named actor") + case attributionNotAttempted: + setCommitRequestCondition(cr, ConditionTypeAuthorAttributed, metav1.ConditionFalse, + crReasonAuthorCaptureDisabled, + "command-author capture is disabled (the validate-operator-types webhook is not "+ + "configured); the request can attach only to a window with no named actor") } } diff --git a/internal/controller/commitrequest_kstatus_test.go b/internal/controller/commitrequest_kstatus_test.go index ba13756b..d77e210f 100644 --- a/internal/controller/commitrequest_kstatus_test.go +++ b/internal/controller/commitrequest_kstatus_test.go @@ -22,7 +22,7 @@ func TestCommitRequestKstatusContract(t *testing.T) { wantMsg string }{ { - name: "committer fallback in the close-delay wait (AuthorAttributed=False is not a failure)", + name: "missing admission author in the close-delay wait (AuthorAttributed=False is not a failure)", conds: []map[string]interface{}{ conditionMap(ConditionTypeReady, "False", crReasonWaitingForCloseDelay, closeDelayMessage), conditionMap(ConditionTypeReconciling, "True", crReasonWaitingForCloseDelay, closeDelayMessage), diff --git a/internal/controller/constants.go b/internal/controller/constants.go index f5146534..a36b92fe 100644 --- a/internal/controller/constants.go +++ b/internal/controller/constants.go @@ -46,9 +46,10 @@ const ( // was named from the submitter captured at admission. It is binary and immediately // settled (no Unknown, no timeout): True (AttributedFromAdmission) when the // validate-operator-types webhook recorded the submitter, False (CommitterFallback) when - // no admission record exists — the webhook is not configured — and the commit is - // authored by the configured committer. False is not a failure and does not affect - // Ready (docs/spec/commitrequest-admission-authorship.md §5). + // capture ran but found no record, or False (AuthorCaptureDisabled) when capture is off. + // False is not a failure and does not affect Ready: the + // request claims no actor and can attach only to an unnamed watch window + // (docs/architecture.md#commitrequest-finalize). ConditionTypeAuthorAttributed = "AuthorAttributed" // ConditionTypePushed indicates whether a CommitRequest's commit reached the // remote repository. diff --git a/internal/git/attribution_outcome_test.go b/internal/git/attribution_outcome_test.go new file mode 100644 index 00000000..9218d36d --- /dev/null +++ b/internal/git/attribution_outcome_test.go @@ -0,0 +1,259 @@ +// SPDX-License-Identifier: Apache-2.0 + +package git + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/ConfigButler/gitops-reverser/internal/types" +) + +// attributedEvent builds a commit-shaped event carrying an explicit attribution outcome. +func attributedEvent(username string, outcome AttributionOutcome) Event { + return Event{ + Identifier: types.NewResourceIdentifier("", "v1", "configmaps", "team-a", "app"), + Operation: "CREATE", + UserInfo: UserInfo{Username: username}, + Attribution: outcome, + GitTargetName: "target", + GitTargetNamespace: "default", + } +} + +func commitWrite(events ...Event) PendingWrite { + return PendingWrite{Kind: PendingWriteCommit, Events: events} +} + +// An attribution that RAN and did not resolve must NOT be authored by the committer. Authoring +// it as the committer is what made a lost actor byte-identical to a configured-author commit, +// so the loss was invisible in git history. +func TestCommitOptionsFor_UnresolvedAttributionUsesSentinelAuthor(t *testing.T) { + config := ResolveCommitConfig(nil) + when := time.Now() + + options := commitOptionsFor(commitWrite(attributedEvent("", AttributionUnresolved)), config, nil, when) + + require.NotNil(t, options.Author) + require.NotNil(t, options.Committer) + assert.Equal(t, UnresolvedAuthorDisplayName, options.Author.Name) + assert.Equal(t, UnresolvedAuthorEmail, options.Author.Email) + assert.Equal(t, DefaultCommitterName, options.Committer.Name, + "the operator really did write the commit, so it stays the committer") + assert.NotEqual(t, options.Committer.Name, options.Author.Name, + "an unresolved attribution must be distinguishable from a configured-author commit") +} + +// Configured-author mode is untouched: nothing was attempted, so the committer legitimately +// authors the commit. This is the case that MUST stay identical to the sentinel case's +// opposite — the whole point is that the two are now told apart. +func TestCommitOptionsFor_NotAttemptedStillAuthorsAsCommitter(t *testing.T) { + config := ResolveCommitConfig(nil) + when := time.Now() + + options := commitOptionsFor(commitWrite(attributedEvent("", AttributionNotAttempted)), config, nil, when) + + require.NotNil(t, options.Author) + assert.Equal(t, DefaultCommitterName, options.Author.Name) + assert.Equal(t, DefaultCommitterEmail, options.Author.Email) + assert.Equal(t, options.Committer.Name, options.Author.Name) +} + +func TestCommitOptionsFor_ResolvedAttributionNamesTheActor(t *testing.T) { + config := ResolveCommitConfig(nil) + + options := commitOptionsFor( + commitWrite(attributedEvent("jane@acme.com", AttributionResolved)), config, nil, time.Now()) + + require.NotNil(t, options.Author) + assert.Equal(t, "jane@acme.com", options.Author.Name) + assert.NotEqual(t, UnresolvedAuthorDisplayName, options.Author.Name) +} + +// The sentinel is written into a git signature header, so it must survive the same safety +// check every other author value does — spaces and parentheses included. +func TestUnresolvedAuthor_IsSafeInASignatureHeader(t *testing.T) { + user := UnresolvedAuthor() + assert.True(t, isSafeSignatureField(user.DisplayName), + "the display form must be placeable verbatim in a git signature") + assert.Equal(t, UnresolvedAuthorDisplayName, authorName(user)) + assert.Equal(t, UnresolvedAuthorEmail, authorEmail(user), + "the reserved .invalid address must pass email validation and be used verbatim") +} + +// author_kind drives dashboards. An unresolved attribution must never be counted as a named +// user: that would make a DEGRADING attribution path read as an improving one. +func TestAuthorKind_ClassifiesAllFourKinds(t *testing.T) { + for _, tc := range []struct { + name string + write PendingWrite + want string + }{ + { + name: "resolved human is a user", + write: commitWrite(attributedEvent("jane@acme.com", AttributionResolved)), + want: authorKindUser, + }, + { + name: "resolved service account is a serviceaccount", + write: commitWrite(attributedEvent("system:serviceaccount:flux-system:kustomize", AttributionResolved)), + want: authorKindServiceAccount, + }, + { + name: "attribution never attempted is the committer", + write: commitWrite(attributedEvent("", AttributionNotAttempted)), + want: authorKindCommitter, + }, + { + name: "attribution attempted and unresolved is its own kind", + write: commitWrite(attributedEvent("", AttributionUnresolved)), + want: authorKindUnresolved, + }, + { + name: "an atomic write never attempts attribution", + write: PendingWrite{Kind: PendingWriteAtomic}, + want: authorKindCommitter, + }, + } { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.want, tc.write.authorKind()) + }) + } +} + +// Even if a sentinel username ever reached authorKind through a path that lost the outcome, +// it must not be silently counted as a real user. This pins the intent rather than the +// mechanism, so a future refactor that drops the outcome check fails here. +func TestAuthorKind_SentinelUsernameIsNeverCountedAsAUser(t *testing.T) { + write := commitWrite(attributedEvent(UnresolvedAuthorUsername, AttributionUnresolved)) + assert.Equal(t, authorKindUnresolved, write.authorKind()) + assert.NotEqual(t, authorKindUser, write.authorKind()) +} + +// Window coalescing is keyed on (outcome, author): two unresolved events belong together +// exactly as they did when both carried an empty username, and an unresolved event must never +// join a resolved one. +func TestOpenWindow_CanAppendMatchesOnOutcomeAndAuthor(t *testing.T) { + unresolved := attributedEvent("", AttributionUnresolved) + window := newOpenWindow(unresolved, nil) + + assert.True(t, window.canAppend(attributedEvent("", AttributionUnresolved)), + "two unresolved events share one window") + assert.False(t, window.canAppend(attributedEvent("jane@acme.com", AttributionResolved)), + "a named actor must not be folded into an unresolved window") + assert.False(t, window.canAppend(attributedEvent("", AttributionNotAttempted)), + "configured-author writes must not be folded into an unresolved window") +} + +func TestOpenWindow_ResolvedWindowsStillSplitByAuthor(t *testing.T) { + window := newOpenWindow(attributedEvent("jane@acme.com", AttributionResolved), nil) + + assert.True(t, window.canAppend(attributedEvent("jane@acme.com", AttributionResolved))) + assert.False(t, window.canAppend(attributedEvent("bob@acme.com", AttributionResolved))) +} + +// THE REGRESSION THIS DESIGN EXISTS TO AVOID. +// +// A CommitRequest that could not be attributed carries an empty author, and so does a window +// whose attribution ran and named nobody — the sentinel is a git-signature rendering only and +// never reaches the window's Author. The two therefore still match on the author, and the +// outcomes must not be allowed to push them apart: the request's outcome comes from command +// authorship and the window's from mirrored-resource attribution, which are configured +// independently, so they routinely differ. See the full cross-subsystem matrix in +// TestMatchesWindow_AcrossIndependentlyConfiguredSubsystems. +func TestPendingCommitRequest_UnresolvedRequestAttachesToUnresolvedWindow(t *testing.T) { + window := newOpenWindow(attributedEvent("", AttributionUnresolved), nil) + require.Empty(t, window.Author, + "an unresolved window carries no author; the sentinel is applied at the write path") + + request := &pendingCommitRequest{ + author: "", // an unnamed CommitRequest claims no actor + attribution: AttributionUnresolved, + gitTargetName: "target", + gitTargetNamespace: "default", + } + assert.True(t, request.matchesWindow(window), + "an unnamed CommitRequest must attach to the unresolved window it was meant for") + + // The default deployment: command-author capture is off, so the request says "not + // attempted" while the window says "unresolved". Neither names an actor, so they match. + notAttempted := &pendingCommitRequest{ + author: "", + attribution: AttributionNotAttempted, + gitTargetName: "target", + gitTargetNamespace: "default", + } + assert.True(t, notAttempted.matchesWindow(window), + "a request from a deployment with command-author capture off must still attach; "+ + "requiring the two subsystems' outcomes to be equal drops the user's commit message") +} + +func TestPendingCommitRequest_DoesNotCrossAttributionOutcomes(t *testing.T) { + resolvedWindow := newOpenWindow(attributedEvent("jane@acme.com", AttributionResolved), nil) + + unresolvedRequest := &pendingCommitRequest{ + author: "", + attribution: AttributionUnresolved, + gitTargetName: "target", + gitTargetNamespace: "default", + } + assert.False(t, unresolvedRequest.matchesWindow(resolvedWindow), + "an unattributed request must not claim a named actor's window") + + resolvedRequest := &pendingCommitRequest{ + author: "jane@acme.com", + attribution: AttributionResolved, + gitTargetName: "target", + gitTargetNamespace: "default", + } + assert.True(t, resolvedRequest.matchesWindow(resolvedWindow)) + + otherActor := &pendingCommitRequest{ + author: "bob@acme.com", + attribution: AttributionResolved, + gitTargetName: "target", + gitTargetNamespace: "default", + } + assert.False(t, otherActor.matchesWindow(resolvedWindow), + "a resolved request still matches on the actor") +} + +func TestPendingCommitRequest_StillScopedToOneGitTarget(t *testing.T) { + window := newOpenWindow(attributedEvent("", AttributionUnresolved), nil) + request := &pendingCommitRequest{ + author: "", + attribution: AttributionUnresolved, + gitTargetName: "other-target", + gitTargetNamespace: "default", + } + assert.False(t, request.matchesWindow(window), + "the outcome match must not widen attachment across GitTargets") +} + +// The sentinel is scoped to the git author header and deliberately does NOT reach message +// bodies or user-authored {{.Username}} templates — it is derived at the write path from the +// outcome, never stamped onto the event (see UnresolvedAuthor). An unresolved event therefore +// renders {{.Username}} exactly as configured-author mode always has: empty. Pushing a magic +// token in here would change the commit text of every deployment that has attribution misses +// and force user templates to special-case a value they never had to handle. +func TestRenderEventCommitMessage_UnresolvedUsernameStaysEmptyInTemplates(t *testing.T) { + config := ResolveCommitConfig(nil) + config.Message.EventTemplate = "{{.Operation}} by {{.Username}}" + + // The production shape: attachAuthor sets UserInfo only when the outcome is resolved. + event := attributedEvent("", AttributionUnresolved) + message, err := renderEventCommitMessage(event, config) + + require.NoError(t, err) + assert.Equal(t, "CREATE by ", message, + "the sentinel must not leak into user-authored templates") + + // The identity a human actually sees for this commit lives in the git author header, which + // TestCommitOptionsFor_UnresolvedAttributionUsesSentinelAuthor pins. + options := commitOptionsFor(commitWrite(event), ResolveCommitConfig(nil), nil, time.Now()) + require.NotNil(t, options.Author) + assert.Equal(t, UnresolvedAuthorDisplayName, options.Author.Name) +} diff --git a/internal/git/commit.go b/internal/git/commit.go index 9b587ec9..a10fdc27 100644 --- a/internal/git/commit.go +++ b/internal/git/commit.go @@ -186,6 +186,15 @@ func commitOptionsFor( ) *git.CommitOptions { committer := operatorSignature(config, when) author := pendingWrite.AuthorUserInfo() + // An attribution that RAN and did not resolve is authored by the sentinel, not by the + // committer. Authoring it as the committer is what made a lost actor byte-identical to a + // configured-author commit, so the loss was invisible in Git history. + if pendingWrite.AttributionOutcome() == AttributionUnresolved { + author = UnresolvedAuthor() + } + // Reaching here with an empty username now means only "attribution was never attempted" + // (configured-author mode, reconcile/resync writes), where the committer genuinely IS the + // author. if author.Username == "" { return &git.CommitOptions{ Author: committer, diff --git a/internal/git/commit_request_attach.go b/internal/git/commit_request_attach.go index 6ca847d9..d0368d4b 100644 --- a/internal/git/commit_request_attach.go +++ b/internal/git/commit_request_attach.go @@ -43,8 +43,7 @@ type FinalizeResult struct { } // AttachCommitRequest is the "bind this CommitRequest's message to the author's -// open window, then finalize that window after the grace" work item (§6.4 of -// docs/spec/commitrequest-design.md). It rides the same per-worker FIFO +// open window, then finalize that window after the grace" work item. It rides the same per-worker FIFO // event queue as resource events, so by audit-stream ordering it is processed // after every earlier write for that worker. Re-sends are idempotent: the worker // keys pending requests by identity and keeps the first finalize deadline. @@ -56,11 +55,17 @@ type AttachCommitRequest struct { Name string UID string - // Author is the effective user that requested the finalize, attributed from - // the CommitRequest's own create audit event. Only a window whose author + // Author is the effective user that requested the finalize, captured from + // validating admission. Only a window whose author // matches is attached; this binds "the open window" to "the requesting // author's open window". Author string + // Attribution is the outcome of attributing THIS CommitRequest, from the command-authorship + // path. It is matched alongside Author rather than inferred from it, because an empty Author + // alone cannot say whether an actor was sought: it is both "attribution is off" and + // "attribution ran and named nobody". Only the NamesActor half is compared against the + // window's outcome — see matchesWindow for why the enums themselves must not be. + Attribution AttributionOutcome // GitTargetName / GitTargetNamespace scope the finalize to one GitTarget. GitTargetName string GitTargetNamespace string @@ -70,7 +75,7 @@ type AttachCommitRequest struct { Message string // CloseDelaySeconds is the close-delay collect window: the worker closes the // attached window and finalizes it at receipt + CloseDelaySeconds (the delay is - // anchored at attribution, §6.4.4). + // anchored at attach receipt). CloseDelaySeconds int32 } @@ -93,6 +98,7 @@ func (a AttachCommitRequest) id() commitRequestID { type pendingCommitRequest struct { id commitRequestID author string + attribution AttributionOutcome gitTargetName string gitTargetNamespace string message string @@ -103,13 +109,33 @@ type pendingCommitRequest struct { attached bool } -// matchesWindow reports whether the request identifies the given open window by -// author and GitTarget. +// matchesWindow reports whether the request identifies the given open window, by GitTarget and +// by author. +// +// The two attribution outcomes compared here are produced by DIFFERENT, INDEPENDENTLY +// CONFIGURED subsystems: the window's comes from mirrored-resource attribution +// (--author-attribution, audit facts), the request's from command authorship +// (--admission-webhook, its own Redis corner) — see cmd/main.go:311-316. So they are matched on +// AttributionOutcome.NamesActor, not for enum equality. Requiring the enums to be equal silently +// couples the two flags: with attribution off and the webhook on the window says "not attempted" +// while the request says "unresolved", and with attribution on but missing and the webhook off +// it is the other way round. Both are real deployments, both attach correctly today, and exact +// equality would stop both — dropping the user's commit message into a separate default-message +// commit with no error anywhere. +// +// The author comparison stays UNCONDITIONAL, and the outcome class is an additional guard on +// top of it — never a replacement for it. Skipping the author check when neither side names an +// actor looks equivalent (an unnamed actor leaves the author empty on both sides, so the two +// empties compare equal anyway) but is not: it makes cross-author attachment depend on the +// outcome fields being right, so any path that leaves an outcome unset while the author IS set +// would let one author's request finalize another's window. Comparing both costs nothing and +// keeps "bob never claims alice's window" true regardless of what the outcomes say. func (p *pendingCommitRequest) matchesWindow(w *openWindow) bool { if p == nil || w == nil { return false } return p.author == w.Author && + p.attribution.NamesActor() == w.Attribution.NamesActor() && p.gitTargetName == w.GitTarget && p.gitTargetNamespace == w.GitTargetNamespace } diff --git a/internal/git/commit_request_attach_loop.go b/internal/git/commit_request_attach_loop.go index 01dd9b81..d3658eb3 100644 --- a/internal/git/commit_request_attach_loop.go +++ b/internal/git/commit_request_attach_loop.go @@ -6,8 +6,7 @@ import ( "time" ) -// This file holds the worker-loop side of CommitRequest eager attach (§6.4 of -// docs/spec/commitrequest-design.md). All methods run on the single event +// This file holds the worker-loop side of CommitRequest eager attach. All methods run on the single event // loop goroutine, so pendingCRs needs no locking; only the resolved-outcome table // (BranchWorker.crOutcomes) crosses goroutines and is mutex-guarded. // @@ -77,12 +76,13 @@ func (l *branchWorkerEventLoop) handleAttachCommitRequest(req *AttachCommitReque if _, exists := l.pendingCRs[id]; exists { return // idempotent re-send: keep the first finalize deadline. } - // Anchor the grace at receipt (≈ the attribution moment, §6.4.4), not at object + // Anchor the grace at receipt (≈ the attribution moment), not at object // creation: under a delayed ingestion pipeline this lets delaySeconds cover only // the inter-stream spread instead of the absolute latency. l.pendingCRs[id] = &pendingCommitRequest{ id: id, author: req.Author, + attribution: req.Attribution, gitTargetName: req.GitTargetName, gitTargetNamespace: req.GitTargetNamespace, message: req.Message, @@ -110,7 +110,7 @@ func (l *branchWorkerEventLoop) serviceCommitRequests() { // attachWaitingCommitRequests binds the oldest waiting same-author request to the // open window when it is unclaimed. A window carries at most one request; a second -// waits for the next window (§6.4.6). +// waits for the next window. func (l *branchWorkerEventLoop) attachWaitingCommitRequests() { if l.openWindow == nil || l.openWindow.pendingCR != nil { return diff --git a/internal/git/commit_request_attach_test.go b/internal/git/commit_request_attach_test.go index b542995a..f967fe27 100644 --- a/internal/git/commit_request_attach_test.go +++ b/internal/git/commit_request_attach_test.go @@ -487,3 +487,130 @@ func TestAttach_ConflictReplayResolvesToPostReplaySHA(t *testing.T) { require.NoError(t, err) assert.Equal(t, message, commit.Message, "the replayed commit keeps the user's message") } + +// TestAttributionOutcome_ZeroValueIsNotAttempted pins the coupling the whole matching rule rests on: +// the zero value of AttributionOutcome must BE AttributionNotAttempted. +// +// Most producers of an Event never assign Attribution — reconcile, resync, bootstrap, and +// configured-author mode's early return in watch.Manager.attachAuthor all leave it zero, and all +// of them mean "no actor was sought". When this constant was "not_attempted" the zero value was a +// silent fourth state that compared equal to no named outcome, and no test caught it because +// every case set both sides to the zero value and compared "" to "". +func TestAttributionOutcome_ZeroValueIsNotAttempted(t *testing.T) { + var zero AttributionOutcome + assert.Equal(t, AttributionNotAttempted, zero, + "the zero value must be AttributionNotAttempted; three doc comments and every non-live "+ + "event path depend on it") + assert.False(t, zero.NamesActor(), "an unset outcome names no actor") + assert.False(t, AttributionUnresolved.NamesActor(), "attribution ran and named nobody") + assert.True(t, AttributionResolved.NamesActor(), "only a resolved outcome names an actor") +} + +// TestMatchesWindow_AcrossIndependentlyConfiguredSubsystems is the permanent regression test for +// the merge blocker: in the DEFAULT deployment no CommitRequest could attach to any window, so +// the user's commit message was silently dropped and the change landed under the generated +// message with no error anywhere. +// +// The window's outcome and the request's outcome come from two subsystems configured by two +// different flags (--author-attribution vs --admission-webhook, cmd/main.go:311-316), so this +// walks every pair either side can actually produce and names the deployment that produces it. +// TestCommitRequest_OutcomesAgree in internal/controller pins that these are the real +// values the two producers emit; here we pin what the matching rule does with them. +func TestMatchesWindow_AcrossIndependentlyConfiguredSubsystems(t *testing.T) { + const otherAuthor = "bob" + + tests := []struct { + name string + deployment string + window AttributionOutcome + windowAuth string + request AttributionOutcome + reqAuth string + want bool + }{ + { + name: "attribution off, webhook off", + deployment: "the DEFAULT deployment: --author-attribution=false, --admission-webhook=false", + window: AttributionNotAttempted, request: AttributionNotAttempted, want: true, + }, + { + name: "attribution off, webhook on but missed", + deployment: "webhook enabled without Redis, or no admission record for the request", + window: AttributionNotAttempted, request: AttributionUnresolved, want: true, + }, + { + name: "attribution on but unresolved, webhook off", + deployment: "attribution enabled, no audit fact within the grace; webhook off", + window: AttributionUnresolved, request: AttributionNotAttempted, want: true, + }, + { + name: "attribution on but unresolved, webhook on but missed", + deployment: "both enabled, neither named an actor", + window: AttributionUnresolved, request: AttributionUnresolved, want: true, + }, + { + name: "both resolved to the same actor", + deployment: "the fully configured deployment, request and window agree", + window: AttributionResolved, windowAuth: "alice", + request: AttributionResolved, reqAuth: "alice", want: true, + }, + { + name: "both resolved to different actors", + deployment: "two humans editing the same GitTarget; must never cross-attach", + window: AttributionResolved, windowAuth: "alice", + request: AttributionResolved, reqAuth: otherAuthor, want: false, + }, + { + name: "window named an actor, request did not", + deployment: "alice's window must not absorb an unattributable request", + window: AttributionResolved, windowAuth: "alice", + request: AttributionNotAttempted, want: false, + }, + { + name: "request named an actor, window did not", + deployment: "alice's request must not claim an unattributable window", + window: AttributionNotAttempted, + request: AttributionResolved, reqAuth: "alice", want: false, + }, + { + name: "different authors, neither outcome names an actor", + deployment: "not producible today (a set author implies a resolved outcome), but the " + + "author check must not become conditional on the outcome: if it did, any path that " + + "left an outcome unset while the author was set would let bob finalize alice's window", + window: AttributionNotAttempted, windowAuth: "alice", + request: AttributionNotAttempted, reqAuth: otherAuthor, want: false, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + p := &pendingCommitRequest{ + author: tc.reqAuth, + attribution: tc.request, + gitTargetName: crTarget, + gitTargetNamespace: "default", + } + w := &openWindow{ + Author: tc.windowAuth, + Attribution: tc.window, + GitTarget: crTarget, + GitTargetNamespace: "default", + } + assert.Equal(t, tc.want, p.matchesWindow(w), tc.deployment) + }) + } +} + +// TestMatchesWindow_GitTargetAlwaysScopes checks the GitTarget scope survives the outcome +// rework: no attribution pairing may let a request finalize another target's window. +func TestMatchesWindow_GitTargetAlwaysScopes(t *testing.T) { + for _, o := range []AttributionOutcome{AttributionNotAttempted, AttributionUnresolved, AttributionResolved} { + p := &pendingCommitRequest{attribution: o, gitTargetName: crTarget, gitTargetNamespace: "default"} + assert.False(t, p.matchesWindow(&openWindow{ + Attribution: o, GitTarget: "team-b", GitTargetNamespace: "default", + }), "outcome %q must not match across GitTarget names", o) + assert.False(t, p.matchesWindow(&openWindow{ + Attribution: o, GitTarget: crTarget, GitTargetNamespace: "other", + }), "outcome %q must not match across GitTarget namespaces", o) + } +} diff --git a/internal/git/open_window.go b/internal/git/open_window.go index 517cd75a..876fc5db 100644 --- a/internal/git/open_window.go +++ b/internal/git/open_window.go @@ -9,6 +9,14 @@ package git type openWindow struct { // Author is event.UserInfo.Username verbatim. Author string + // Attribution is the window's attribution outcome, taken from its first event. It pairs + // with Author for coalescing: two events share a window only when BOTH the outcome and the + // author agree, so an unresolved event never coalesces with a resolved one that happens + // to carry an empty username. Exact enum equality is right HERE — canAppend compares two + // events from the same watch pipeline, so the outcomes are directly comparable. It is not + // right when comparing against a CommitRequest, which a different subsystem attributes; + // see pendingCommitRequest.matchesWindow. + Attribution AttributionOutcome // GitTarget is the target this window is bound to. Each finalized commit // covers exactly one target so per-target encryption and bootstrap // configuration apply unambiguously. @@ -43,6 +51,7 @@ const groupedCommitOperationKinds = 3 func newOpenWindow(e Event, writer eventContentWriter) *openWindow { return &openWindow{ Author: e.UserInfo.Username, + Attribution: e.Attribution, GitTarget: e.GitTargetName, GitTargetNamespace: e.GitTargetNamespace, pathToEvent: make(map[string]Event), @@ -54,7 +63,8 @@ func (w *openWindow) canAppend(e Event) bool { if w == nil { return false } - return e.UserInfo.Username == w.Author && + return e.Attribution == w.Attribution && + e.UserInfo.Username == w.Author && e.GitTargetName == w.GitTarget && e.GitTargetNamespace == w.GitTargetNamespace } diff --git a/internal/git/pending_writes.go b/internal/git/pending_writes.go index e3a9371c..f607e2bf 100644 --- a/internal/git/pending_writes.go +++ b/internal/git/pending_writes.go @@ -239,8 +239,23 @@ const ( authorKindUser = "user" authorKindServiceAccount = "serviceaccount" authorKindCommitter = "committer" + // authorKindUnresolved is attribution that RAN and did not name an actor. It is its own + // kind on purpose: classifying it as "user" would make a lost actor indistinguishable + // from a named one in dashboards, so a degrading attribution path would read as an + // improving one. See docs/interpreting-metrics.md. + authorKindUnresolved = "unresolved" ) +// AttributionOutcome returns the attribution outcome for commit-shaped pending writes. It +// mirrors AuthorUserInfo: the window is single-author, so the first event's outcome describes +// the whole write. Atomic and empty writes never attempt attribution. +func (p PendingWrite) AttributionOutcome() AttributionOutcome { + if p.Kind == PendingWriteAtomic || len(p.Events) == 0 { + return AttributionNotAttempted + } + return p.Events[0].Attribution +} + func (p PendingWrite) createdCommit() bool { if p.Kind == PendingWriteResync { return p.Committed != nil && *p.Committed @@ -248,7 +263,13 @@ func (p PendingWrite) createdCommit() bool { return !p.CommitSHA.IsZero() } +// authorKind classifies a commit's author for the commits_total metric. It reads the +// attribution OUTCOME first: an unresolved attribution is its own kind, never "user", so a +// lost actor can never be counted as a named one. func (p PendingWrite) authorKind() string { + if p.AttributionOutcome() == AttributionUnresolved { + return authorKindUnresolved + } author := p.Author() if author == "" { return authorKindCommitter diff --git a/internal/git/resync_flush.go b/internal/git/resync_flush.go index f546edf3..aaab9988 100644 --- a/internal/git/resync_flush.go +++ b/internal/git/resync_flush.go @@ -128,8 +128,7 @@ func (l *branchWorkerEventLoop) applyResync(req *ResyncRequest) { // commit is now in pendingWrites and must reach the remote — or the resync itself // committed. Any finalize that closes a window must schedule its push, or the // window's commit is stranded: committed locally but never pushed (the - // stranded-write fix, docs/spec/commitrequest-design.md §6.4.2; the - // failure analyzed in github-e2e-per-type-tail-failure-investigation.md). + // stranded-write fix in the CommitRequest window contract). // maybeSchedulePush no-ops when nothing is pending, so a pure no-op resync that // closed no window stays a no-op and does not disturb the push cooldown. if committed || closedWindow { diff --git a/internal/git/resync_push_test.go b/internal/git/resync_push_test.go index 74df2c5c..ad3460cd 100644 --- a/internal/git/resync_push_test.go +++ b/internal/git/resync_push_test.go @@ -38,7 +38,7 @@ func TestEnqueueResync_ReportsEnqueueOutcome(t *testing.T) { } // TestHandleResyncRequest_ClosedWindowIsPushedEvenWhenNoOpResync pins the -// stranded-write fix (docs/spec/commitrequest-design.md §6.4.2, §9.5): a +// stranded-write fix in the CommitRequest window contract: a // resync that closes a live commit window but commits nothing of its own must // still schedule the window's commit for push. Before the fix maybeSchedulePush // ran only when the resync itself committed, so a window the resync closed was diff --git a/internal/git/types.go b/internal/git/types.go index 277e665c..6135b030 100644 --- a/internal/git/types.go +++ b/internal/git/types.go @@ -17,6 +17,90 @@ import ( "github.com/ConfigButler/gitops-reverser/internal/types" ) +// AttributionOutcome records what happened when the operator tried to name the actor behind a +// change. It is carried EXPLICITLY rather than inferred from the author identity, because the +// author string is load-bearing in several places (window grouping, commit-message templates, +// the author_kind metric) and overloading it to also mean "attribution failed" made a silent +// failure indistinguishable from correct configured-author behaviour. See +// docs/architecture.md#author-and-committer-identity-in-git. +type AttributionOutcome string + +const ( + // AttributionNotAttempted is configured-author mode: attribution is switched off, so the + // committer legitimately IS the author and no actor was ever sought. + // + // It is deliberately the EMPTY string, so that it is also the ZERO VALUE of the type. Most + // paths that build an Event never assign Attribution at all — reconcile, resync, bootstrap, + // and configured-author mode's early return in the watch pipeline — and every one of them + // means exactly "no actor was sought". Any other string would make the zero value a silent + // fourth state equal to none of the three named outcomes, which is precisely the bug that + // stopped every CommitRequest attaching in the default deployment. Nothing serializes this + // value (it reaches no CRD field and no metric label; authorKind branches on the typed + // value), so the empty string costs nothing. TestAttributionZeroValueIsNotAttempted pins it. + AttributionNotAttempted AttributionOutcome = "" + // AttributionResolved means an audit fact named the actor. + AttributionResolved AttributionOutcome = "resolved" + // AttributionUnresolved means attribution ran and did not arrive at an actor. + // + // Deliberately "unresolved", not "failed": the lookup collapses several genuinely + // different situations into one miss — no fact was ever produced (correct; not every + // change has an audited human actor), a cancelled wait, a Redis read error, and a + // malformed value all return the same not-found. Calling that a failure would assert a + // fault the operator cannot prove. + AttributionUnresolved AttributionOutcome = "unresolved" +) + +// NamesActor reports whether the outcome carries an actor to compare against. +// +// This is the ONLY distinction that survives across subsystem boundaries. Whether an outcome +// is "not attempted" or "unresolved" is a property of how the subsystem that produced it is +// configured — and the mirrored-resource attribution path (--author-attribution) and the +// command-authorship path (--admission-webhook) are configured independently of each other +// (cmd/main.go:311-316). Two independently configured producers can therefore disagree about +// the enum while agreeing perfectly about the thing that matters: whether there is an actor. +// Compare the enums across that boundary and you couple the two flags; compare NamesActor and +// you do not. Within a single subsystem the enum itself is meaningful and IS compared directly +// (openWindow.canAppend), because both sides come from the same producer. +func (o AttributionOutcome) NamesActor() bool { + return o == AttributionResolved +} + +// UnresolvedAuthor is the identity written to the Git AUTHOR HEADER when attribution ran and +// did not resolve an actor. It exists so an unresolved attribution is visible in `git log` +// instead of being indistinguishable from a configured-author commit. +// +// Scope: the git author header, and nothing else. It is DERIVED at the write path +// (commitOptionsFor) from the carried AttributionOutcome — it is never stamped onto an Event. +// The outcome is the fact; this identity is one rendering of it. So the sentinel deliberately +// does NOT reach window grouping, the grouped commit-message body, or user-authored +// {{.Username}} templates: those keep the empty author they have always had for an unnamed +// actor, on both this path and in configured-author mode. Pushing a magic token into message +// bodies would change the commit text of every existing deployment that has attribution misses, +// and force user templates to special-case a value they never had to handle. +// +// Three fields, three different strings, because the header needs all three: +// - Username is the stable machine token, so tooling that parses the header has something +// greppable that will not drift with the human-facing wording. +// - DisplayName is what a human reads in `git log`, so it leads with what they care about. +// - Email uses the RFC 2606 reserved .invalid TLD, so it can never collide with a real +// address and never routes mail. +func UnresolvedAuthor() UserInfo { + return UserInfo{ + Username: UnresolvedAuthorUsername, + DisplayName: UnresolvedAuthorDisplayName, + Email: UnresolvedAuthorEmail, + } +} + +const ( + // UnresolvedAuthorUsername is the stable machine token for an unresolved attribution. + UnresolvedAuthorUsername = "attribution-unresolved" + // UnresolvedAuthorDisplayName is the human-facing git author name. + UnresolvedAuthorDisplayName = "unknown (attribution unresolved)" + // UnresolvedAuthorEmail is a reserved-invalid address (RFC 2606). + UnresolvedAuthorEmail = "attribution-unresolved@gitops-reverser.invalid" +) + const ( // DefaultCommitterName matches the default operator identity in Git history. DefaultCommitterName = "GitOps Reverser" @@ -201,8 +285,7 @@ type PendingWrite struct { Committed *bool // CommitRequest, when set, is the CommitRequest claiming this write: it is - // resolved Committed (with CommitSHA) once this write is pushed (§6.5 of - // docs/spec/commitrequest-design.md). It rides the write through the + // resolved Committed (with CommitSHA) once this write is pushed. It rides the write through the // push cooldown and the conflict rebase-replay, so the result follows the data. CommitRequest *commitRequestID // CommitSHA is the hash of the commit this write created, captured in @@ -378,6 +461,16 @@ type Event struct { // UserInfo contains user information for commit messages. UserInfo UserInfo + // Attribution records whether naming the actor was attempted and whether it succeeded. + // It is the authority for author rendering, the author_kind metric, and CommitRequest + // window matching — none of which may infer the outcome from UserInfo, because an empty + // or sentinel username cannot distinguish "attribution is off" from "attribution ran and + // found nothing". The zero value is AttributionNotAttempted — the constant is defined as the + // empty string precisely so that it is — which is correct for every non-live path + // (reconcile, resync, bootstrap) where no actor is ever sought, and for configured-author + // mode. attachAuthor is the only assignment to this field outside tests. + Attribution AttributionOutcome + // Path is the POSIX-like relative path prefix for this event's files. // This comes from the GitTarget that triggered this event. // Empty string means write to repository root. diff --git a/internal/giteaclient/bootstrap.go b/internal/giteaclient/bootstrap.go index 1a8182eb..9a577dec 100644 --- a/internal/giteaclient/bootstrap.go +++ b/internal/giteaclient/bootstrap.go @@ -122,18 +122,20 @@ func (c *Client) EnsureOrgRepo( } } -// ListRepoHooks returns every webhook configured on owner/repo. +// hookPageSize / maxHookPages bound the webhook listing. /repos/{owner}/{repo}/hooks paginates +// exactly like the keys endpoint — it is clamped to MAX_RESPONSE_ITEMS and has no "return +// everything" mode — so an unpaginated read goes blind past the first page. Latent rather than +// live today only because e2e creates a fresh repo per spec and never accumulates a page of +// hooks on one repo; the callers that delete hooks would silently miss the rest. +const ( + hookPageSize = 50 + maxHookPages = 200 +) + +// ListRepoHooks returns every webhook configured on owner/repo, following pagination. func (c *Client) ListRepoHooks(ctx context.Context, owner, repo string) ([]RepoHook, error) { - path := "/repos/" + PathEscape(owner) + "/" + PathEscape(repo) + "/hooks" - var hooks []RepoHook - code, raw, err := c.Do(ctx, http.MethodGet, path, nil, &hooks) - if err != nil { - return nil, err - } - if code != http.StatusOK { - return nil, unexpectedStatus(http.MethodGet, path, code, raw) - } - return hooks, nil + base := "/repos/" + PathEscape(owner) + "/" + PathEscape(repo) + "/hooks" + return listPaginated[RepoHook](ctx, c, base, hookPageSize, maxHookPages) } // DeleteRepoHook removes a single repository webhook. Missing hooks are treated as success. diff --git a/internal/giteaclient/bootstrap_test.go b/internal/giteaclient/bootstrap_test.go index 839a3ae9..d0a28360 100644 --- a/internal/giteaclient/bootstrap_test.go +++ b/internal/giteaclient/bootstrap_test.go @@ -150,12 +150,21 @@ func webhookTestHandler(t *testing.T) http.HandlerFunc { w.WriteHeader(http.StatusNoContent) case r.Method == http.MethodGet && r.URL.Path == "/repos/testorg/demo/hooks": w.Header().Set("Content-Type", "application/json") + // The hooks listing paginates; everything past page 1 is empty, and that empty page + // is what terminates ListRepoHooks. + if r.URL.Query().Get("page") != "1" { + _, _ = w.Write([]byte(`[]`)) + return + } _, _ = w.Write([]byte( `[{"id":42,"type":"gitea","active":true,` + `"config":{"url":"http://receiver.example/hook/demo","content_type":"json"}}]`, )) default: - t.Fatalf("unexpected request: %s %s", r.Method, r.URL.Path) + // t.Errorf, not t.Fatalf: Fatal from a handler goroutine Goexits it mid-response and + // the client reports a spurious EOF on top of the real failure. + t.Errorf("unexpected request: %s %s", r.Method, r.URL.Path) + w.WriteHeader(http.StatusBadRequest) } } } diff --git a/internal/giteaclient/keys.go b/internal/giteaclient/keys.go index 08e0b41b..e007acc3 100644 --- a/internal/giteaclient/keys.go +++ b/internal/giteaclient/keys.go @@ -24,18 +24,31 @@ func NormalizeAuthorizedKey(k string) string { return fields[0] + " " + fields[1] } -// ListUserKeys returns every public key registered for the named user. +// keyPageSize is the per-page item count REQUESTED when listing keys. It is a throughput hint +// only — never a termination signal. Gitea clamps every request down to MAX_RESPONSE_ITEMS +// (ToCorrectPageSize), which is operator-configurable, so the server's actual page size may be +// smaller than this and the loop must not assume otherwise. +const keyPageSize = 50 + +// maxKeyPages bounds the pagination loop. Termination depends on the server eventually +// returning an empty page, which a server that ignores the `page` parameter never does — it +// would serve a full first page forever, and the loop would spin until the caller's context +// deadline and report an undiagnosable "context deadline exceeded" instead of naming the cause. +// At keyPageSize this ceiling is 10 000 keys, far past any real user. +const maxKeyPages = 200 + +// ListUserKeys returns EVERY public key on the named user, following pagination. +// +// The pagination is load-bearing, not tidiness. Gitea's default page size is 30, and this +// listing backs FindUserKeyByTitle, which EnsureUserKeyAsAdmin uses to decide whether a title +// is already taken. An unpaginated listing therefore stops seeing keys once a user holds more +// than one page of them, and "ensure" starts failing with `422 Key title has been used` for a +// key that demonstrably exists — the caller cannot see it to replace it. The e2e suite hits +// this reliably: it registers ~24 seeded transport keys per full run against a long-lived +// Gitea, so the second run onward exceeds one page and the stable-titled `playground` key +// becomes unreachable. func (c *Client) ListUserKeys(ctx context.Context, login string) ([]PublicKey, error) { - var keys []PublicKey - path := "/users/" + PathEscape(login) + "/keys" - code, raw, err := c.Do(ctx, http.MethodGet, path, nil, &keys) - if err != nil { - return nil, err - } - if code != http.StatusOK { - return nil, unexpectedStatus(http.MethodGet, path, code, raw) - } - return keys, nil + return listPaginated[PublicKey](ctx, c, "/users/"+PathEscape(login)+"/keys", keyPageSize, maxKeyPages) } // FindUserKey looks up a public key on the named user by key material, diff --git a/internal/giteaclient/keys_test.go b/internal/giteaclient/keys_test.go index 0f154502..e482dc31 100644 --- a/internal/giteaclient/keys_test.go +++ b/internal/giteaclient/keys_test.go @@ -4,8 +4,12 @@ package giteaclient import ( "context" + "fmt" "net/http" "net/http/httptest" + "strconv" + "strings" + "sync" "testing" ) @@ -47,30 +51,41 @@ func newEnsureUserKeyAsAdminTestHandler( ) http.Handler { t.Helper() + // These handlers report with t.Errorf, never t.Fatalf: Fatal from a non-test goroutine runs + // runtime.Goexit on the handler, killing the connection mid-response, so the client reports a + // spurious EOF on top of the real failure. mux := http.NewServeMux() mux.HandleFunc("/users/giteaadmin/keys", func(w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodGet { - t.Fatalf("unexpected request: %s %s", r.Method, r.URL.Path) + t.Errorf("unexpected request: %s %s", r.Method, r.URL.Path) + w.WriteHeader(http.StatusBadRequest) + return } w.Header().Set("Content-Type", "application/json") - if !*deleteCalled { - _, _ = w.Write([]byte( - `[{"id":7,"title":"E2E Transport Key playground","key":"ssh-rsa old-key","fingerprint":"old"}]`, - )) + // Honour the page parameter: everything past the first page is empty, which is what + // terminates ListUserKeys. + if r.URL.Query().Get("page") != "1" || *deleteCalled { + _, _ = w.Write([]byte(`[]`)) return } - _, _ = w.Write([]byte(`[]`)) + _, _ = w.Write([]byte( + `[{"id":7,"title":"E2E Transport Key playground","key":"ssh-rsa old-key","fingerprint":"old"}]`, + )) }) mux.HandleFunc("/admin/users/giteaadmin/keys/7", func(w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodDelete { - t.Fatalf("unexpected request: %s %s", r.Method, r.URL.Path) + t.Errorf("unexpected request: %s %s", r.Method, r.URL.Path) + w.WriteHeader(http.StatusBadRequest) + return } *deleteCalled = true w.WriteHeader(http.StatusNoContent) }) mux.HandleFunc("/admin/users/giteaadmin/keys", func(w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodPost { - t.Fatalf("unexpected request: %s %s", r.Method, r.URL.Path) + t.Errorf("unexpected request: %s %s", r.Method, r.URL.Path) + w.WriteHeader(http.StatusBadRequest) + return } *createCalled = true w.Header().Set("Content-Type", "application/json") @@ -87,9 +102,15 @@ func TestClientFindUserKeyByTitle_ReturnsMatch(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodGet || r.URL.Path != "/users/giteaadmin/keys" { - t.Fatalf("unexpected request: %s %s", r.Method, r.URL.Path) + t.Errorf("unexpected request: %s %s", r.Method, r.URL.Path) // never Fatal off the test goroutine + w.WriteHeader(http.StatusBadRequest) + return } w.Header().Set("Content-Type", "application/json") + if r.URL.Query().Get("page") != "1" { + _, _ = w.Write([]byte(`[]`)) // the empty page that terminates the listing + return + } _, _ = w.Write([]byte( `[` + `{"id":1,"title":"first","key":"ssh-ed25519 AAAA first","fingerprint":"fp1"},` + @@ -111,3 +132,264 @@ func TestClientFindUserKeyByTitle_ReturnsMatch(t *testing.T) { t.Fatalf("FindUserKeyByTitle() = %#v, want key id 2", key) } } + +// keysPageJSON renders one page of the fake keys listing: ids [start, end) of `total`, where +// the LAST key of the whole set carries lastTitle so a title-search must reach the final page. +func keysPageJSON(start, end, total int, lastTitle string) string { + var b strings.Builder + b.WriteString("[") + for i := start; i < end; i++ { + if i > start { + b.WriteString(",") + } + title := fmt.Sprintf("key-%d", i) + if i == total-1 { + title = lastTitle + } + fmt.Fprintf(&b, `{"id":%d,"title":%q,"key":"ssh-ed25519 AAAA k%d","fingerprint":"fp%d"}`, + i+1, title, i, i) + } + b.WriteString("]") + return b.String() +} + +// keysServer is a fake /users/{login}/keys endpoint that records what the client asked for and +// what it got wrong. +// +// Handler goroutine failures are RECORDED, not raised: calling t.Fatal from an httptest handler +// runs runtime.Goexit on the handler goroutine, which kills the connection mid-response and +// surfaces a spurious `EOF` from the client on top of the real message — two reported errors for +// one fault, with the misleading one first. Assertions run on the test goroutine via check(). +type keysServer struct { + mu sync.Mutex + // serverPageSize is what the server actually returns per page, regardless of the limit the + // client requests. Gitea's ToCorrectPageSize clamps to MAX_RESPONSE_ITEMS, which operators + // can set below our requested keyPageSize. + serverPageSize int + total int + lastTitle string + // ignorePageParam models a server that serves page 1 forever. + ignorePageParam bool + pagesServed []int + problems []string +} + +func (s *keysServer) failf(format string, args ...any) { + s.mu.Lock() + defer s.mu.Unlock() + s.problems = append(s.problems, fmt.Sprintf(format, args...)) +} + +// check surfaces recorded handler problems on the test goroutine, where t.Fatal is safe. +func (s *keysServer) check(t *testing.T) { + t.Helper() + s.mu.Lock() + defer s.mu.Unlock() + if len(s.problems) > 0 { + t.Fatalf("fake keys server rejected the client's requests:\n %s", strings.Join(s.problems, "\n ")) + } +} + +func (s *keysServer) requests() []int { + s.mu.Lock() + defer s.mu.Unlock() + return append([]int(nil), s.pagesServed...) +} + +// requireKeysPageRequest validates the request is an explicitly paginated keys read and returns +// the requested page. An unpaginated read is the bug this file exists to prevent. +func (s *keysServer) requireKeysPageRequest(r *http.Request) (int, bool) { + if r.Method != http.MethodGet || r.URL.Path != "/users/giteaadmin/keys" { + s.failf("unexpected request: %s %s", r.Method, r.URL.Path) + return 0, false + } + query := r.URL.Query() + page := query.Get("page") + if page == "" { + s.failf("ListUserKeys must request an explicit page; an unpaginated read silently truncates at 30") + return 0, false + } + if got := query.Get("limit"); got != strconv.Itoa(keyPageSize) { + s.failf("limit = %q, want %d", got, keyPageSize) + return 0, false + } + n, err := strconv.Atoi(page) + if err != nil { + s.failf("bad page parameter %q: %v", page, err) + return 0, false + } + return n, true +} + +func (s *keysServer) handler() http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + page, ok := s.requireKeysPageRequest(r) + if !ok { + w.WriteHeader(http.StatusBadRequest) + return + } + s.mu.Lock() + s.pagesServed = append(s.pagesServed, page) + tooMany := len(s.pagesServed) > maxKeyPages+1 + s.mu.Unlock() + if tooMany { + s.failf("client exceeded the maxKeyPages cap") + w.WriteHeader(http.StatusInternalServerError) + return + } + + if s.ignorePageParam { + page = 1 + } + start := (page - 1) * s.serverPageSize + w.Header().Set("Content-Type", "application/json") + if start >= s.total { + _, _ = w.Write([]byte(`[]`)) + return + } + _, _ = w.Write([]byte(keysPageJSON(start, min(start+s.serverPageSize, s.total), s.total, s.lastTitle))) + }) +} + +// Gitea paginates key listings: DEFAULT_PAGING_NUM is 30 and MAX_RESPONSE_ITEMS is 50, and +// `ToCorrectPageSize` clamps a limit of 0 or less straight back to the default — so there is +// no server-side "return everything" mode for this endpoint. An unpaginated listing therefore +// goes blind past the first page, which silently breaks FindUserKeyByTitle and makes +// EnsureUserKeyAsAdmin fail with `422 Key title has been used` for a key that exists but +// cannot be seen (and so cannot be replaced). +// +// The fake below serves a key set larger than one page, with the sought title on the LAST +// page. Without the page loop the client only ever sees page 1 and the title is reported +// missing. +func TestClientListUserKeys_FollowsPaginationBeyondTheFirstPage(t *testing.T) { + t.Parallel() + + const ( + total = 124 // > 2 full pages, so a single-page read cannot pass by luck + lastTitle = "E2E Transport Key playground" + ) + fake := &keysServer{serverPageSize: keyPageSize, total: total, lastTitle: lastTitle} + server := httptest.NewServer(fake.handler()) + defer server.Close() + + client := New(server.URL, "admin", "password") + + keys, err := client.ListUserKeys(context.Background(), "giteaadmin") + fake.check(t) + if err != nil { + t.Fatalf("ListUserKeys() error = %v", err) + } + if len(keys) != total { + t.Fatalf("ListUserKeys() returned %d keys, want all %d — pagination was not followed", len(keys), total) + } + // Four requests, not three: termination is an EMPTY page, so the 24-key page 3 does not end + // the loop. That extra round trip is the price of being correct for any server page size. + if got := fake.requests(); len(got) != 4 { + t.Fatalf("pages requested = %v, want 4 (three pages of keys, then the empty page that ends it)", got) + } + + // The point of the loop: a title on a later page must still be findable, because that is + // what EnsureUserKeyAsAdmin needs in order to replace a stale key instead of colliding. + found, ok, err := client.FindUserKeyByTitle(context.Background(), "giteaadmin", lastTitle) + fake.check(t) + if err != nil { + t.Fatalf("FindUserKeyByTitle() error = %v", err) + } + if !ok { + t.Fatal("FindUserKeyByTitle() did not find a key that exists on a later page") + } + if found.ID != total { + t.Fatalf("FindUserKeyByTitle() = id %d, want %d", found.ID, total) + } +} + +// A server whose page size is SMALLER than the limit we request must still be paginated +// correctly. MAX_RESPONSE_ITEMS is operator-configurable and ToCorrectPageSize clamps every +// request down to it, so against such a server EVERY page is short — and a loop that stops on a +// short page stops after page 1, byte-for-byte re-creating the original bug (FindUserKeyByTitle +// missing an existing key, then `422 Key title has been used`). The e2e lab pins no [api] block +// in gitea-values.yaml, so today's pass depends only on an unpinned server default. +func TestClientListUserKeys_ServerPageSizeSmallerThanRequested(t *testing.T) { + t.Parallel() + + const ( + serverPageSize = 30 // MAX_RESPONSE_ITEMS set below our requested keyPageSize + total = 100 + lastTitle = "E2E Transport Key playground" + ) + fake := &keysServer{serverPageSize: serverPageSize, total: total, lastTitle: lastTitle} + server := httptest.NewServer(fake.handler()) + defer server.Close() + + client := New(server.URL, "admin", "password") + + keys, err := client.ListUserKeys(context.Background(), "giteaadmin") + fake.check(t) + if err != nil { + t.Fatalf("ListUserKeys() error = %v", err) + } + if len(keys) != total { + t.Fatalf("ListUserKeys() returned %d keys, want all %d — the loop stopped on a short page, "+ + "which is not the last page when the server clamps below the requested limit", len(keys), total) + } + + _, ok, err := client.FindUserKeyByTitle(context.Background(), "giteaadmin", lastTitle) + fake.check(t) + if err != nil { + t.Fatalf("FindUserKeyByTitle() error = %v", err) + } + if !ok { + t.Fatal("FindUserKeyByTitle() did not find an existing key: this is the `422 Key title has " + + "been used` failure the pagination exists to prevent") + } +} + +// A listing that exactly fills a page must still terminate: the client asks for one more page, +// gets an empty body, and stops. +func TestClientListUserKeys_ExactPageMultipleTerminates(t *testing.T) { + t.Parallel() + + fake := &keysServer{serverPageSize: keyPageSize, total: keyPageSize, lastTitle: "unused"} + server := httptest.NewServer(fake.handler()) + defer server.Close() + + keys, err := New(server.URL, "admin", "password").ListUserKeys(context.Background(), "giteaadmin") + fake.check(t) + if err != nil { + t.Fatalf("ListUserKeys() error = %v", err) + } + if len(keys) != keyPageSize { + t.Fatalf("ListUserKeys() returned %d keys, want %d", len(keys), keyPageSize) + } + if got := fake.requests(); len(got) != 2 { + t.Fatalf("requests = %v, want 2 (the full page, then the empty page that ends the loop)", got) + } +} + +// A server that ignores the `page` parameter serves a full first page forever. Without a cap the +// loop spins until the caller's context deadline and reports `context deadline exceeded`, which +// names neither the endpoint nor the cause. The cap turns it into a diagnosable error, bounded. +func TestClientListUserKeys_CapsPagesAgainstANonPaginatingServer(t *testing.T) { + t.Parallel() + + fake := &keysServer{ + serverPageSize: keyPageSize, + total: 1_000_000, + lastTitle: "unused", + ignorePageParam: true, + } + server := httptest.NewServer(fake.handler()) + defer server.Close() + + _, err := New(server.URL, "admin", "password").ListUserKeys(context.Background(), "giteaadmin") + fake.check(t) + if err == nil { + t.Fatal("ListUserKeys() returned no error against a server that never paginates") + } + if !strings.Contains(err.Error(), "ignore the page parameter") { + t.Fatalf("ListUserKeys() error = %v, want it to name the non-paginating server", err) + } + if got := len(fake.requests()); got != maxKeyPages { + t.Fatalf("requests = %d, want exactly the %d-page cap", got, maxKeyPages) + } +} diff --git a/internal/giteaclient/pagination.go b/internal/giteaclient/pagination.go new file mode 100644 index 00000000..9d99962e --- /dev/null +++ b/internal/giteaclient/pagination.go @@ -0,0 +1,50 @@ +// SPDX-License-Identifier: Apache-2.0 + +package giteaclient + +import ( + "context" + "fmt" + "net/http" +) + +// listPaginated walks every page of a Gitea list endpoint and returns the concatenated items. +// +// Two rules, both of which a hand-rolled loop tends to get wrong: +// +// - ONLY an empty page terminates. A short page is NOT necessarily the last page: Gitea's +// ToCorrectPageSize clamps the requested limit down to MAX_RESPONSE_ITEMS, which operators +// configure, so against a server whose cap is below pageSize EVERY page is short. Stopping +// on a short page would stop after page 1 and silently return a truncated list — the exact +// failure mode pagination is here to prevent. The cost is one extra round trip. +// - The loop is capped. A server that ignores the `page` parameter serves a full first page +// forever; without a cap this spins until the caller's context deadline and reports +// `context deadline exceeded`, naming neither the endpoint nor the cause. +// +// page is 1-indexed (Gitea routers/api/v1/utils/page.go, models/db/list.go). +func listPaginated[T any]( + ctx context.Context, + c *Client, + base string, + pageSize, maxPages int, +) ([]T, error) { + var all []T + for page := 1; page <= maxPages; page++ { + var items []T + path := fmt.Sprintf("%s?page=%d&limit=%d", base, page, pageSize) + code, raw, err := c.Do(ctx, http.MethodGet, path, nil, &items) + if err != nil { + return nil, err + } + if code != http.StatusOK { + return nil, unexpectedStatus(http.MethodGet, path, code, raw) + } + if len(items) == 0 { + return all, nil + } + all = append(all, items...) + } + return nil, fmt.Errorf( + "listing %s did not terminate within %d pages of %d: the server appears to ignore the "+ + "page parameter", base, maxPages, pageSize) +} diff --git a/internal/giteaclient/pagination_test.go b/internal/giteaclient/pagination_test.go new file mode 100644 index 00000000..dab4980d --- /dev/null +++ b/internal/giteaclient/pagination_test.go @@ -0,0 +1,58 @@ +// SPDX-License-Identifier: Apache-2.0 + +package giteaclient + +import ( + "context" + "errors" + "io" + "net/http" + "strings" + "testing" +) + +type roundTripperFunc func(*http.Request) (*http.Response, error) + +func (f roundTripperFunc) RoundTrip(request *http.Request) (*http.Response, error) { + return f(request) +} + +// TestListPaginated_PropagatesRequestFailures keeps transport and non-200 errors observable to callers. +func TestListPaginated_PropagatesRequestFailures(t *testing.T) { + tests := []struct { + name string + transport http.RoundTripper + want string + }{ + { + name: "transport error", + transport: roundTripperFunc(func(*http.Request) (*http.Response, error) { + return nil, errors.New("connection refused") + }), + want: "connection refused", + }, + { + name: "unexpected status", + transport: roundTripperFunc(func(*http.Request) (*http.Response, error) { + return &http.Response{ + StatusCode: http.StatusTooManyRequests, + Body: io.NopCloser(strings.NewReader("rate limited")), + Header: make(http.Header), + }, nil + }), + want: "HTTP 429: rate limited", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + client := New("https://gitea.example.test/api/v1", "", "") + client.HTTPClient = &http.Client{Transport: tc.transport} + + _, err := listPaginated[string](context.Background(), client, "/items", 50, 1) + if err == nil || !strings.Contains(err.Error(), tc.want) { + t.Fatalf("listPaginated() error = %v, want substring %q", err, tc.want) + } + }) + } +} diff --git a/internal/queue/attribution_index_deletecollection_test.go b/internal/queue/attribution_index_deletecollection_test.go index f6c73d6b..b7f792f0 100644 --- a/internal/queue/attribution_index_deletecollection_test.go +++ b/internal/queue/attribution_index_deletecollection_test.go @@ -123,7 +123,8 @@ func TestRecordDeleteCollectionFacts_HollowBodyWritesNothing(t *testing.T) { // TestRecordDeleteCollectionFacts_PartialListOnlyWritesPresent proves a partial body // (a large collection delete that returned only some items) attributes what it has and -// leaves the rest to the committer fallback (the watch + sweep backstop state). +// leaves the rest without a usable fact; live attributed writes use the explicit +// unresolved author for those events. func TestRecordDeleteCollectionFacts_PartialListOnlyWritesPresent(t *testing.T) { idx := newTestAttributionIndex(t) ctx := context.Background() diff --git a/internal/watch/author_resolver.go b/internal/watch/author_resolver.go index 1f8c000f..25d8533b 100644 --- a/internal/watch/author_resolver.go +++ b/internal/watch/author_resolver.go @@ -66,11 +66,22 @@ type CursorStore interface { // AuthorResolver names the commit author for a live watch event from audit facts. type AuthorResolver interface { - // ResolveAuthor returns the author UserInfo for a watch event, or ok=false to - // commit as the configured committer. It may wait up to the grace window for a - // matching fact; it never blocks indefinitely and never returns an error path — - // an absent fact is a committer commit, not a failure. exactCapable distinguishes + // ResolveAuthor returns the author UserInfo for a watch event together with the + // attribution OUTCOME. It may wait up to the grace window for a matching fact; it never + // blocks indefinitely and never returns an error path. exactCapable distinguishes // ADDED/MODIFIED events (true) from known RV-mismatch removals (false). + // + // The outcome is returned explicitly rather than as an ok bool because the two possible + // "no author" cases are NOT the same and callers must be able to tell them apart: + // AttributionNotAttempted (configured-author mode — the committer legitimately is the + // author) versus AttributionUnresolved (attribution ran and found nothing — a gap worth + // surfacing). An empty UserInfo cannot distinguish them, which is exactly how the loss + // stayed invisible. A resolved outcome always carries a non-empty UserInfo. + // + // In production this method only ever returns the latter two: configured-author mode is + // expressed by leaving Manager.AuthorResolver nil (attachAuthor returns early, leaving the + // event's zero AttributionNotAttempted), never by constructing a resolver over a nil + // lookup. cmd/main.go:258 only builds one with a non-nil index. ResolveAuthor( ctx context.Context, providerName string, @@ -78,7 +89,7 @@ type AuthorResolver interface { uid k8stypes.UID, rv string, exactCapable bool, - ) (git.UserInfo, bool) + ) (git.UserInfo, git.AttributionOutcome) } type attributionResolver struct { @@ -106,47 +117,52 @@ func (r *attributionResolver) ResolveAuthor( uid k8stypes.UID, rv string, exactCapable bool, -) (git.UserInfo, bool) { +) (git.UserInfo, git.AttributionOutcome) { start := time.Now() + // A nil lookup is configured-author mode: attribution was never switched on, so nothing + // was attempted and the committer legitimately authors the commit. Defensive only — + // production expresses that mode with a nil Manager.AuthorResolver, so this branch is + // unreachable there (cmd/main.go:258 always passes a non-nil index). if r.lookup == nil { recordAttributionResolution(ctx, gvr, queue.AttributionAbsent, time.Since(start)) - return git.UserInfo{}, false + return git.UserInfo{}, git.AttributionNotAttempted } deadline := time.Now().Add(r.grace) for { resolution := r.lookup.LookupAuthorResolution(ctx, providerName, gvr, uid, rv, exactCapable) if resolution.Result != queue.AttributionAbsent { - ui, ok, result := r.userInfoForResolution(resolution) + ui, outcome, result := r.userInfoForResolution(resolution) recordAttributionResolution(ctx, gvr, result, time.Since(start)) - return ui, ok + return ui, outcome } if !time.Now().Before(deadline) { recordAttributionResolution(ctx, gvr, queue.AttributionAbsent, time.Since(start)) - return git.UserInfo{}, false + return git.UserInfo{}, git.AttributionUnresolved } if !sleepOrDone(ctx, attributionPollInterval) { recordAttributionResolution(ctx, gvr, queue.AttributionAbsent, time.Since(start)) - return git.UserInfo{}, false + return git.UserInfo{}, git.AttributionUnresolved } } } // userInfoForResolution turns a matched fact into a commit author. The matched // actor — human or service account — is always named by its own username; a fact -// with no author falls back to the committer (ok=false). +// that carries no author is UNRESOLVED, not not-attempted: attribution ran, found a +// fact, and still could not name anyone. func (r *attributionResolver) userInfoForResolution( resolution queue.AuthorResolution, -) (git.UserInfo, bool, queue.AttributionResult) { +) (git.UserInfo, git.AttributionOutcome, queue.AttributionResult) { fact := resolution.Fact result := resolution.Result if fact.Author == "" { - return git.UserInfo{}, false, result + return git.UserInfo{}, git.AttributionUnresolved, result } return git.UserInfo{ Username: fact.Author, DisplayName: fact.DisplayName, Email: fact.Email, - }, true, result + }, git.AttributionResolved, result } func recordAttributionResolution( diff --git a/internal/watch/author_resolver_test.go b/internal/watch/author_resolver_test.go index 33e1f780..bf2cb732 100644 --- a/internal/watch/author_resolver_test.go +++ b/internal/watch/author_resolver_test.go @@ -13,6 +13,7 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" k8stypes "k8s.io/apimachinery/pkg/types" + "github.com/ConfigButler/gitops-reverser/internal/git" "github.com/ConfigButler/gitops-reverser/internal/queue" "github.com/ConfigButler/gitops-reverser/internal/telemetry" ) @@ -51,8 +52,8 @@ func TestAuthorResolver_HumanHit(t *testing.T) { } r := NewAuthorResolver(lookup, DefaultAttributionGraceWindow, logr.Discard()) - ui, ok := r.ResolveAuthor(context.Background(), "prod-eu-1", resolverGVR, "uid-1", "101", true) - require.True(t, ok) + ui, outcome := r.ResolveAuthor(context.Background(), "prod-eu-1", resolverGVR, "uid-1", "101", true) + require.Equal(t, git.AttributionResolved, outcome) assert.Equal(t, "alice", ui.Username) assert.Equal(t, "a@x.io", ui.Email) assert.Equal(t, 1, lookup.calls) @@ -75,8 +76,9 @@ func TestAuthorResolver_ServiceAccountIsNamed(t *testing.T) { } r := NewAuthorResolver(lookup, DefaultAttributionGraceWindow, logr.Discard()) - ui, ok := r.ResolveAuthor(context.Background(), "prod-eu-1", resolverGVR, "uid-1", "101", true) - require.True(t, ok, "a matched service account is named, not collapsed to the committer") + ui, outcome := r.ResolveAuthor(context.Background(), "prod-eu-1", resolverGVR, "uid-1", "101", true) + require.Equal(t, git.AttributionResolved, outcome, + "a matched service account is named, not collapsed to the committer") assert.Equal(t, sa, ui.Username) count, ok := telemetry.CollectInt64Sum(reader, "gitopsreverser_attribution_resolutions_total", @@ -90,14 +92,16 @@ func TestAuthorResolver_ServiceAccountIsNamed(t *testing.T) { assert.Equal(t, uint64(1), waitCount) } -func TestAuthorResolver_MissExpiresToCommitter(t *testing.T) { +func TestAuthorResolver_MissExpiresToUnresolved(t *testing.T) { lookup := &fakeLookup{resolution: queue.AuthorResolution{Result: queue.AttributionAbsent}, hitAfter: 1000} r := NewAuthorResolver(lookup, 0, logr.Discard()) - // A zero grace does a single lookup and, on a miss, ships as committer (ok=false). - // There is no longer a miss-marker write-back. - _, ok := r.ResolveAuthor(context.Background(), "prod-eu-1", resolverGVR, "uid-1", "101", true) - assert.False(t, ok) + // A zero grace does a single lookup and, on a miss, reports UNRESOLVED — attribution ran + // and did not name anyone. It is deliberately not NotAttempted, which would claim + // attribution was never switched on. There is no miss-marker write-back. + ui, outcome := r.ResolveAuthor(context.Background(), "prod-eu-1", resolverGVR, "uid-1", "101", true) + assert.Equal(t, git.AttributionUnresolved, outcome) + assert.Empty(t, ui.Username, "an unresolved attribution names nobody") assert.Equal(t, 1, lookup.calls) } @@ -111,8 +115,8 @@ func TestAuthorResolver_DeleteEventIsNotExactCapable(t *testing.T) { } r := NewAuthorResolver(lookup, DefaultAttributionGraceWindow, logr.Discard()) - _, ok := r.ResolveAuthor(context.Background(), "prod-eu-1", resolverGVR, "uid-1", "999", false) - require.True(t, ok) + _, outcome := r.ResolveAuthor(context.Background(), "prod-eu-1", resolverGVR, "uid-1", "999", false) + require.Equal(t, git.AttributionResolved, outcome) assert.False(t, lookup.lastExactCapable, "a removal event may consult the /last pointer") } @@ -126,14 +130,38 @@ func TestAuthorResolver_WaitsThroughGraceWindowForLateFact(t *testing.T) { } r := NewAuthorResolver(lookup, 2*time.Second, logr.Discard()) - ui, ok := r.ResolveAuthor(context.Background(), "prod-eu-1", resolverGVR, "uid-1", "101", true) - require.True(t, ok) + ui, outcome := r.ResolveAuthor(context.Background(), "prod-eu-1", resolverGVR, "uid-1", "101", true) + require.Equal(t, git.AttributionResolved, outcome) assert.Equal(t, "bob", ui.Username) assert.GreaterOrEqual(t, lookup.calls, 3) } -func TestAuthorResolver_NilLookupIsCommitter(t *testing.T) { +// A nil lookup is configured-author mode: attribution was never switched on, so the outcome +// must be NotAttempted — not Unresolved. Conflating the two is what made a lost actor +// indistinguishable from a deployment that simply does not do attribution. +func TestAuthorResolver_NilLookupIsNotAttempted(t *testing.T) { r := NewAuthorResolver(nil, DefaultAttributionGraceWindow, logr.Discard()) - _, ok := r.ResolveAuthor(context.Background(), "prod-eu-1", resolverGVR, "uid-1", "101", true) - assert.False(t, ok) + + ui, outcome := r.ResolveAuthor(context.Background(), "prod-eu-1", resolverGVR, "uid-1", "101", true) + + assert.Equal(t, git.AttributionNotAttempted, outcome, + "attribution that was never enabled has not failed — the committer legitimately authors") + assert.Empty(t, ui.Username) +} + +// A fact that exists but carries no author is also unresolved, not not-attempted: attribution +// ran, found something, and still could not name anyone. +func TestAuthorResolver_AuthorlessFactIsUnresolved(t *testing.T) { + lookup := &fakeLookup{ + resolution: queue.AuthorResolution{ + Fact: queue.AuthorFact{Author: ""}, + Result: queue.AttributionExactUser, + }, + hitAfter: 1, + } + r := NewAuthorResolver(lookup, DefaultAttributionGraceWindow, logr.Discard()) + + _, outcome := r.ResolveAuthor(context.Background(), "prod-eu-1", resolverGVR, "uid-1", "101", true) + + assert.Equal(t, git.AttributionUnresolved, outcome) } diff --git a/internal/watch/manager_snapshot_test.go b/internal/watch/manager_snapshot_test.go index 179e1180..97210aa3 100644 --- a/internal/watch/manager_snapshot_test.go +++ b/internal/watch/manager_snapshot_test.go @@ -21,10 +21,7 @@ import ( itypes "github.com/ConfigButler/gitops-reverser/internal/types" ) -var ( - secretsGVR = schema.GroupVersionResource{Group: "", Version: "v1", Resource: "secrets"} - nodesGVR = schema.GroupVersionResource{Group: "", Version: "v1", Resource: "nodes"} -) +var secretsGVR = schema.GroupVersionResource{Group: "", Version: "v1", Resource: "secrets"} // makeScheme returns a scheme with core Kubernetes types registered. func makeScheme(t *testing.T) *runtime.Scheme { @@ -83,47 +80,10 @@ func addSecretsWatchRule(store *rulestore.RuleStore) { ) } -// addClusterWatchRule registers a cluster-scoped ClusterWatchRule for my-target. -func addClusterWatchRule(store *rulestore.RuleStore, name, resource string) { - store.AddOrUpdateClusterWatchRule( - configv1alpha3.ClusterWatchRule{ - ObjectMeta: metav1.ObjectMeta{Name: name}, - Spec: configv1alpha3.ClusterWatchRuleSpec{ - TargetRef: configv1alpha3.NamespacedTargetReference{Name: "my-target", Namespace: "gitops-reverser"}, - Rules: []configv1alpha3.ClusterResourceRule{{ - APIGroups: []string{""}, APIVersions: []string{"v1"}, Resources: []string{resource}, - Scope: configv1alpha3.ResourceScopeCluster, - }}, - }, - }, - "my-target", "gitops-reverser", "provider", "gitops-reverser", "main", "live", - ) -} - func myTargetRef() itypes.ResourceReference { return itypes.NewResourceReference("my-target", "gitops-reverser") } -// A reconcile fails closed while the cluster API surface has not been observed yet (an -// empty/unready discovery leaves the type registry unready): sweeping a mark over an -// unobserved surface would delete the mirror. -func TestResolveSnapshotGVRs_FailsClosedWhenRegistryNotReady(t *testing.T) { - store := rulestore.NewStore() - addSecretsWatchRule(store) - empty := apiResourceDiscovery(staticCatalogDiscovery{}) - m := &Manager{ - Log: logr.Discard(), - RuleStore: store, - resourceCatalog: NewAPIResourceCatalog(), - discoveryClient: func() (apiResourceDiscovery, error) { return empty, nil }, - } - - _, err := m.resolveSnapshotGVRs(context.Background(), myTargetRef()) - require.Error(t, err, "an unobserved API surface must abort the gather rather than sweep") - assert.Contains(t, err.Error(), "has not been observed yet") -} - -// A normally-served target has no retained types, so the snapshot is not blocked. func TestRetainedWatchedTypes_NoneWhenAllServed(t *testing.T) { store := rulestore.NewStore() addSecretsWatchRule(store) @@ -148,52 +108,3 @@ func TestGVKListSummary(t *testing.T) { assert.Contains(t, got, "Kind=ConfigMap") assert.Contains(t, got, "Kind=Deployment") } - -// resolveSnapshotGVRs scopes a namespaced resource to its rule namespace and a -// cluster-scoped resource cluster-wide (no namespaces). -func TestResolveSnapshotGVRs_ScopesNamespacedAndClusterWide(t *testing.T) { - store := rulestore.NewStore() - addSecretsWatchRule(store) - addClusterWatchRule(store, "cwr-nodes", "nodes") - - m := streamingManager(t, gitTargetFixture(), store) - gvrs, err := m.resolveSnapshotGVRs(context.Background(), myTargetRef()) - require.NoError(t, err) - - byGVR := map[schema.GroupVersionResource][]string{} - for _, sg := range gvrs { - byGVR[sg.gvr] = append(byGVR[sg.gvr], sg.namespace) - } - assert.Equal(t, []string{"ns-a"}, byGVR[secretsGVR], "namespaced resource scoped to its rule namespace") - assert.Equal(t, []string{""}, byGVR[nodesGVR], "cluster-scoped resource gathers under the cluster-wide scope") -} - -// A wildcard resource pattern expands to every served namespaced resource in the group, -// so the snapshot is not silently narrowed. -func TestResolveSnapshotGVRs_WildcardResourceExpands(t *testing.T) { - store := rulestore.NewStore() - store.AddOrUpdateWatchRule( - configv1alpha3.WatchRule{ - ObjectMeta: metav1.ObjectMeta{Name: "wr-all", Namespace: "ns-a"}, - Spec: configv1alpha3.WatchRuleSpec{ - TargetRef: configv1alpha3.LocalTargetReference{Name: "my-target"}, - Rules: []configv1alpha3.ResourceRule{{ - APIGroups: []string{""}, APIVersions: []string{"v1"}, Resources: []string{"*"}, - }}, - }, - }, - "my-target", "gitops-reverser", "provider", "gitops-reverser", "main", "live", - ) - - m := streamingManager(t, gitTargetFixture(), store) - gvrs, err := m.resolveSnapshotGVRs(context.Background(), myTargetRef()) - require.NoError(t, err) - - resources := map[string]struct{}{} - for _, sg := range gvrs { - resources[sg.gvr.Resource] = struct{}{} - } - assert.Contains(t, resources, "configmaps") - assert.Contains(t, resources, "secrets") - assert.Contains(t, resources, "services") -} diff --git a/internal/watch/scope_resolve.go b/internal/watch/scope_resolve.go index 13f6dbc2..37a3248c 100644 --- a/internal/watch/scope_resolve.go +++ b/internal/watch/scope_resolve.go @@ -3,7 +3,6 @@ package watch import ( - "context" "fmt" "sort" "strings" @@ -40,114 +39,6 @@ type ClusterSnapshot struct { CoverageHead string } -// snapshotGVR is one resolved watched resource type paired with the ONE namespace scope to -// gather it under: an empty namespace means cluster-wide (every namespace), exactly as it -// does for a dynamic client List and for targetWatchKey. -// -// A type followed both cluster-wide and in named namespaces yields one entry per scope — -// the same distinct set targetWatchSpecs streams — so a gather's scope is always exactly -// one stream's scope, and therefore exactly the scope its mark-and-sweep may delete over. -type snapshotGVR struct { - gvr schema.GroupVersionResource - namespace string -} - -// resolveSnapshotGVRForType resolves one watched type's (GVR, namespace-scope) set for a -// GitTarget, with the same fail-closed discipline as resolveSnapshotGVRs but scoped to the -// single type. It returns one entry per scope the type is gathered under, so a type followed -// both cluster-wide and in a named namespace reconciles as the two streams it actually is. The -// bool is false when this GitTarget does not watch the type (so there is nothing to reconcile). -// It refuses (error) when the surface is unobserved or the type is currently `retained` (a -// wobble) — the per-type expression of the anti-sweep invariant (R9/R11). -func (m *Manager) resolveSnapshotGVRForType( - ctx context.Context, - gitDest types.ResourceReference, - gvr schema.GroupVersionResource, -) ([]snapshotGVR, bool, error) { - if err := m.RefreshAPIResourceCatalog(ctx); err != nil { - return nil, false, fmt.Errorf("refresh API resource catalog for %s: %w", gitDest.String(), err) - } - m.refreshWatchedTypeTables() - - reg := m.registryForGitTarget(gitDest) - if !reg.Ready() { - return nil, false, fmt.Errorf( - "aborting per-type reconcile for %s: the cluster API surface has not been observed yet", - gitDest.String()) - } - - table := m.residentWatchedTypeTable(gitDest) - var watched *WatchedType - for i := range table.Types { - if table.Types[i].GVR == gvr { - watched = &table.Types[i] - break - } - } - if watched == nil { - return nil, false, nil - } - - if typeWobbling(reg, gvr) { - return nil, false, fmt.Errorf( - "aborting per-type reconcile for %s: %s within the removal grace (currently unserved); "+ - "refusing to reconcile a reduced view", - gitDest.String(), gvr.String()) - } - - return snapshotGVRScopes(*watched), true, nil -} - -// snapshotGVRScopes projects one watched type into its per-scope gather entries: one entry -// per namespace scope the type is streamed under, cluster-wide ("") included. It is the -// single projection both read sites share, so the whole-target and per-type paths cannot -// disagree about a type's scope. -func snapshotGVRScopes(wt WatchedType) []snapshotGVR { - out := make([]snapshotGVR, 0, len(wt.NamespaceOps)) - for _, ns := range wt.WatchScopes() { - out = append(out, snapshotGVR{gvr: wt.GVR, namespace: ns}) - } - return out -} - -// resolveSnapshotGVRs returns the GitTarget's watched (GVR, namespace-scope) set, read from the -// resident watched-type table. It refreshes the trusted API catalog, the registry, and the -// table first, then fails closed if the registry is not ready — a reconcile must never be built -// from an unobserved API surface, and a mark-and-sweep over a reduced view would delete KRM from -// git. A type that briefly leaves discovery stays followable (and so stays in the table) for the -// registry's removal grace, so a transient wobble never sweeps git. It is the scope side of the -// splice and the demand Declare (DEC-L3). -func (m *Manager) resolveSnapshotGVRs( - ctx context.Context, - gitDest types.ResourceReference, -) ([]snapshotGVR, error) { - if err := m.RefreshAPIResourceCatalog(ctx); err != nil { - return nil, fmt.Errorf("refresh API resource catalog for %s: %w", gitDest.String(), err) - } - m.refreshWatchedTypeTables() - - if !m.registryForGitTarget(gitDest).Ready() { - return nil, fmt.Errorf( - "aborting scope resolution for %s: the cluster API surface has not been observed yet; "+ - "refusing to reconcile a partial cluster view", - gitDest.String()) - } - - table := m.residentWatchedTypeTable(gitDest) - - // A watched type the registry holds as `retained` is followable under the removal grace but - // is not actually served right now (a discovery wobble). Reconciling it would sweep a reduced - // view and delete a still-valid mirror, so fail closed until the wobble resolves. - if retained := m.retainedWatchedTypes(gitDest, table); len(retained) > 0 { - return nil, fmt.Errorf( - "aborting scope resolution for %s: %s within the removal grace (currently unserved); "+ - "refusing to sweep a reduced cluster view", - gitDest.String(), gvkListSummary(retained)) - } - - return snapshotGVRsFromTable(table), nil -} - // retainedWatchedTypes returns the GVKs of the target's watched types the registry currently // holds as `retained` (followable under the grace, but not served right now), resolved against // the GitTarget's OWN source cluster's registry. @@ -189,25 +80,6 @@ func gvkListSummary(gvks []schema.GroupVersionKind) string { return fmt.Sprintf("%d watched types [%s]", len(parts), strings.Join(parts, ", ")) } -// snapshotGVRsFromTable projects a watched-type table into the deterministic, sorted -// (GVR, namespace) set: one entry per type per namespace scope, with an empty namespace -// meaning cluster-wide. A type followed both cluster-wide and in a named namespace yields -// both entries — it must agree with targetWatchSpecs scope for scope, or the plan hash and -// the running streams describe different mirrors. -func snapshotGVRsFromTable(table WatchedTypeTable) []snapshotGVR { - out := make([]snapshotGVR, 0, len(table.Types)) - for _, wt := range table.Types { - out = append(out, snapshotGVRScopes(wt)...) - } - sort.Slice(out, func(i, j int) bool { - if out[i].gvr.String() == out[j].gvr.String() { - return out[i].namespace < out[j].namespace - } - return out[i].gvr.String() < out[j].gvr.String() - }) - return out -} - // desiredFromObject converts a materialized object into a desired resource, pairing the // GVR-derived API identity with the sanitized object the writer will materialise. It is shared // by the splice's scope projection (splice_snapshot.go) so a reconcile's desired set is shaped diff --git a/internal/watch/scope_resolve_test.go b/internal/watch/scope_resolve_test.go index 81fc1a99..419d1847 100644 --- a/internal/watch/scope_resolve_test.go +++ b/internal/watch/scope_resolve_test.go @@ -36,25 +36,3 @@ func TestDesiredFromObject(t *testing.T) { _, ok = desiredFromObject(configMapGVR, (*unstructured.Unstructured)(nil)) assert.False(t, ok, "a nil object is not a desired entry") } - -// The snapshot read site must project the same scope set targetWatchSpecs streams. A fix that -// lands only in the watch path leaves the plan hash and the running streams describing -// different mirrors — and, once a per-scope replay drives the mark-and-sweep, a gather wider -// than its stream deletes managed documents that were never in that stream's scope. -func TestSnapshotGVRsFromTable_NamedAndClusterWideScopesStayDistinctEntries(t *testing.T) { - table := WatchedTypeTable{ - Types: []WatchedType{{ - GVR: configMapGVR, - NamespaceOps: map[string]OperationSet{ - "": {"UPDATE": struct{}{}}, - "team-a": {"CREATE": struct{}{}}, - }, - }}, - } - - got := snapshotGVRsFromTable(table) - - require.Len(t, got, 2, "a cluster-wide selection must not swallow the named-namespace gather") - assert.Equal(t, snapshotGVR{gvr: configMapGVR, namespace: ""}, got[0], "cluster-wide scope sorts first") - assert.Equal(t, snapshotGVR{gvr: configMapGVR, namespace: "team-a"}, got[1]) -} diff --git a/internal/watch/snapshot_stream_type_test.go b/internal/watch/snapshot_stream_type_test.go index c2194c02..e7d4aee0 100644 --- a/internal/watch/snapshot_stream_type_test.go +++ b/internal/watch/snapshot_stream_type_test.go @@ -3,40 +3,16 @@ package watch import ( - "context" "testing" - "github.com/go-logr/logr" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" "k8s.io/apimachinery/pkg/runtime/schema" - - "github.com/ConfigButler/gitops-reverser/internal/rulestore" ) // configmapsGVR is a second served namespaced type, used to prove per-type scope resolution // is membership-exact. var configmapsGVR = schema.GroupVersionResource{Group: "", Version: "v1", Resource: "configmaps"} -// resolveSnapshotGVRForType fails closed when the API surface has not been observed yet — the -// per-type expression of the never-reconcile-a-partial-view invariant. -func TestResolveSnapshotGVRForType_FailsClosedWhenRegistryNotReady(t *testing.T) { - store := rulestore.NewStore() - addSecretsWatchRule(store) - empty := apiResourceDiscovery(staticCatalogDiscovery{}) - m := &Manager{ - Log: logr.Discard(), - RuleStore: store, - resourceCatalog: NewAPIResourceCatalog(), - discoveryClient: func() (apiResourceDiscovery, error) { return empty, nil }, - } - - _, _, err := m.resolveSnapshotGVRForType(context.Background(), myTargetRef(), secretsGVR) - require.Error(t, err, "an unobserved API surface must abort the per-type gather") - assert.Contains(t, err.Error(), "has not been observed yet") -} - -// tableWatchesGVR reports membership of a type in a GitTarget's resident table. func TestTableWatchesGVR(t *testing.T) { table := WatchedTypeTable{Types: []WatchedType{{GVR: secretsGVR}}} assert.True(t, tableWatchesGVR(table, secretsGVR), "a watched type is reported present") diff --git a/internal/watch/target_watch.go b/internal/watch/target_watch.go index 21965cfc..40006c05 100644 --- a/internal/watch/target_watch.go +++ b/internal/watch/target_watch.go @@ -152,10 +152,29 @@ func (m *Manager) replaceGitTargetWatches( ops := table.operationsFor(watchKey) go m.runTargetWatch(childCtx, log, table.GitDest, watchKey, ops) } - log.Info("watch-first target watch set reconciled", "watchCount", len(keys)) + // Name every declared stream, not just the count. A GVR appearing twice — once + // cluster-wide ("") and once under a named namespace — means the same object is + // delivered on two streams, which is legitimate scoping but doubles the events for + // objects in that namespace. That is invisible in a bare count. + log.Info("watch-first target watch set reconciled", + "watchCount", len(keys), "streams", describeWatchKeys(keys, specs)) return nil } +// describeWatchKeys renders the declared streams as "@=" +// so a declare log names exactly what is being watched and under which operation filter. +func describeWatchKeys(keys []targetWatchKey, specs map[targetWatchKey]string) string { + parts := make([]string, 0, len(keys)) + for _, key := range keys { + scope := key.Namespace + if scope == "" { + scope = "*cluster-wide*" + } + parts = append(parts, fmt.Sprintf("%s@%s=%s", key.GVR.String(), scope, specs[key])) + } + return strings.Join(parts, " | ") +} + func (m *Manager) prepareTargetWatchSetReplacementLocked( key string, specs map[targetWatchKey]string, @@ -724,13 +743,18 @@ func (m *Manager) routeLiveTargetWatchEvent( // attribution index. The live object still carries its UID and resourceVersion // here (sanitize strips them inside targetWatchGitEvent), so the resolver joins on // the strongest available key. Configured-author mode (nil resolver) leaves UserInfo -// zero, so the writer authors the commit as the configured committer. +// zero, so the writer authors that commit as the configured committer. func (m *Manager) attachAuthor( ctx context.Context, event *git.Event, gvr schema.GroupVersionResource, u *unstructured.Unstructured, ) { + // A nil resolver is configured-author mode: nothing is attempted, and the event's zero + // Attribution is already AttributionNotAttempted — the constant is the empty string so that + // this early return needs no stamp. Do not "fix" this by giving the constant a name-shaped + // value; every non-live path (reconcile, resync, bootstrap) relies on the same zero value, + // and a non-empty constant turns all of them into a fourth state that matches nothing. if m.AuthorResolver == nil { return } @@ -743,9 +767,15 @@ func (m *Manager) attachAuthor( // ClusterProvider name — this event was watched on. It keys the author read against exactly // the facts the audit handler recorded for that cluster, so a fact from cluster A can never // name the author of an object watched on cluster B. - if userInfo, ok := m.AuthorResolver.ResolveAuthor( + userInfo, outcome := m.AuthorResolver.ResolveAuthor( ctx, event.SourceCluster, gvr, u.GetUID(), u.GetResourceVersion(), exactCapable, - ); ok { + ) + // Stamp the outcome even when no actor was named: an unresolved attribution is a fact the + // writer, the author_kind metric, and CommitRequest matching all need. Leaving it at the + // zero value would say "attribution was never attempted", which is exactly the conflation + // that made this loss invisible. + event.Attribution = outcome + if outcome == git.AttributionResolved { event.UserInfo = userInfo } } diff --git a/internal/watch/target_watch_test.go b/internal/watch/target_watch_test.go index 89d6ee17..18cca748 100644 --- a/internal/watch/target_watch_test.go +++ b/internal/watch/target_watch_test.go @@ -166,7 +166,7 @@ func TestRouteLiveTargetWatchEvent_ForwardsObjectEventsAsCommitter(t *testing.T) assert.Equal(t, "CREATE", event.Operation) assert.Equal(t, "target", event.GitTargetName) assert.Equal(t, "default", event.GitTargetNamespace) - assert.Empty(t, event.UserInfo.Username, "unattributed watch events commit as the configured committer") + assert.Empty(t, event.UserInfo.Username, "configured-author watch events leave the actor empty") assert.NotNil(t, event.Object) assert.Empty(t, event.Object.GetResourceVersion(), "live events are sanitized before entering Git") } diff --git a/internal/webhook/validate_operator_types_handler.go b/internal/webhook/validate_operator_types_handler.go index a9a843bc..6637876f 100644 --- a/internal/webhook/validate_operator_types_handler.go +++ b/internal/webhook/validate_operator_types_handler.go @@ -59,8 +59,8 @@ type CommandAuthorRecorder interface { // does one thing: for a command kind (a CommitRequest) it captures the authenticated // submitter into the CommandAuthorStore and always allows — pure observation with a // single side effect (a Redis upsert), never a rejection, so a user's command never -// depends on it succeeding (a missed capture degrades to a committer-authored commit; -// see docs/spec/commitrequest-admission-authorship.md §2, §4). It dispatches on the +// depends on it succeeding (a missed capture leaves the request without a claimed actor; +// see docs/spec/commitrequest-admission-authorship.md). It dispatches on the // resource (isCommandKind today), so a future config-validation branch for non-command // kinds slots in alongside without disturbing this one. type ValidateOperatorTypesHandler struct { @@ -70,7 +70,7 @@ type ValidateOperatorTypesHandler struct { // Handle records {uid → author} for an admitted command CREATE before the object // persists (the authorship invariant, §2), then allows. Every early return still // allows: a non-command kind, a dry-run, a missing uid, or an unauthenticated request -// simply records nothing and falls back to the committer downstream. +// simply records nothing and leaves the request without a claimed actor downstream. func (h *ValidateOperatorTypesHandler) Handle(ctx context.Context, req admission.Request) admission.Response { log := logf.FromContext(ctx).WithName("validate-operator-types") @@ -87,7 +87,7 @@ func (h *ValidateOperatorTypesHandler) Handle(ctx context.Context, req admission // No store means the webhook is running without a Redis backend (admission is on by // default, but command-author capture is Redis-backed). Allow without recording; the - // controller then finalizes as the committer (AuthorAttributed=False). + // controller then uses the no-actor path (AuthorAttributed=False). if h.Store == nil { return admission.Allowed("no author store: not recorded") } @@ -112,16 +112,16 @@ func (h *ValidateOperatorTypesHandler) Handle(ctx context.Context, req admission } // Synchronous, best-effort: the write completes before we return, so it is present - // before the object is visible (the authorship invariant, §2). A failure degrades to - // the committer — never block the command. + // before the object is visible. A failure leaves the request without a claimed actor — + // never block the command. if err := h.Store.RecordCommandAuthor(ctx, uid, author); err != nil { - log.Error(err, "record command author failed; will fall back to committer", + log.Error(err, "record command author failed; request will claim no actor", "resource", gr.String(), "namespace", req.Namespace, "name", req.Name, "uid", uid) return admission.Allowed("author record failed") } // Info (not V(1)) on purpose: command CREATEs are rare, and this line proves the // webhook was actually invoked and the author captured — the first thing to check - // when a CommitRequest unexpectedly commits as the committer. + // when a CommitRequest unexpectedly reports AuthorAttributed=False. log.Info("recorded command author at admission", "resource", gr.String(), "namespace", req.Namespace, "name", req.Name, "uid", uid, "author", author.Author) diff --git a/test/e2e/Taskfile.yml b/test/e2e/Taskfile-e2e.yml similarity index 100% rename from test/e2e/Taskfile.yml rename to test/e2e/Taskfile-e2e.yml diff --git a/test/e2e/author_mode_test.go b/test/e2e/author_mode_test.go new file mode 100644 index 00000000..39e091a5 --- /dev/null +++ b/test/e2e/author_mode_test.go @@ -0,0 +1,50 @@ +// SPDX-License-Identifier: Apache-2.0 + +package e2e + +import "testing" + +// TestConfiguredAuthorModeFromArgs_AllScenarios verifies that the author-mode decision is read from +// the Deployment's args. Both defaults are +// ENABLED in cmd/main.go — so an absent flag means attribution mode, and only an explicit +// opt-out selects configured-author mode. Getting this backwards silently flips the commit +// author assertion instead of failing it. +// +// Note there is no `["--redis-addr="]` row: clearing Redis while attribution stays at its +// default of true is FATAL at startup ("redis-addr is required when author-attribution is +// enabled", cmd/main.go), so it is not a mode this probe can ever be asked to classify. Every +// configured-author row below therefore turns attribution off explicitly. +func TestConfiguredAuthorModeFromArgs_AllScenarios(t *testing.T) { + // The real jsonpath output shape: a bracketed, quoted, comma-separated list. + const live = `["--metrics-bind-address=:8443","--admission-webhook",` + + `"--redis-addr=valkey.valkey-e2e.svc.cluster.local:6379","--redis-insecure"]` + + for _, tc := range []struct { + name string + args string + want bool + }{ + {"live e2e deployment attributes authors", live, false}, + {"no flags at all means attribution (both defaults enabled)", `[]`, false}, + {"explicit attribution opt-out", `["--redis-addr=valkey:6379","--author-attribution=false"]`, true}, + {"attribution off and redis cleared", `["--author-attribution=false","--redis-addr="]`, true}, + {"bare --author-attribution is the true form, not an opt-out", `["--author-attribution"]`, false}, + + // Go's flag package parses booleans with strconv.ParseBool, so every one of these is a + // real opt-out a chart value could produce (e.g. `--set attribution.enabled=0`). Matching + // only the literal "=false" would classify them as attribution mode while the controller + // runs configured-author, flipping the commit-author assertion instead of failing it. + {"numeric false", `["--author-attribution=0","--redis-addr="]`, true}, + {"short false", `["--author-attribution=f","--redis-addr="]`, true}, + {"capitalised false", `["--author-attribution=False","--redis-addr="]`, true}, + {"upper false", `["--author-attribution=FALSE","--redis-addr="]`, true}, + {"numeric true is not an opt-out", `["--author-attribution=1"]`, false}, + {"capitalised true is not an opt-out", `["--author-attribution=True"]`, false}, + } { + t.Run(tc.name, func(t *testing.T) { + if got := configuredAuthorModeFromArgs(tc.args); got != tc.want { + t.Fatalf("configuredAuthorModeFromArgs(%s) = %v, want %v", tc.args, got, tc.want) + } + }) + } +} diff --git a/test/e2e/commit_author_attribution_e2e_test.go b/test/e2e/commit_author_attribution_e2e_test.go index 09cb140f..0789f416 100644 --- a/test/e2e/commit_author_attribution_e2e_test.go +++ b/test/e2e/commit_author_attribution_e2e_test.go @@ -86,7 +86,7 @@ var _ = Describe("Commit Author Attribution", Label("manager"), Ordered, func() // Authorship only flows through the per-event audit tail. A ConfigMap created // while the configmaps type is still building its first checkpoint would land in - // the unattributed baseline splice instead (fallback author), so wait until the + // the configured-author baseline splice instead, so wait until the // GitTarget reports StreamsRunning before producing the impersonated writes. waitForStreamsRunning(gitTargetName, testNs) }) diff --git a/test/e2e/commit_request_e2e_test.go b/test/e2e/commit_request_e2e_test.go index 0331bbbb..5823fbde 100644 --- a/test/e2e/commit_request_e2e_test.go +++ b/test/e2e/commit_request_e2e_test.go @@ -144,7 +144,7 @@ var _ = Describe("Commit Request", Label("commit-request", "audit-consumer"), Or "status.sha should match the SHA of the commit on the branch\n%s", recentCommitDiagnostics(repo.CheckoutDir, basePath)) - // Exactly one new commit (UC1, §8.1): this is the first commit on a fresh + // Exactly one new commit: this is the first commit on a fresh // repo, so main holds exactly one commit — the save did not also trigger a // stray second commit. g.Expect(mustCommitCount(repo.CheckoutDir)).To(Equal(1), @@ -214,7 +214,7 @@ var _ = Describe("Commit Request", Label("commit-request", "audit-consumer"), Or // validating admission runs, so the validate-operator-types webhook records the // submitter keyed by uid with no response-body name recovery (the old audit // generateName headache is gone, see - // docs/spec/commitrequest-admission-authorship.md §8). This spec proves a + // docs/spec/commitrequest-admission-authorship.md). This spec proves a // generateName CommitRequest still finalizes and becomes Ready. It is skipped in // configured-author mode, where the edit's window is committer-authored and the // named admission author would not match it end to end. @@ -267,8 +267,8 @@ var _ = Describe("Commit Request", Label("commit-request", "audit-consumer"), Or // The UC2 suite exercises a `kubectl apply` bundle that includes a CommitRequest // as its FIRST document — the deliberately-hard ordering where the save intent -// arrives before the work it is meant to save (docs/spec/commitrequest-design.md -// §2 UC2, §6.2, §8.2). A non-zero spec.closeDelaySeconds is the close-delay collect +// arrives before the work it is meant to save (docs/spec/commitrequest-design.md). A non-zero +// spec.closeDelaySeconds is the close-delay collect // window that lets the bundle's resources arrive and join the same window after the // CommitRequest is attributed, so the whole bundle lands in ONE commit carrying // the CommitRequest's message. @@ -352,7 +352,7 @@ var _ = Describe("Commit Request Bundle (UC2)", Label("commit-request", "audit-c By("applying a bundle whose FIRST document is a CommitRequest, then three Deployments") // closeDelaySeconds is sized to comfortably exceed the bundle's per-type ingestion - // spread so the close-delay collect window (§6.2) is deterministic. + // spread so the close-delay collect window is deterministic. var bundle strings.Builder bundle.WriteString(commitRequestManifest(testNs, commitRequestName, gitTargetName, message, 8)) for _, name := range deployNames { @@ -382,7 +382,7 @@ var _ = Describe("Commit Request Bundle (UC2)", Label("commit-request", "audit-c // Exactly one commit on the fresh repo's main: the entire bundle — applied // across the CommitRequest's attribution and the per-type Deployment stream — - // collapsed into a single commit (§8.2 step 4). + // collapsed into a single commit. g.Expect(mustCommitCount(repo.CheckoutDir)).To(Equal(1), "the whole bundle must land in exactly one commit\n%s", recentCommitDiagnostics(repo.CheckoutDir, basePath)) diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index 3e378a9e..00a219bf 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -5,10 +5,12 @@ package e2e import ( "errors" "fmt" + "math" "os" "os/exec" "os/signal" "path/filepath" + "strconv" "strings" "syscall" "testing" @@ -50,7 +52,7 @@ var _ = SynchronizedBeforeSuite(func() []byte { // it is not inherited by the detached `kubectl port-forward` children that // prepare spawns — so it releases cleanly when this process exits, instead of // being pinned past the run. Destructive standalone tasks (clean-cluster) - // honor the same lock via a flock precondition; see test/e2e/Taskfile.yml. + // honor the same lock via a flock precondition; see test/e2e/Taskfile-e2e.yml. acquireE2ERunLock() if img := os.Getenv("PROJECT_IMAGE"); img == "" { @@ -73,6 +75,9 @@ var _ = SynchronizedBeforeSuite(func() []byte { // if the assertion fails — it also releases on process exit). var _ = SynchronizedAfterSuite(func() {}, func() { defer releaseE2ERunLock() + // Diagnostic first, so its numbers reach the artifacts even when the gating + // audit invariant below fails the suite. + reportAttributionStats() assertNoAnomalousAuditOutcomes() }) @@ -121,12 +126,162 @@ func assertNoAnomalousAuditOutcomes() { _, _ = fmt.Fprintf(GinkgoWriter, "✅ no anomalous audit outcomes (0 error-category events across the run)\n") } +// attributionWaitBuckets mirrors telemetry's attributionWaitBuckets tail. Only the seconds-scale +// boundaries are interesting here: the question this report answers is how close the run came to +// the grace window, and how far past the 3s DEFAULT grace a fact can still arrive. +var attributionWaitBuckets = []float64{0.5, 1, 2, 3, 5, 10} + +// reportAttributionStats prints the run's author-attribution outcome and timing distribution. +// +// It is DIAGNOSTIC, not gating. When no audit fact matches within the grace window, the commit is +// authored as `unknown (attribution unresolved)`, making the missing actor visible in Git history +// and in the metrics below. +// +// The two numbers to read: +// +// - "absent" — resolutions that gave up without naming an actor. Each live commit is authored +// as `unknown (attribution unresolved)`, not as the configured committer. For an e2e mutation +// that should be attributable, this is an audit-attribution configuration or delivery problem. +// - resolutions resolved ABOVE 3s — facts that arrived later than the 3s default grace. Each is +// an attribution the default would have dropped, so this is the direct measure of how much +// headroom --author-attribution-grace is buying (e2e sets 10s; see config/deployment.yaml). +// +// Fact delivery is bounded below by the apiserver's --audit-webhook-batch-max-wait (1s in this +// cluster), so a healthy run still shows most waits in the 0.5-2s range; the tail is what matters. +func reportAttributionStats() { + // Mirrors assertNoAnomalousAuditOutcomes. Not optional: the AfterSuite runs on every leg, + // including ones whose selected specs never touched a metric helper, so promAPI may still be + // nil here. Without this the first queryPrometheus below nil-derefs and PANICS the + // SynchronizedAfterSuite — which fails the whole leg despite every spec passing, and takes + // the gating assertNoAnomalousAuditOutcomes call after it down with it. That is exactly what + // happened to the four small legs (1/3/6/11 specs) in run 29745528349, while the two large + // legs passed only because their specs had already initialized the client. + ensurePrometheusClient() + + total, err := queryPrometheus(`sum(max_over_time(gitopsreverser_attribution_resolutions_total[2h])) or vector(0)`) + if err != nil || total == 0 { + _, _ = fmt.Fprintf(GinkgoWriter, + "ℹ️ attribution stats skipped: no resolutions recorded (configured-author mode or no live events)\n") + return + } + + _, _ = fmt.Fprintf(GinkgoWriter, "\n📊 author attribution — %.0f resolutions this run\n", total) + + var absent float64 + for _, result := range []string{ + "exact_user", "exact_serviceaccount", "weak", "exact_deletecollection_item", "absent", + } { + n, qErr := queryPrometheus(fmt.Sprintf( + `sum(max_over_time(gitopsreverser_attribution_resolutions_total{result=%q}[2h])) or vector(0)`, result)) + if qErr != nil { + continue + } + if result == "absent" { + absent = n + } + _, _ = fmt.Fprintf(GinkgoWriter, " result %-28s = %6.0f (%5.1f%%)\n", result, n, 100*n/total) + } + + // Cumulative histogram: le=X is "waited at most X seconds". + // + // Split by result, because the two populations answer different questions. For RESOLVED + // results the tail says how close fact delivery ran to the window — anything above 3s only + // succeeded because e2e widens the grace past the 3s default. For "absent" the wait is just + // the grace window being spent, so a cluster of absents at the ceiling means "waited the + // whole time and nothing ever came", NOT "arrived slightly too late". + _, _ = fmt.Fprintf(GinkgoWriter, " wait distribution (cumulative):\n") + printWaitBuckets("resolved", `,result!="absent"`) + printWaitBuckets("absent ", `,result="absent"`) + + resolvedOverDefault, err := queryPrometheus( + `(sum(max_over_time(gitopsreverser_attribution_resolution_wait_seconds_count{result!="absent"}[2h])) ` + + `or vector(0)) - (sum(max_over_time(` + + `gitopsreverser_attribution_resolution_wait_seconds_bucket{result!="absent",le="3.0"}[2h])) or vector(0))`) + if err == nil { + _, _ = fmt.Fprintf(GinkgoWriter, + " %.0f resolution(s) SUCCEEDED after waiting longer than the 3s default grace"+ + " — these are the ones a 3s window would have lost.\n", resolvedOverDefault) + } + if absent > 0 { + _, _ = fmt.Fprintf(GinkgoWriter, + " ⚠ %.0f resolution(s) produced unknown (attribution unresolved) (%.1f%%).\n"+ + " For an e2e mutation that should be attributable, check the audit policy, webhook\n"+ + " route, source identity, and Redis delivery with hack/attribution-diagnostics.sh.\n"+ + " An absent fact is not proof of a configuration defect for every live event: some\n"+ + " changes have no audit actor by construction.\n", + absent, 100*absent/total) + } +} + +// printWaitBuckets renders one cumulative wait histogram. The `le` label is rendered by the +// OTel→Prometheus exporter as "1.0"/"3.0"/"10.0" — NOT "1"/"3"/"10" — so an integral boundary +// must be formatted with a decimal or every query above 0.5 silently matches nothing and +// returns 0, which reads as a plausible (and wrong) distribution rather than as an error. +func printWaitBuckets(label, resultSelector string) { + var prev float64 + for _, le := range attributionWaitBuckets { + leLabel := strconv.FormatFloat(le, 'g', -1, 64) + if le == math.Trunc(le) { + leLabel = fmt.Sprintf("%.1f", le) + } + n, qErr := queryPrometheus(fmt.Sprintf( + `sum(max_over_time(gitopsreverser_attribution_resolution_wait_seconds_bucket{le=%q%s}[2h]))`+ + ` or vector(0)`, leLabel, resultSelector)) + if qErr != nil { + continue + } + _, _ = fmt.Fprintf(GinkgoWriter, " %s <=%5.1fs %6.0f (+%.0f)\n", label, le, n, n-prev) + prev = n + } +} + +// configuredAuthorModeEnabled reports whether the deployed controller authors every commit +// as the configured committer (configured-author mode) rather than naming the real actor +// from audit facts (attribution mode). +// +// It reads the DEPLOYMENT'S ARGS, which are the actual input to the decision. It previously +// grepped `kubectl logs --since=30m` for a startup banner and returned false on any error — +// which silently answers "attribution mode" whenever the controller has been up longer than +// the log window, or when `kubectl logs` picks the wrong pod mid-rollout. That is a probe +// that fails open on the exact question the author assertion turns on, and both author +// values cannot be inferred safely from a single commit: the configured committer is expected in +// configured-author mode, while an attribution miss is explicitly unresolved. A wrong mode answer +// would still select the wrong assertion, so this reads the spec and fails loudly when it cannot. func configuredAuthorModeEnabled() bool { - out, err := kubectlRunInNamespace(namespace, "logs", "deployment/gitops-reverser", "--since=30m") - if err != nil { - return false + out, err := kubectlRunInNamespace(namespace, "get", "deployment", "gitops-reverser", + "-o", "jsonpath={.spec.template.spec.containers[*].args}") + ExpectWithOffset(1, err).NotTo(HaveOccurred(), + "failed to read the controller Deployment's args; the author-mode branch cannot be chosen safely") + return configuredAuthorModeFromArgs(out) +} + +// configuredAuthorModeFromArgs is the pure decision, mirroring the mode switch in +// cmd/main.go: attribution runs only when author attribution is on AND Redis is configured; +// anything else is configured-author mode. Both flags default to ENABLED in cmd/main.go +// (`--author-attribution` defaults true, `--redis-addr` defaults to "valkey:6379"), so an +// absent flag means attribution, and only an explicit opt-out turns it off. +func configuredAuthorModeFromArgs(args string) bool { + attribution := true + redisAddr := "valkey:6379" + for _, arg := range strings.Fields(strings.NewReplacer(`"`, " ", `[`, " ", `]`, " ", `,`, " ").Replace(args)) { + switch { + case arg == "--author-attribution": + // The bare form is the TRUE form for a boolean flag, not an opt-out. + attribution = true + case strings.HasPrefix(arg, "--author-attribution="): + // Parse with strconv.ParseBool, exactly as Go's flag package does. Matching only the + // literal "--author-attribution=false" would read `--set attribution.enabled=0` (or + // `f`, `F`, `False`, `FALSE`) as attribution ENABLED while the controller runs + // configured-author — silently swapping the commit-author assertion instead of + // failing it, which is the whole failure mode this probe exists to prevent. + if v, err := strconv.ParseBool(strings.TrimPrefix(arg, "--author-attribution=")); err == nil { + attribution = v + } + case strings.HasPrefix(arg, "--redis-addr="): + redisAddr = strings.TrimPrefix(arg, "--redis-addr=") + } } - return strings.Contains(out, "configured-author mode:") + return !attribution || redisAddr == "" } var _ = AfterEach(func() { diff --git a/test/e2e/helpers.go b/test/e2e/helpers.go index 20f2df71..5f503237 100644 --- a/test/e2e/helpers.go +++ b/test/e2e/helpers.go @@ -64,6 +64,15 @@ func verifyPrometheusAvailable() { // queryPrometheus executes a PromQL query and returns the first scalar value // Returns 0 if no results found func queryPrometheus(query string) (float64, error) { + // A nil client is a wiring mistake in the caller (it skipped ensurePrometheusClient), not a + // query failure. Say so instead of nil-dereferencing: a panic here aborts whatever node is + // running — and from an AfterSuite that fails the entire leg with every spec passing, which + // is unreadable from the console log. + if promAPI == nil { + return 0, fmt.Errorf( + "prometheus client not initialized: call ensurePrometheusClient() before querying %q", query) + } + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) //nolint:mnd // reasonable query timeout defer cancel() diff --git a/test/e2e/setup/argocd/README.md b/test/e2e/setup/argocd/README.md index acdc8d9f..83dcbc3b 100644 --- a/test/e2e/setup/argocd/README.md +++ b/test/e2e/setup/argocd/README.md @@ -1,7 +1,7 @@ # Argo CD (bi-directional e2e corner) The only Argo CD installation in this repository. Installed by the -`_argocd-installed` node in [`test/e2e/Taskfile.yml`](../../Taskfile.yml), which +`_argocd-installed` node in [`test/e2e/Taskfile-e2e.yml`](../../Taskfile-e2e.yml), which only `task test-e2e-bi-directional` and `task argocd-ui` depend on — so the other CI legs never pay for it. @@ -51,7 +51,7 @@ The specs in `test/e2e/argocd_bi_directional_e2e_test.go` assert Argo CD's value format, client-side apply. A floating version would let an upstream default change turn a real behavioural regression into a silent pass. -`ARGOCD_CHART_VERSION` lives in [`test/e2e/Taskfile.yml`](../../Taskfile.yml) and +`ARGOCD_CHART_VERSION` lives in [`test/e2e/Taskfile-e2e.yml`](../../Taskfile-e2e.yml) and pins the chart, which carries a specific Argo CD release (its appVersion; e.g. chart `10.1.3` ships `v3.4.5`). When bumping, re-verify against the upstream source **at the appVersion the chart ships** (a checkout lives at diff --git a/test/e2e/setup/argocd/values.yaml b/test/e2e/setup/argocd/values.yaml index bdfed778..fdf7089a 100644 --- a/test/e2e/setup/argocd/values.yaml +++ b/test/e2e/setup/argocd/values.yaml @@ -3,7 +3,7 @@ # This is the ONLY Argo CD install in the repository. It is applied by the # `_argocd-installed` Taskfile node (helm upgrade --install), which only # `task test-e2e-bi-directional` and `task argocd-ui` depend on. The -# chart version is pinned in ARGOCD_CHART_VERSION (test/e2e/Taskfile.yml); its +# chart version is pinned in ARGOCD_CHART_VERSION (test/e2e/Taskfile-e2e.yml); its # appVersion is the Argo CD release the specs pin behaviour against. # # Design: docs/spec/e2e-bi-directional-corner.md. README.md lists what to