From a478b38120b96e6f49716f2882f1b3c08b3fa779 Mon Sep 17 00:00:00 2001 From: Simon Koudijs Date: Mon, 20 Jul 2026 10:28:07 +0000 Subject: [PATCH 01/17] ci: validate every pull request, not only those based on main MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `ci.yml` triggered on `pull_request: branches: [main]`, so a stacked PR — one based on the previous PR's branch rather than on main — never matched and ran no validation at all. PR #255 (PR 2 of the WatchRule source-namespace stack) merged having run only CodeRabbit and GitGuardian: no lint, no unit tests, no e2e. Retargeting does not rescue it. When the base branch merges, GitHub retargets the still-open PR onto main and emits `pull_request` with action `edited`, which is not in the default trigger set (opened/synchronize/reopened), so nothing re-runs. The PR is then mergeable into main having never been validated by this pipeline. Drop the `branches` filter so validation follows the pull request rather than its base. This costs some runner time on stacked PRs and is what makes a stack reviewable one PR at a time — the workstream in docs/design/watchrule-source-namespace/ is five stacked PRs, so without it four of the five would merge unvalidated. The trust model is unchanged: `pull_request` still grants no secrets and no registry writes, and GitHub independently forces a read-only token on fork PRs. docs/ci-overview.md already documents the trigger as `pull_request`, so this makes the workflow match the documented design. Verified with actionlint. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/ci.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e046c378..037a7024 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,8 +11,16 @@ name: CI on: + # Every pull request, whatever its base — deliberately NOT filtered to `branches: [main]`. + # A stacked PR (PR N based on PR N-1's branch) never matches a main-only filter, so it + # merged with no lint, no unit tests, and no e2e: PR #255 ran only the external review + # bots. Retargeting does not rescue it either — when the base branch merges, GitHub + # retargets the open PR onto main and fires `pull_request` with action `edited`, which + # is not in the default trigger set (opened/synchronize/reopened), so nothing re-runs. + # Validating every base costs a little runner time and is what makes a stack reviewable + # one PR at a time. See docs/design/watchrule-source-namespace/README.md for the stack + # this was found on. pull_request: - branches: [main] workflow_call: outputs: ci-image: From a63532a73fd82876fd423635a6716f8be6fbed50 Mon Sep 17 00:00:00 2001 From: Simon Koudijs Date: Mon, 20 Jul 2026 10:42:09 +0000 Subject: [PATCH 02/17] test(e2e): read the author mode from the Deployment, not from a log grep MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `configuredAuthorModeEnabled` decided which commit-author assertion applies by shelling out to `kubectl logs deployment/gitops-reverser --since=30m` and grepping for a startup banner, returning false on any kubectl error. That is a probe that fails OPEN on the exact question the assertion turns on, in at least three ways: - a controller up longer than 30 minutes has its banner age out of the window; - `kubectl logs deployment/...` can error or pick the wrong pod mid-rollout; - a spec that redeploys the controller in configured-author mode (the quickstart specs do) leaves the banner in the shared log window, so a later spec reads a mode that is no longer current. It matters more than a normal flaky helper because both author values are ambiguous in isolation: "GitOps Reverser" is BOTH the configured-author identity AND the fallback used when attribution finds no matching audit fact (git.DefaultCommitterName). A wrong probe therefore silently swaps the assertion for a weaker one that passes, instead of failing. Read the Deployment's container args, which are the actual input to the mode switch in cmd/main.go, and fail loudly when they cannot be read. The decision is factored into a pure `configuredAuthorModeFromArgs` with unit coverage, so the default-handling is pinned: cmd/main.go defaults BOTH `--author-attribution` (true) and `--redis-addr` ("valkey:6379") to enabled, so an absent flag means attribution mode and only an explicit opt-out selects configured-author mode. Found while investigating an attribution failure on PR 2; see docs/design/watchrule-source-namespace/pr2-e2e-attribution-investigation.md. This is NOT the cause of that failure — the deployment there was genuinely in attribution mode and the probe genuinely returned the right answer — but it made the failure take far longer to interpret than it should have. Co-Authored-By: Claude Opus 4.8 (1M context) --- test/e2e/author_mode_test.go | 34 ++++++++++++++++++++++++++++++ test/e2e/e2e_suite_test.go | 40 ++++++++++++++++++++++++++++++++---- 2 files changed, 70 insertions(+), 4 deletions(-) create mode 100644 test/e2e/author_mode_test.go diff --git a/test/e2e/author_mode_test.go b/test/e2e/author_mode_test.go new file mode 100644 index 00000000..23453dfa --- /dev/null +++ b/test/e2e/author_mode_test.go @@ -0,0 +1,34 @@ +// SPDX-License-Identifier: Apache-2.0 + +package e2e + +import "testing" + +// The author-mode decision must be read from the Deployment's args, and both defaults are +// ENABLED in cmd/main.go — so an absent flag means attribution mode, and only an explicit +// opt-out selects configured-author mode. Getting this backwards silently flips the commit +// author assertion instead of failing it. +func TestConfiguredAuthorModeFromArgs(t *testing.T) { + // The real jsonpath output shape: a bracketed, quoted, comma-separated list. + const live = `["--metrics-bind-address=:8443","--admission-webhook",` + + `"--redis-addr=valkey.valkey-e2e.svc.cluster.local:6379","--redis-insecure"]` + + for _, tc := range []struct { + name string + args string + want bool + }{ + {"live e2e deployment attributes authors", live, false}, + {"no flags at all means attribution (both defaults enabled)", `[]`, false}, + {"explicit attribution opt-out", `["--redis-addr=valkey:6379","--author-attribution=false"]`, true}, + {"redis explicitly cleared", `["--redis-addr="]`, true}, + {"attribution off and redis cleared", `["--author-attribution=false","--redis-addr="]`, true}, + {"bare --author-attribution is the true form, not an opt-out", `["--author-attribution"]`, false}, + } { + t.Run(tc.name, func(t *testing.T) { + if got := configuredAuthorModeFromArgs(tc.args); got != tc.want { + t.Fatalf("configuredAuthorModeFromArgs(%s) = %v, want %v", tc.args, got, tc.want) + } + }) + } +} diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index 3e378a9e..87b63480 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -121,12 +121,44 @@ func assertNoAnomalousAuditOutcomes() { _, _ = fmt.Fprintf(GinkgoWriter, "✅ no anomalous audit outcomes (0 error-category events across the run)\n") } +// configuredAuthorModeEnabled reports whether the deployed controller authors every commit +// as the configured committer (configured-author mode) rather than naming the real actor +// from audit facts (attribution mode). +// +// It reads the DEPLOYMENT'S ARGS, which are the actual input to the decision. It previously +// grepped `kubectl logs --since=30m` for a startup banner and returned false on any error — +// which silently answers "attribution mode" whenever the controller has been up longer than +// the log window, or when `kubectl logs` picks the wrong pod mid-rollout. That is a probe +// that fails open on the exact question the author assertion turns on, and both author +// values are ambiguous on their own: "GitOps Reverser" is BOTH the configured-author +// identity and the fallback used when attribution finds no matching fact +// (git.DefaultCommitterName). A wrong answer here silently flips the assertion instead of +// failing, so this reads the spec and fails loudly when it cannot. func configuredAuthorModeEnabled() bool { - out, err := kubectlRunInNamespace(namespace, "logs", "deployment/gitops-reverser", "--since=30m") - if err != nil { - return false + out, err := kubectlRunInNamespace(namespace, "get", "deployment", "gitops-reverser", + "-o", "jsonpath={.spec.template.spec.containers[*].args}") + ExpectWithOffset(1, err).NotTo(HaveOccurred(), + "failed to read the controller Deployment's args; the author-mode branch cannot be chosen safely") + return configuredAuthorModeFromArgs(out) +} + +// configuredAuthorModeFromArgs is the pure decision, mirroring the mode switch in +// cmd/main.go: attribution runs only when author attribution is on AND Redis is configured; +// anything else is configured-author mode. Both flags default to ENABLED in cmd/main.go +// (`--author-attribution` defaults true, `--redis-addr` defaults to "valkey:6379"), so an +// absent flag means attribution, and only an explicit opt-out turns it off. +func configuredAuthorModeFromArgs(args string) bool { + attribution := true + redisAddr := "valkey:6379" + for _, arg := range strings.Fields(strings.NewReplacer(`"`, " ", `[`, " ", `]`, " ", `,`, " ").Replace(args)) { + switch { + case arg == "--author-attribution=false": + attribution = false + case strings.HasPrefix(arg, "--redis-addr="): + redisAddr = strings.TrimPrefix(arg, "--redis-addr=") + } } - return strings.Contains(out, "configured-author mode:") + return !attribution || redisAddr == "" } var _ = AfterEach(func() { From d41e3920a835373485cf2befbf76b01f5da40f29 Mon Sep 17 00:00:00 2001 From: Simon Koudijs Date: Mon, 20 Jul 2026 11:39:06 +0000 Subject: [PATCH 03/17] chore(dev): make the e2e task surface hard to misuse MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two ergonomics fixes for footguns that each cost a real debugging cycle in the session that landed PR 2. Rename test/e2e/Taskfile.yml to Taskfile-e2e.yml. Every path in that file is relative to the REPO ROOT, because the root Taskfile includes it with `dir: .`. While it was named Taskfile.yml, running `task` from inside test/e2e/ stopped Task's upward search there and executed with test/e2e/ as the working directory, so every one of those paths silently resolved somewhere that does not exist. The observed damage: `task clean-cluster` deleted the k3d cluster (cwd-independent) while its `rm -rf .stamps/cluster/` hit a nonexistent relative path, leaving a stale `ready` stamp that made the next `prepare-e2e` skip cluster creation and fail with "No nodes found for given cluster". With no Taskfile.yml in test/e2e/, the search always reaches the repo root and the working directory is right no matter where the command is typed. docs/tasks-overview.md records why, so it does not get renamed back. Add a PreToolUse hook refusing `task … | …` when pipefail is unset. A pipeline reports the LAST command's status, so `task prepare-e2e | tail -15 && task test-e2e` reported success for a failed prepare and ran the suite against a cluster that was never created — surfacing much later as an unrelated-looking SynchronizedBeforeSuite failure. The root Taskfile sets pipefail for commands inside tasks, which does nothing for a pipeline typed at the shell. .gitignore un-ignores .claude/hooks/ because the tracked settings.json points at the hook: ignoring it would leave everyone else with a hook reference to a file they do not have, failing every Bash call. Co-Authored-By: Claude Opus 4.8 (1M context) --- .claude/hooks/block-unguarded-task-pipe.sh | 53 +++++++++++++++++++ .claude/settings.json | 15 ++++++ .gitignore | 4 ++ Taskfile.yml | 2 +- docs/ci-overview.md | 2 +- .../finished/higher-level-krm-documents.md | 2 +- docs/spec/e2e-bi-directional-corner.md | 4 +- docs/tasks-overview.md | 14 ++++- test/e2e/{Taskfile.yml => Taskfile-e2e.yml} | 0 test/e2e/e2e_suite_test.go | 2 +- test/e2e/setup/argocd/README.md | 4 +- test/e2e/setup/argocd/values.yaml | 2 +- 12 files changed, 94 insertions(+), 10 deletions(-) create mode 100755 .claude/hooks/block-unguarded-task-pipe.sh rename test/e2e/{Taskfile.yml => Taskfile-e2e.yml} (100%) diff --git a/.claude/hooks/block-unguarded-task-pipe.sh b/.claude/hooks/block-unguarded-task-pipe.sh new file mode 100755 index 00000000..3b1d8cdb --- /dev/null +++ b/.claude/hooks/block-unguarded-task-pipe.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +# PreToolUse(Bash) hook: refuse `task … | something` when pipefail is not set. +# +# Why: a pipeline's exit status is the LAST command's status, so +# +# task prepare-e2e | tail -15 && task test-e2e +# +# reports `tail`'s success even when `prepare-e2e` failed — and the suite then +# runs against a cluster that was never created, failing much later in +# SynchronizedBeforeSuite with an unrelated-looking error. This has cost real +# debugging cycles more than once (see docs/design/support-boundary/ +# render-fidelity.md and kustomize-token-writeback-explained.md), which is why +# it is enforced mechanically here rather than written down a third time. +# +# The root Taskfile sets `pipefail` for commands *inside* tasks; that does +# nothing for a pipeline typed at the shell, which is what this guards. +# +# Reads the hook payload on stdin, emits a PreToolUse deny decision on stdout. +set -uo pipefail + +payload="$(cat)" +cmd="$(printf '%s' "${payload}" | jq -r '.tool_input.command // ""')" + +# Already guarded — the caller opted into pipefail semantics explicitly. +case "${cmd}" in +*pipefail*) exit 0 ;; +esac + +# Match a `task` invocation whose stdout is piped onward: +# (^|[;&(|]) start of command, or after a separator (covers && and ||) +# task[[:space:]] the `task` binary, not a word ending in "task" +# ([^|;&]|&[^&])* its arguments — a bare & is allowed so `2>&1` still counts, +# but && ends the command and stops the match +# \|[^|] a real pipe, not || +pipe_re='(^|[;&(|])[[:space:]]*task[[:space:]]([^|;&]|&[^&])*\|[^|]' + +if ! printf '%s' "${cmd}" | grep -Eq "${pipe_re}"; then + exit 0 +fi + +reason='Piping `task` into another command masks its exit status: the pipeline reports the LAST command'\''s status, so a failed task looks like success. This has caused a suite to run against a cluster that was never created. Either drop the pipe and let the output stream, or capture it and check the status yourself: + + task >/tmp/task.log 2>&1; rc=$?; tail -20 /tmp/task.log; exit $rc + +If you genuinely want the pipe, prefix the command with `set -o pipefail;`.' + +jq -nc --arg r "${reason}" '{ + hookSpecificOutput: { + hookEventName: "PreToolUse", + permissionDecision: "deny", + permissionDecisionReason: $r + } +}' diff --git a/.claude/settings.json b/.claude/settings.json index e619624c..608eddba 100644 --- a/.claude/settings.json +++ b/.claude/settings.json @@ -10,5 +10,20 @@ "Bash(KUBEBUILDER_ASSETS=/workspaces/gitops-reverser/bin/k8s/1.35.0-linux-amd64 go test -count=1 -timeout 90s -ginkgo.focus=\"Should clear LastCommit to prevent information disclosure\" ./internal/controller/)", "Bash(gh pr *)" ] + }, + "hooks": { + "PreToolUse": [ + { + "matcher": "Bash", + "hooks": [ + { + "type": "command", + "command": "bash \"$CLAUDE_PROJECT_DIR/.claude/hooks/block-unguarded-task-pipe.sh\"", + "timeout": 10, + "statusMessage": "Checking for unguarded task pipelines" + } + ] + } + ] } } diff --git a/.gitignore b/.gitignore index 37decd61..3e3d6cc3 100644 --- a/.gitignore +++ b/.gitignore @@ -59,5 +59,9 @@ loadtest .env .claude/* +# Hook scripts are referenced by the tracked .claude/settings.json, so they must +# be tracked too — otherwise everyone else pulls a hook pointing at a file they +# do not have, and every Bash tool call fails with "No such file". +!.claude/hooks/ .agents/* /external-sources/ diff --git a/Taskfile.yml b/Taskfile.yml index c8e992a2..02ac12fc 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -21,7 +21,7 @@ includes: dir: . flatten: true e2e: - taskfile: ./test/e2e/Taskfile.yml + taskfile: ./test/e2e/Taskfile-e2e.yml dir: . flatten: true diff --git a/docs/ci-overview.md b/docs/ci-overview.md index edf6cd97..bf1ed753 100644 --- a/docs/ci-overview.md +++ b/docs/ci-overview.md @@ -150,7 +150,7 @@ GitHub's OIDC identity and logged in the public Rekor transparency log. | Concern | Lives in | | --- | --- | -| What gets checked (lint, unit, e2e, packaging) | [Taskfile-build.yml](../Taskfile-build.yml), [test/e2e/Taskfile.yml](../test/e2e/Taskfile.yml) — see [tasks-overview.md](tasks-overview.md) | +| What gets checked (lint, unit, e2e, packaging) | [Taskfile-build.yml](../Taskfile-build.yml), [test/e2e/Taskfile-e2e.yml](../test/e2e/Taskfile-e2e.yml) — see [tasks-overview.md](tasks-overview.md) | | Tool versions | [.devcontainer/Dockerfile](../.devcontainer/Dockerfile) (single source for devcontainer *and* CI) | | Validation pipeline | [.github/workflows/ci.yml](../.github/workflows/ci.yml) | | Release pipeline | [.github/workflows/release.yml](../.github/workflows/release.yml) | diff --git a/docs/design/support-boundary/finished/higher-level-krm-documents.md b/docs/design/support-boundary/finished/higher-level-krm-documents.md index 887df749..30cd4530 100644 --- a/docs/design/support-boundary/finished/higher-level-krm-documents.md +++ b/docs/design/support-boundary/finished/higher-level-krm-documents.md @@ -99,7 +99,7 @@ two reasons: Flux CRDs (`helmreleases.helm.toolkit.fluxcd.io`, `kustomizations.kustomize.toolkit.fluxcd.io`) are established during standard -e2e setup ([../../../../test/e2e/Taskfile.yml](../../../../test/e2e/Taskfile.yml), +e2e setup ([../../../../test/e2e/Taskfile-e2e.yml](../../../../test/e2e/Taskfile-e2e.yml), `_flux-installed`), so the live pin uses a `HelmRelease`. The Argo and KRO controllers are not in the base cluster; their kinds are pinned at the unit level, where the analyzer/editor need only the document bytes — no controller, diff --git a/docs/spec/e2e-bi-directional-corner.md b/docs/spec/e2e-bi-directional-corner.md index 356dd780..a3fb5ef3 100644 --- a/docs/spec/e2e-bi-directional-corner.md +++ b/docs/spec/e2e-bi-directional-corner.md @@ -211,7 +211,7 @@ parallel with both. `BI_DIRECTIONAL_GINKGO_PROCS` is therefore `3`. Filter changes so the default suite stops paying for it: -- `test/e2e/Taskfile.yml`: `E2E_LABEL_FILTER` default `!image-refresh` +- `test/e2e/Taskfile-e2e.yml`: `E2E_LABEL_FILTER` default `!image-refresh` → `!image-refresh && !bi-directional` - `.github/workflows/ci.yml`, `full-core` leg: `!manager && !image-refresh` → `!manager && !image-refresh && !bi-directional` @@ -234,7 +234,7 @@ chart**. `_argocd-installed` `helm pull`s the pinned chart into `.stamps/cluster//argocd/` (retryable network step) and `helm upgrade --install`s it with `values.yaml` and `--wait`, then gates on CRD establishment. -- `ARGOCD_CHART_VERSION` (e.g. `10.1.3`) lives in `test/e2e/Taskfile.yml` and pins +- `ARGOCD_CHART_VERSION` (e.g. `10.1.3`) lives in `test/e2e/Taskfile-e2e.yml` and pins the chart, which carries the Argo CD release the specs pin behaviour against (its appVersion, e.g. `v3.4.5`). Unlike `FLUX_VERSION` it is not a devcontainer env var: nothing outside the cluster needs an `argocd` binary — the specs drive Argo diff --git a/docs/tasks-overview.md b/docs/tasks-overview.md index 5550a0f4..c8275b8f 100644 --- a/docs/tasks-overview.md +++ b/docs/tasks-overview.md @@ -34,10 +34,22 @@ Task keeps the behavior easier to follow: `flatten: true` so everything shares one flat command surface, and sets `run: once` so the e2e DAG can fan out through `deps:` without a shared node starting twice. - [Taskfile-build.yml](../Taskfile-build.yml): build, codegen, lint, and unit-test tasks. -- [test/e2e/Taskfile.yml](../test/e2e/Taskfile.yml): the e2e bring-up DAG. +- [test/e2e/Taskfile-e2e.yml](../test/e2e/Taskfile-e2e.yml): the e2e bring-up DAG. Run `task` (or `task help`) to list everything. +> **Why `Taskfile-e2e.yml` and not `Taskfile.yml`?** Deliberate — please don't rename it +> back. Every path in that file is relative to the **repo root** (`.stamps/cluster/…`, +> `test/e2e/cluster/audit`, `find cmd/mutation-capture-lab …`), because the root Taskfile +> includes it with `dir: .`. If it were named `Taskfile.yml`, `task` run from inside +> `test/e2e/` would stop its upward search there and execute with `test/e2e/` as the working +> directory — every one of those paths would then silently resolve to a non-existent +> location. `task clean-cluster` would delete no stamps while still deleting the cluster, +> leaving a stale `ready` stamp that makes the next `prepare-e2e` skip cluster creation and +> fail with `No nodes found for given cluster`. With the current name there is no +> `Taskfile.yml` in `test/e2e/` to stop the search, so `task` walks up to the repo root and +> runs with the correct working directory no matter where you invoke it from. + ## Most important tasks | Task | What it does | When to run it | diff --git a/test/e2e/Taskfile.yml b/test/e2e/Taskfile-e2e.yml similarity index 100% rename from test/e2e/Taskfile.yml rename to test/e2e/Taskfile-e2e.yml diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index 87b63480..81f42781 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -50,7 +50,7 @@ var _ = SynchronizedBeforeSuite(func() []byte { // it is not inherited by the detached `kubectl port-forward` children that // prepare spawns — so it releases cleanly when this process exits, instead of // being pinned past the run. Destructive standalone tasks (clean-cluster) - // honor the same lock via a flock precondition; see test/e2e/Taskfile.yml. + // honor the same lock via a flock precondition; see test/e2e/Taskfile-e2e.yml. acquireE2ERunLock() if img := os.Getenv("PROJECT_IMAGE"); img == "" { diff --git a/test/e2e/setup/argocd/README.md b/test/e2e/setup/argocd/README.md index acdc8d9f..83dcbc3b 100644 --- a/test/e2e/setup/argocd/README.md +++ b/test/e2e/setup/argocd/README.md @@ -1,7 +1,7 @@ # Argo CD (bi-directional e2e corner) The only Argo CD installation in this repository. Installed by the -`_argocd-installed` node in [`test/e2e/Taskfile.yml`](../../Taskfile.yml), which +`_argocd-installed` node in [`test/e2e/Taskfile-e2e.yml`](../../Taskfile-e2e.yml), which only `task test-e2e-bi-directional` and `task argocd-ui` depend on — so the other CI legs never pay for it. @@ -51,7 +51,7 @@ The specs in `test/e2e/argocd_bi_directional_e2e_test.go` assert Argo CD's value format, client-side apply. A floating version would let an upstream default change turn a real behavioural regression into a silent pass. -`ARGOCD_CHART_VERSION` lives in [`test/e2e/Taskfile.yml`](../../Taskfile.yml) and +`ARGOCD_CHART_VERSION` lives in [`test/e2e/Taskfile-e2e.yml`](../../Taskfile-e2e.yml) and pins the chart, which carries a specific Argo CD release (its appVersion; e.g. chart `10.1.3` ships `v3.4.5`). When bumping, re-verify against the upstream source **at the appVersion the chart ships** (a checkout lives at diff --git a/test/e2e/setup/argocd/values.yaml b/test/e2e/setup/argocd/values.yaml index bdfed778..fdf7089a 100644 --- a/test/e2e/setup/argocd/values.yaml +++ b/test/e2e/setup/argocd/values.yaml @@ -3,7 +3,7 @@ # This is the ONLY Argo CD install in the repository. It is applied by the # `_argocd-installed` Taskfile node (helm upgrade --install), which only # `task test-e2e-bi-directional` and `task argocd-ui` depend on. The -# chart version is pinned in ARGOCD_CHART_VERSION (test/e2e/Taskfile.yml); its +# chart version is pinned in ARGOCD_CHART_VERSION (test/e2e/Taskfile-e2e.yml); its # appVersion is the Argo CD release the specs pin behaviour against. # # Design: docs/spec/e2e-bi-directional-corner.md. README.md lists what to From 46cbfed28f5b65c3c0f620e85831d49d3adea4c7 Mon Sep 17 00:00:00 2001 From: Simon Koudijs Date: Mon, 20 Jul 2026 11:39:26 +0000 Subject: [PATCH 04/17] diag(watch): name declared streams, and record the attribution investigation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An e2e failure while landing PR 2 — a commit that should have been authored jane@acme.com was authored "GitOps Reverser" — took far too long to diagnose, and was wrongly blamed on PR 2 three separate times. This adds the instrumentation that settled it and writes down the answer. Findings, measured rather than argued: - Attribution silently loses ~10% of actors under parallel load. The resolver waits DefaultAttributionGraceWindow (3s) for a matching audit fact and then commits as the configured committer. Measured fact-delivery lag on an IDLE cluster is 835-1101ms, bimodal — the signature of the apiserver's --audit-webhook-batch-max-wait=1s. So a 3s budget runs against a ~1s fixed batching delay with ~2s of headroom, and under load the tail crosses it. Resolver wait by result: weak 0.001s, exact_user 0.533s, absent 2.268s. - PR 2 is NOT the cause. Attribution loss rates over ~85 resolutions per arm are 8.0% with PR 2 and 9.6% without; configmaps alone lose 10.0% on baseline. The spec is ~10%-flaky on its own, which no three-sample comparison could separate from an effect. - The loss is invisible by construction: git.DefaultCommitterName ("GitOps Reverser") is ALSO the configured-author identity, so a lost-actor commit is indistinguishable from a correct configured-author commit by inspection. targetWatchSpecs now names every declared stream as "@=" instead of logging a bare count. A GVR appearing twice for one GitTarget means the same object is delivered on two streams — legitimate scoping after PR 2, but invisible in a count. This is how the real case-C fan-out was found (unsupported-folder-dest carries both a cluster-wide and a named configmaps stream). hack/attribution-diagnostics.sh answers the question that splits the diagnosis: is the fact ABSENT from Redis (never delivered) or PRESENT but written too late for the grace window? Fact keys carry a 10-minute TTL, so the remaining TTL back-computes when each landed. docs/design/watchrule-source-namespace/ gains the resolution, the full session log including the reversals, and a watch-construction overview answering how watches are built and at what cardinality (one raw watch per (GitTarget, GVR, namespace) — no reuse across GitTargets). The grace-window fix itself is deliberately NOT in this commit: it is a product behaviour change (a write with no audit fact would wait longer before committing) and deserves its own review. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../pr2-e2e-attribution-investigation.md | 251 ++++++++++++++ .../pr2-session-log-and-findings.md | 322 ++++++++++++++++++ .../watch-construction-overview.md | 206 +++++++++++ hack/attribution-diagnostics.sh | 105 ++++++ internal/watch/target_watch.go | 21 +- 5 files changed, 904 insertions(+), 1 deletion(-) create mode 100644 docs/design/watchrule-source-namespace/pr2-e2e-attribution-investigation.md create mode 100644 docs/design/watchrule-source-namespace/pr2-session-log-and-findings.md create mode 100644 docs/design/watchrule-source-namespace/watch-construction-overview.md create mode 100755 hack/attribution-diagnostics.sh diff --git a/docs/design/watchrule-source-namespace/pr2-e2e-attribution-investigation.md b/docs/design/watchrule-source-namespace/pr2-e2e-attribution-investigation.md new file mode 100644 index 00000000..b8ce6efd --- /dev/null +++ b/docs/design/watchrule-source-namespace/pr2-e2e-attribution-investigation.md @@ -0,0 +1,251 @@ +# Investigation — the author-attribution e2e failure seen while landing PR 2 + +> Working document, opened 2026-07-20 while landing +> [PR 2](pr2-stream-scope-collapse.md). **Status: unresolved.** It records what was +> measured, what each measurement does and does not establish, and where the reasoning +> went wrong, so the next person does not repeat the loop. +> +> **RESOLVED — see [§0](#0-resolution). PR 2 did not cause this.** The failure is a +> pre-existing ~10% attribution loss: the 3s grace window is too tight against an audit +> delivery lag that is ~1s even on an idle cluster. Everything below §0 is the working +> record, including three conclusions that were later overturned; it is kept because the +> reversals are the useful part. +> +> The headline: a commit that should have been authored `jane@acme.com` was authored +> `GitOps Reverser`. Attribution was genuinely lost — this is not a test asserting the +> wrong branch. + +## 0. Resolution + +**The mechanism, measured.** + +Attribution resolves a watch event against an audit fact, waiting up to +`DefaultAttributionGraceWindow` = **3 seconds** +([author_resolver.go](../../../internal/watch/author_resolver.go)). If no fact matches, the +commit is authored by the configured committer — silently, because +`git.DefaultCommitterName` ("GitOps Reverser") is *also* the configured-author identity. + +Measured fact-delivery lag on an **idle** cluster (12 samples, ConfigMap creates with +`--as=jane@acme.com`, polling Valkey for the fact key): + +~~~text +835 837 838 839 839 1092 1095 1096 1099 1099 1100 1101 (ms) +~~~ + +Bimodal at ~840ms and ~1100ms — the signature of the apiserver's +`--audit-webhook-batch-max-wait=1s` ([start-cluster.sh](../../../test/e2e/cluster/start-cluster.sh)). +So the 3s budget runs against a baseline lag of ~1s that is **dominated by a fixed batching +delay**, leaving roughly 2s of headroom for queueing. Under parallel load the tail crosses +3s and the actor is lost. + +Resolver wait times by result confirm it: + +| result | mean wait | +|---|---| +| `weak` | 0.001s | +| `exact_user` | 0.533s | +| `absent` | **2.268s** — polled the grace out and gave up | + +**Not a key-shape problem.** Every ConfigMap uid in Valkey has *both* an exact-`rv` key and a +`:last` key (46 of 46), so facts are indexed correctly. They simply are not there yet. + +**PR 2 is exonerated, by rate rather than by argument.** Attribution loss measured across +whole runs (~85 resolutions each, far more power than one spec's pass/fail): + +| Code | absent | total | loss | +|---|---|---|---| +| PR 2 present | 7 | 88 | **8.0%** | +| PR 2 absent (`4f37759`) | 8 | 83 | **9.6%** | + +Indistinguishable, with baseline marginally *worse*. Per-resource on baseline: +**configmaps = 5 absent / 49 = 10.0% loss** — the failing spec's own resource already loses +1 in 10 actors *without* PR 2. That makes the spec ~10%-flaky per run on its own, and the +2-fail-of-3 versus 0-fail-of-3 split that looked decisive is within the noise of that base +rate. + +**What to fix.** The grace window, not the stream scoping: + +1. Raise `DefaultAttributionGraceWindow` well above the batching delay (3s → ~10s). Cost: a + write with genuinely no audit fact waits the full grace before committing. +2. And/or lower `--audit-webhook-batch-max-wait` in the recommended audit config, which + attacks the dominant fixed term rather than padding around it. +3. Independently: **the fallback is silent.** `absent` is a counter nobody looks at. Losing + the actor is a correctness failure for a product whose promise is naming the actor; it + should be observable at the commit, not only in a metric. + +Reusable diagnostic: [`hack/attribution-diagnostics.sh`](../../../hack/attribution-diagnostics.sh) +separates "fact never delivered" from "fact arrived too late". + +## The failing spec, end to end + +`Manager WatchRule ConfigMap and Secret >> should create Git commit when ConfigMap is added +via WatchRule` +([watchrule_configmap_secret_e2e_test.go:446](../../../test/e2e/watchrule_configmap_secret_e2e_test.go#L446)), +failing at [:581](../../../test/e2e/watchrule_configmap_secret_e2e_test.go#L581): + +~~~text +Expected : GitOps Reverser +to contain substring : jane@acme.com +~~~ + +What the spec does: + +1. Creates a `GitTarget` (`watchrule-configmap-test-dest`) and a `WatchRule` selecting + ConfigMaps in the spec's own namespace, then waits for `Ready`. +2. Creates a ConfigMap **impersonating a user** — `kubectl --as=jane@acme.com` + ([:491](../../../test/e2e/watchrule_configmap_secret_e2e_test.go#L491)). The + impersonation is the whole point: the identity must survive into the Git commit. +3. `Eventually`-polls a local checkout of the Gitea repo until the ConfigMap's YAML exists, + the commit message carries `[CREATE]` and `v1/configmaps/test-configmap`, and the commit + **author** (`git log -1 --pretty=%an`) is the impersonated user. + +The author assertion is branched +([:576-579](../../../test/e2e/watchrule_configmap_secret_e2e_test.go#L576-L579)): + +~~~go +if configuredAuthorModeEnabled() { + g.Expect(author).To(ContainSubstring("GitOps Reverser")) +} else { + g.Expect(author).To(ContainSubstring("jane@acme.com")) +} +~~~ + +### How attribution is supposed to work + +The controller runs in one of two modes. + +- **configured-author mode** — no Redis / audit disabled. Every commit is authored by the + configured committer identity. `cmd/main.go` announces it at startup + ([main.go:274-277](../../../cmd/main.go#L274-L277)). +- **attribution mode** — the default here. The live watch event that produced the write is + matched against an audit-webhook fact carrying the acting user, and that user becomes the + commit author. When no fact matches, the commit falls back to + `DefaultCommitterName = "GitOps Reverser"` + ([internal/git/types.go:22](../../../internal/git/types.go#L22)). + +**`GitOps Reverser` is therefore ambiguous.** It is both the configured-author identity and +the fallback when attribution finds nothing. The observed string alone does not say which +happened — which is exactly why the first reading of this failure was wrong. + +### The mode probe is a log grep, and it fails open + +`configuredAuthorModeEnabled()` +([e2e_suite_test.go:124-130](../../../test/e2e/e2e_suite_test.go#L124-L130)) decides which +branch applies by shelling out to `kubectl logs deployment/gitops-reverser --since=30m` and +grepping for `configured-author mode:`. It returns `false` on **any** kubectl error. + +Two independent ways that misreports: + +- The controller has been up longer than 30 minutes, so the startup banner ages out of the + window. +- `kubectl logs deployment/...` errors or picks the wrong pod mid-rollout. The failing run's + event dump shows two ReplicaSets scaled up at once, so a rollout was in progress. + +Either makes the probe answer `false` — "attribution mode" — and demand `jane@acme.com` +regardless of how the controller is actually configured. It is also re-invoked on every +`Eventually` retry, which is why the failure timeline is full of repeated `kubectl logs` +calls. + +**This was checked and is NOT what happened.** The local deployment emits no +`configured-author mode:` banner at all, so it genuinely runs in attribution mode, the probe +correctly returned `false`, and the spec correctly demanded `jane@acme.com`. The commit fell +back to the default committer, so **attribution was really lost**. + +The probe is still a latent trap worth fixing independently — it can only ever fail in the +direction of demanding attribution that may not be configured. + +## What was measured + +| # | Code under test | Scope | Attribution spec | Notes | +|---|---|---|---|---| +| 1 | PR 2 branch | full suite, 56 specs | **FAILED** | also failed `playground` (see contamination below) | +| 2 | PR 2 branch | full suite, 56 specs | **FAILED** | clean cluster; 55 passed / 1 failed | +| 3 | `4f37759` — PR 2 code absent | `manager` subset, 43 specs | **passed** | 43 passed / 0 failed | +| 4 | `10a530a` — main, PR 2 present | `manager` subset | *running* | the controlled comparison | +| 5 | `10a530a` — main, PR 2 present | **CI**, all 6 sharded legs | **passed** | run 29734882468 | + +Run 5 is the strongest evidence so far that PR 2 did not break attribution. The merged +change was validated on clean GitHub infrastructure by `release.yml` (whose first job is +`uses: ./.github/workflows/ci.yml`), and **every** e2e leg passed — including +`E2E (full-manager)`, the leg that contains this exact spec, since +`Manager WatchRule ConfigMap and Secret` carries the `manager` label. + +That is not the same as proving the local failure is environmental. CI shards the suite +across six legs, so each leg runs at materially lower contention than one 56-spec local run +at `--procs=4` — which is precisely the load difference hypothesised below. CI passing is +consistent with both "PR 2 is innocent" and "the spec is load-sensitive and CI never applies +enough load". It rules out a deterministic regression; it does not rule out a latent race +that this change makes marginally easier to hit. + +Run 4 is the one that discriminates, because it differs from run 3 in exactly one variable. +Runs 2 and 3 differ in **two** — the code *and* the suite scope/parallel load — so run 3 +alone does not convict PR 2. + +Why load plausibly matters: the assertion is an `Eventually` with a 30s budget, and +attribution requires an audit fact to arrive and match within a window. A 56-spec run at +`--procs=4` on a dev container is a materially different timing environment from a 43-spec +run. + +## Why PR 2 was argued to be irrelevant — and why that argument was over-trusted + +PR 2 replaced `SnapshotNamespaces()` with `WatchScopes()` and made both read sites project +one stream per namespace scope. The claim was that this is a **strict no-op** unless a single +`WatchedType` holds both the cluster-wide `""` key and a named-namespace key: + +- only named keys → old returned the sorted names, new returns the same; +- only `""` → old returned `nil` and the caller synthesised `targetWatchKey{GVR, ""}`, new + returns `[""]` and builds the identical key. + +The failing spec creates one `GitTarget` with a single namespaced `WatchRule` and no +co-resident `ClusterWatchRule`, so it never reaches the changed branch. + +The code reading still looks correct. The error was **procedural**: this argument was used +to explain away three successive failures without ever running the cheap control that would +test it. A structural argument about which branch executes is not evidence about a +timing-sensitive integration failure. The control costs ~6 minutes and should have been run +after the first failure, not the third. + +## Confounders that invalidated earlier runs + +- **Self-inflicted cluster contamination.** A targeted `commit-request`-only run left Gitea + SSH keys registered; the next full run failed `playground` with + `HTTP 422: Key title has been used`. Any verdict from that run is untrustworthy. +- **`clean-cluster` run from the wrong directory.** It does `rm -rf .stamps/cluster/` on + a *relative* path, so running it from `test/e2e` deleted nothing real while k3d still + removed the cluster — leaving a stale `ready` stamp. `prepare-e2e` then skipped cluster + creation and everything failed with `No nodes found for given cluster`. +- **Exit status masked by a pipe.** `task prepare-e2e | tail -15 && task test-e2e` takes + `tail`'s exit status, so a failed prepare reported success and the suite ran against a + cluster that did not exist. Use `set -o pipefail` or capture exit codes separately. + +Net: of four runs, one was contaminated and two tested nothing. + +## Not to be confused with + +Three consecutive runs each failed a *different* timing-sensitive spec — `playground` + +`Commit Request Bundle (UC2)`, then `Commit Request generateName`, then this one. The +commit-request failures were separately re-run in isolation and **all 4 commit-request specs +passed**, which is why they were written off. That pattern is what made "this suite is flaky +here" attractive. It may still be true. It is not established. + +## Open questions + +1. **Does the `manager` subset fail on main?** (run 4). If yes, PR 2 is implicated and main + is currently broken — revert first, diagnose second. If no, the variable is suite load. +2. **Does CI reproduce it?** `release.yml`'s first job is `uses: ./.github/workflows/ci.yml`, + which has an `e2e` job, so the main build validates the merged PR 2 on clean + infrastructure — an independent read. +3. **If it is load, where exactly is the window lost?** The audit fact must arrive and match + before the write is authored. Worth instrumenting rather than inferring. +4. **Should the mode probe be fixed regardless?** It is a `--since=30m` log grep that fails + open. A deterministic signal — a status field, or asserting the deployment's flags — would + remove a whole class of misreads. + +## If it turns out to be PR 2 + +The place to look first is the no-op claim, since that is the load-bearing assumption. Test +it directly: build the `WatchedTypeTable` this spec actually produces and assert +`targetWatchSpecs` output is byte-identical before and after the change. If some GitTarget in +that namespace acquires both a `""` and a named scope through a path not anticipated, the +extra stream would deliver the same object twice, and a duplicate delivery racing the audit +match is a plausible mechanism for losing the attributed author. diff --git a/docs/design/watchrule-source-namespace/pr2-session-log-and-findings.md b/docs/design/watchrule-source-namespace/pr2-session-log-and-findings.md new file mode 100644 index 00000000..f273efac --- /dev/null +++ b/docs/design/watchrule-source-namespace/pr2-session-log-and-findings.md @@ -0,0 +1,322 @@ +# Session log — landing PR 2, and the attribution failure it surfaced + +> Written 2026-07-20, at the end of the session that landed +> [PR 2](pr2-stream-scope-collapse.md) and opened the branch for +> [PR 3](pr3-clusterwatchrule-target-admission.md). Companion to +> [pr2-e2e-attribution-investigation.md](pr2-e2e-attribution-investigation.md), which holds +> the mechanism detail; this page is the readable top-to-bottom account: what was done, what +> was measured, what was **ruled out and on what evidence**, and what is still open. +> +> **Bottom line: PR 2 is merged to main (#255) and is NOT the cause of the e2e failure.** +> The cause is a pre-existing ~10% attribution loss — the 3s grace window is too tight +> against an audit delivery lag of ~1s (dominated by apiserver audit batching). Mechanism, +> measurements, and fix options: [§0 of the investigation](pr2-e2e-attribution-investigation.md#0-resolution). +> +> This page keeps the working record, including **three conclusions I reached confidently and +> then had to reverse**. That sequence is the most useful thing here. + +--- + +## 1. What was set out to do + +1. Settle an open question left by PR 1 — one unexplained e2e failure. +2. Implement PR 2 (a cluster-wide selection must not collapse named-namespace streams). +3. Prepare PR 3. + +All three happened. A fourth thing happened that nobody planned: a genuine, pre-existing +attribution defect surfaced — ~10% of actors silently lost under load — plus four pieces of +test/CI infrastructure that were quietly lying to us. + +--- + +## 2. PR 1's open question — closed + +**Question:** the full suite failed `Commit Request Bundle (UC2)` +([commit_request_e2e_test.go:377](../../../test/e2e/commit_request_e2e_test.go#L377)) waiting +for a CommitRequest to become Ready. PR 1 changed `renderReconcileCommitMessage`'s signature +and added `Namespace` to `ReconcileCommitMessageData` — the only plausible contact point. + +**Done:** re-ran with +`E2E_LABEL_FILTER='!image-refresh && !bi-directional && !source-cluster && commit-request'`. + +**Result: all 4 commit-request specs passed.** Did not reproduce. + +**Conclusion:** not a PR 1 regression. Note the caveat that only became visible later — a +single non-reproduction is weak evidence, and this suite turned out to have a *different* +failing spec on each subsequent run. This closure is "did not reproduce", not "proven +unrelated". + +--- + +## 3. PR 2 — what shipped + +**The defect.** `SnapshotNamespaces()` returned `nil` whenever a `WatchedType` carried the +`""` namespace key, and `nil` meant *all namespaces* at every read site. A WatchRule scoped +to one namespace and a ClusterWatchRule scoped cluster-wide, on the same GVR and GitTarget, +collapsed into one all-namespaces stream. The named scope survived only in the plan hash, and +`targetWatchSpecs` substituted `NamespaceOps[""]` for its operation set — so a `CREATE`-only +named rule co-resident with an `UPDATE`-only cluster-wide rule lost its filter too. + +**The fix.** `SnapshotNamespaces` → `WatchedType.WatchScopes`, returning every scope +including the cluster-wide `""` instead of collapsing to it, with **both** read sites +projecting one stream per scope: + +- `targetWatchSpecs` keys a watch per scope, each carrying `NamespaceOps[ns]`. +- `snapshotGVRsFromTable` **and** `resolveSnapshotGVRForType` project through one shared + `snapshotGVRScopes` helper. `snapshotGVR.namespaces []string` became a single + `namespace string`, matching `targetWatchKey` and matching how a dynamic client `List` + takes a namespace (`""` = all). + +**Design choice — distinct streams, not subtraction.** The spec allowed either. "All +namespaces except `team-a`" is not expressible as a watch, and the subtraction variant would +need a sweep scope shaped as a complement, which `git.ResyncScope` cannot represent. + +**The replaced test.** `TestBuildWatchedTypeTable_ClusterWideOverridesNamedNamespaces` +asserted the collapse *as intended*. Deleted, not skipped. Three replacements, each verified +to fail against a restored pre-fix collapse, each on its own assertion: table level, +`targetWatchSpecs` (streams + op sets), `snapshotGVRsFromTable`. + +**Merged** as #255. Base was auto-retargeted from the PR 1 branch to `main` when #254 merged. + +--- + +## 4. The e2e story, in order + +| # | Code | Scope | Result | Verdict on this run | +|---|---|---|---|---| +| 0 | PR 1 branch | `commit-request` only | 4/4 passed | closed §2 | +| 1 | PR 2 branch | full, 56 specs | 2 failed: `playground`, `CommitRequest generateName` | **void** — cluster contaminated by run 0 | +| 2 | PR 2 branch | full, 56 specs | 55 passed, **1 failed**: attribution | valid | +| 3 | `4f37759`, PR 2 **absent** | `manager`, 43 specs | 43 passed, **0 failed** | valid control | +| 4 | `10a530a`, PR 2 **present** | `manager` | 40 passed, **1 failed**: attribution | **valid, controlled** | +| 5 | `10a530a` main, PR 2 present | **CI**, 6 sharded legs | **all passed** | valid, different environment | +| 6 | `4f37759`, PR 2 absent | `manager` | 43 passed, **0 failed** | second control sample | +| 7 | PR 2 + stream instrumentation | `manager` | 43 passed, **0 failed** | **PR 2 passes** — split the "controlled" story open | +| 8 | PR 2 (run 7), metrics | — | absent **7 / 88 = 8.0%** | the measurement that settled it | +| 9 | `4f37759` baseline, metrics | — | absent **8 / 83 = 9.6%** | baseline is *worse* | + +Runs 3, 4 and 6 looked decisive — same subset, same machine, control passing 2/2 and PR 2 +failing 2/2, on a diff verified to be exactly PR 2's 8 files (the Gitea unpin `27bf27c` is an +ancestor of **both** commits, so it cannot explain anything). **That conclusion was wrong.** +Run 7 passed with PR 2 present, and runs 8–9 replaced the coin-flip with a rate measured over +~85 resolutions per arm: **8.0% loss with PR 2, 9.6% without**. Per-resource on baseline, +configmaps alone lose **10.0%**. The spec is ~10%-flaky on its own; three samples per arm +could never separate that from an effect, and I treated a 2-vs-0 split as proof. + +**The control was verified to be a real control.** `git diff --stat 4f37759 10a530a` is +exactly the 8 `internal/watch` files of PR 2 and nothing else — in particular the Gitea unpin +(12.5.1 → 12.7.0) and dependency refresh in `27bf27c` are ancestors of **both** commits, so +they cannot explain the difference. This was checked precisely because a stacked branch can +easily lag main by unrelated commits; here it does not. + +Run 5 pulls the other way: main's `Release` build runs `ci.yml` (whose first job everything +else `needs`), and **every** e2e leg passed — including `full-manager`, which contains this +exact spec. But CI shards across six legs, so each runs at materially lower contention than +one 56-spec local run at `--procs=4`. At the time this was read as "rules out a deterministic +regression but not a latent race" — correct as far as it went, and in hindsight CI was simply +sampling the same ~10% lottery at lower load. + +--- + +## 5. The failing spec, explained + +`Manager WatchRule ConfigMap and Secret >> should create Git commit when ConfigMap is added +via WatchRule` +([watchrule_configmap_secret_e2e_test.go:446](../../../test/e2e/watchrule_configmap_secret_e2e_test.go#L446)). + +It proves a human's identity survives from `kubectl` into a Git commit author: + +1. Create a `GitTarget` + a `WatchRule` selecting ConfigMaps in its namespace; wait for Ready. +2. Create a ConfigMap **impersonating a user**: `kubectl --as=jane@acme.com`. The + impersonation is the point of the spec. +3. Poll a checkout of the Gitea repo until the YAML exists, the commit message carries + `[CREATE]` and `v1/configmaps/test-configmap`, and `git log -1 --pretty=%an` is the + impersonated user. + +**The two modes.** In **configured-author mode** (attribution off, or no Redis) every commit +is authored by the configured committer. In **attribution mode** (the default here) the live +watch event is matched against an audit-webhook fact carrying the acting user; with no match +it falls back to `DefaultCommitterName`. + +**The ambiguity that cost hours.** `DefaultCommitterName` is `"GitOps Reverser"` +([internal/git/types.go:22](../../../internal/git/types.go#L22)) — *also* the configured-author +identity. Observing `GitOps Reverser` means **either** "configured-author mode, working" **or** +"attribution mode, attribution lost". The string alone does not say which. + +**The failure:** expected `jane@acme.com`, got `GitOps Reverser`. + +--- + +## 6. What was ruled out, and on what evidence + +Each of these was a live hypothesis. None survived. + +### 6.1 A PR 1 regression — ruled out +Re-ran the commit-request specs in isolation: 4/4 passed (§2). + +### 6.2 The `playground` Gitea failure — ruled out (self-inflicted) +`HTTP 422: Key title has been used`. A previous targeted run had registered the SSH key and +never ran playground's cleanup. Environmental contamination from my own partial run, not a +code defect. Invalidated run 1 entirely. + +### 6.3 A mid-run controller restart disrupting the spec — ruled out +A spec does `rollout restart` the controller (explaining a 3-minute-old pod in a 10-minute +suite), but [restart_reconcile_e2e_test.go:34](../../../test/e2e/restart_reconcile_e2e_test.go#L34) +is decorated `Serial`, so Ginkgo runs it after all parallel specs. It cannot have overlapped. + +### 6.4 The test asserting the wrong branch — ruled out +The probe *could* misreport (§7.2). It did not here: the Deployment carries `--redis-addr=…` +with no `--author-attribution=false`, and the controller emits no `configured-author mode:` +banner. So the deployment is genuinely in attribution mode, the probe genuinely returned the +right answer, and the spec was right to demand `jane@acme.com`. + +**This also rules out the "after 30 minutes it stops detecting the mode" theory** for *this* +failure. Aging pushes the probe toward `false` = "expect `jane@acme.com`" — which is what it +already correctly returned. Aging can only cause a *false failure* in the opposite +configuration: a controller truly in configured-author mode, up >30 minutes. Real bug, wrong +suspect. + +### 6.4a The control being contaminated by unrelated main commits — ruled out +See §4: `4f37759` and `10a530a` differ by exactly PR 2's 8 files. Checked explicitly. + +### 6.5 "This suite is just flaky here" — RULED OUT +Argued three times on a code-reading (§6.6), then declared dead when the control came back +2/2 versus 0/2 — and then **reinstated by measurement**. Aggregate attribution-loss rates are +8.0% (PR 2) versus 9.6% (baseline), and configmaps alone lose 10.0% on baseline, so the spec +is ~10%-flaky per run on its own and the fail/pass split was noise. The spec *is* flaky here; +what was wrong was calling it flaky without evidence, and then calling it non-flaky on +evidence too thin to carry the claim. See [§0](pr2-e2e-attribution-investigation.md#0-resolution). + +### 6.6 "PR 2 is a strict no-op in this configuration" — code-true, but insufficient +The claim: the change alters nothing unless a single `WatchedType` holds **both** the `""` +key and a named key. +- only named keys → old returned the sorted names, new returns the same; +- only `""` → old returned `nil` and the caller synthesised `targetWatchKey{GVR, ""}`; new + returns `[""]` and builds the identical key. + +The failing spec has one `GitTarget` with a single namespaced `WatchRule` and no co-resident +`ClusterWatchRule`, so it should never reach the changed branch. **The code reading still +looks correct — and the experiment contradicts it.** The experiment outranks the reading. + +**The search space is small.** `WatchScopes()` has exactly ONE production consumer: +`targetWatchSpecs` ([target_watch.go:218](../../../internal/watch/target_watch.go#L218)). +Everything else PR 2 touched — `snapshotGVRsFromTable`, `resolveSnapshotGVRs`, +`resolveSnapshotGVRForType`, `snapshotGVRScopes` — has **no production callers at all** +(verified by grep on main, excluding tests). So the entire runtime effect of PR 2 reduces to +whatever `targetWatchSpecs` now returns. + +**One genuine behavioural difference found so far**, and it is not obviously the culprit: +when a `WatchedType` has an **empty** `NamespaceOps` map, the old code fell into its +`len(namespaces) == 0` branch and synthesised a **cluster-wide** watch key +`targetWatchKey{GVR, ""}`; the new code iterates an empty scope list and creates **no key at +all**. Production should never produce an empty map (`buildWatchedTypeTable` always records +at least one namespace per selection), and the failure mode does not match — a missing watch +would mean the ConfigMap never reaches Git, whereas here the file and commit landed and only +the *author* was wrong. Recorded because it is a real, untested divergence. + +Next step is instrumentation rather than more reading: capture the controller's +`watch-first target watch set reconciled` lines (they log `watchCount`) from a failing main +run and a passing baseline run and diff the declared watch sets for the spec's GitTarget. If +they are identical, the cause is downstream of the watch set and the reading is right; if +they differ, the difference names the bug. + +--- + +## 7. Infrastructure defects found (all real, all fixed or filed) + +### 7.1 Stacked PRs ran no CI at all — fixed +`ci.yml` triggered on `pull_request: branches: [main]`. A stacked PR based on the previous +PR's branch never matched. **#255 merged having run only CodeRabbit and GitGuardian — no +lint, no unit tests, no e2e.** Retargeting does not rescue it: when the base merges, GitHub +retargets and fires `pull_request` with action `edited`, which is not in the default trigger +set, so nothing re-runs. + +Fixed by dropping the `branches` filter (PR #257). Without it, four of this workstream's five +PRs would merge unvalidated. **Caveat: the fix cannot validate itself** — #257 targets `main`, +so it runs CI regardless. First real proof is the next PR against a non-`main` base. + +### 7.2 The author-mode probe was a log grep that fails open — fixed +`configuredAuthorModeEnabled` ran `kubectl logs … --since=30m`, grepped for a startup banner, +and returned `false` on any error. Three ways to misreport: banner ages out after 30 minutes; +`kubectl logs deployment/…` errors or picks the wrong pod mid-rollout; **the quickstart specs +deliberately redeploy the controller in configured-author mode, leaving that banner in the +shared log window for later specs to misread.** Given §5's ambiguity, a wrong answer silently +swaps in a weaker assertion that *passes*. + +Fixed in #257: read the Deployment's container args, fail loudly if unreadable, with a pure +`configuredAuthorModeFromArgs` under unit test pinning the defaults (`cmd/main.go` defaults +**both** `--author-attribution` true and `--redis-addr` `valkey:6379`, so an absent flag means +attribution mode). The change can only make the assertion **stricter**. + +That quickstart contamination path is also a plausible reason this spec looks healthier in CI +than locally: CI shards `quickstart-install` and `full-manager` into separate legs; locally +they share one cluster and one log window. + +### 7.3 `task clean-cluster` is cwd-sensitive — filed, not fixed +It does `rm -rf .stamps/cluster/` on a **relative** path. Run from `test/e2e` it deletes +nothing real while k3d still removes the cluster, leaving a stale `ready` stamp. `prepare-e2e` +then skips cluster creation and everything fails with `No nodes found for given cluster`. Cost +one wasted cycle. **Always run task targets from the repo root.** + +### 7.4 Piping a task to `tail` masks its exit status — process fix +`task prepare-e2e | tail -15 && task test-e2e` takes `tail`'s status, so a failed prepare +reported success and the suite ran against a nonexistent cluster, producing a `BeforeSuite` +failure that then had to be unwound. Use `set -o pipefail` or capture exit codes separately. + +--- + +## 8. Process lessons (mine) + +- **A structural argument is not evidence about a timing-sensitive integration failure.** The + §6.6 no-op reading was used to wave off three consecutive failures. The control that tested + it cost ~6 minutes and should have been run after the *first* failure, not the third. +- **Two variables changed at once.** Run 2 (PR 2, full suite) vs run 3 (baseline, subset) + differed in both code and load, so run 3 alone could not convict or acquit. Only runs 3+4 + isolated it. +- **I contaminated my own evidence twice** — the Gitea key (§6.2), then the cwd/pipefail pair + (§7.3, §7.4) which produced two runs that tested nothing. Of six runs, one was void and two + were empty. +- **Stated confidence outran the evidence** in three successive summaries. The maintainer's + "I don't remember flakes around this test" was worth more than any of them. + +--- + +## 9. Open questions — resolved + +1. **Did the control hold at n=2?** Yes (baseline 2/2 passed) — and it did not matter. The + aggregate loss rate settled it instead: 8.0% with PR 2 versus 9.6% without. +2. **How could §6.6 be code-true and the experiment still fail?** Both were right. The code + reading was correct (PR 2 is a no-op for that spec's GitTarget); the experiment was + sampling a ~10%-lossy process three times per arm and I read the split as signal. + Instrumentation did find real case-C fan-out + (`unsupported-folder-dest` gets two `configmaps` streams), but on a different GitTarget in + a different namespace, and audit volume did not correlate with failure — the failing run + had the *lowest* volume. +3. **Is attribution reliable under 4-proc load?** **No, and that is the real defect.** ~10% of + ConfigMap resolutions lose the actor, independent of PR 2. +4. **Does #257 go green?** Open at time of writing. + +## 10. State at end of session + +- **Merged to main:** #254 (PR 1), #255 (PR 2). PR 2 is correct and stays. +- **Open:** #257 on `feat/watchrule-src-ns-pr3-clusterwatchrule-target-admission` — the CI + trigger fix, the author-mode probe fix, per-stream declare logging, + `hack/attribution-diagnostics.sh`, and these findings. **PR 3's actual ClusterWatchRule + admission fix is not written yet.** +- **The real open defect:** attribution silently loses ~10% of actors under parallel load. + The 3s grace window is too tight against a ~1s audit batching delay. Fix options in + [§0](pr2-e2e-attribution-investigation.md#0-resolution). This is a correctness bug in the + product's core promise — naming who made a change — and it is worth more than the PR 2 + question that surfaced it. +- **PR 3 groundwork:** design verified against the tree; every load-bearing claim holds. One + gap found: the design says only a ClusterProvider → ClusterWatchRules mapper is missing, but + admission is evaluated against **namespace labels**, and the GitTarget controller carries a + matching `namespaceToGitTargets` watch. The CWR controller watches only GitTarget and + GitProvider, so revocation by *relabelling a namespace* would converge for GitTargets and + never for ClusterWatchRules. **PR 3 needs both mappers plus a revocation test for the label + path.** +- **Also fixed by a parallel agent, and worth knowing:** `test/e2e/Taskfile.yml` is now + `Taskfile-e2e.yml` so that `task` invoked from inside `test/e2e/` cannot stop its upward + search there and run with the wrong working directory — the structural cause of the + `clean-cluster` incident in §7.3. A `PreToolUse` hook now refuses unguarded `task … | …` + pipelines (§7.4). diff --git a/docs/design/watchrule-source-namespace/watch-construction-overview.md b/docs/design/watchrule-source-namespace/watch-construction-overview.md new file mode 100644 index 00000000..fc410f0f --- /dev/null +++ b/docs/design/watchrule-source-namespace/watch-construction-overview.md @@ -0,0 +1,206 @@ +# How watches are built — an overview + +> Written 2026-07-20 while investigating the PR 2 attribution regression +> ([session log](pr2-session-log-and-findings.md)). Describes the tree as it is **today**, +> verified against the code, not as designed. Every claim here was checked by reading the +> named function. +> +> Short answers to the three questions this page exists for: +> **(1)** Yes — watches follow from the GitTarget reconcile, but the *content* is projected +> from the rule set, not from the GitTarget. **(2)** No reuse across GitTargets: every +> GitTarget opens its own watches, even for an identical (GVR, namespace). **(3)** One watch +> per `(GitTarget, GVR, namespace-scope)` triple — so yes, combinations. + +## 1. The build pipeline + +Rules are compiled into a store; the store is projected into a per-GitTarget table; the table +is projected into a set of watch keys; each key becomes one goroutine holding one API watch. + +```mermaid +flowchart TD + subgraph CRs["Custom resources"] + WR["WatchRule
targetRef is namespace-local"] + CWR["ClusterWatchRule
targetRef names any namespace"] + GT["GitTarget"] + end + + subgraph Compile["Rule compilation"] + WRC["WatchRuleReconciler"] + CWRC["ClusterWatchRuleReconciler"] + BS["bootstrap.go
seeds the store on restart"] + RS[("RuleStore")] + end + + subgraph Project["Projection — watched_type_resolver.go"] + REG[("typeset.Registry
followable set, per source cluster")] + RWT["resolveWatchedTypeTables()"] + BWT["buildWatchedTypeTable()
fold selections by GVR"] + TBL[("WatchedTypeTable
per GitTarget")] + end + + subgraph Declare["Declaration — target_watch.go"] + DECL["DeclareForGitTarget()
from the GitTarget reconcile"] + EGW["EnsureGitTargetWatches()"] + RGW["replaceGitTargetWatches()"] + TWS["targetWatchSpecs()
the only consumer of WatchScopes"] + RUN["runTargetWatch() — one goroutine per key"] + API["dynamic client Watch()
one raw watch each"] + end + + WR --> WRC --> RS + CWR --> CWRC --> RS + BS --> RS + GT --> DECL + + RS --> RWT + REG --> RWT + RWT --> BWT --> TBL + TBL --> RGW + + DECL --> EGW --> RGW --> TWS --> RUN --> API +``` + +**What triggers a rebuild.** `refreshWatchedTypeTables()` is gated on three fingerprints, so +the common no-change reconcile is a cheap compare rather than a rescan: the rule set +(`rulesFingerprint`), every active cluster's discovery revision +(`activeRegistriesFingerprint`), and the GitTarget→source-cluster mapping +(`clusterMappingFingerprint`). + +**Your intuition is right, with one correction.** The watch set *is* driven by the GitTarget +reconcile — `DeclareForGitTarget` is what creates the `targetWatches` entry. But the GitTarget +contributes only its *identity and destination*; **what** to watch comes entirely from the +rules that point at it. A GitTarget with no rules gets an empty table and no streams. That +asymmetry matters: the rule controllers can change a resident table, but +`refreshRunningTargetWatches` only refreshes GitTargets that are **already running**, so a +rule alone cannot bootstrap a stream. (This is the incidental protection that +[PR 3](pr3-clusterwatchrule-target-admission.md) exists to make deliberate.) + +## 2. The namespace scope of a selection + +The one rule that determines everything downstream: + +| Rule kind | Namespace recorded in the selection | +|---|---| +| `WatchRule` | the rule's own source namespace (`rule.Source.Namespace`) | +| `ClusterWatchRule` | **always `""`** — regardless of the rule's `scope:` | + +`""` means *cluster-wide* — a watch with no namespace, i.e. all namespaces. A cluster-scoped +type (CRDs, Namespaces) can only ever be `""`, because `collectWatchRuleSelections` hardcodes +`ResourceScopeNamespaced` and so a WatchRule can never match a cluster-scoped record. + +`buildWatchedTypeTable` then folds selections into one `WatchedType` per GVR, carrying +`NamespaceOps: map[namespace]OperationSet` — the union of operation filters per namespace. + +## 3. Cardinality — what actually gets opened + +**One watch per `(GitTarget, GVR, namespace-scope)`.** Nothing is shared. + +```mermaid +flowchart LR + subgraph T1["GitTarget A"] + A1["configmaps @ team-a"] + A2["secrets @ team-a"] + end + subgraph T2["GitTarget B"] + B1["configmaps @ team-a
same GVR+ns as A1 —
still a separate watch"] + B2["crds @ *cluster-wide*"] + end + subgraph T3["GitTarget C — case C"] + C1["configmaps @ *cluster-wide*"] + C2["configmaps @ team-a
same object delivered twice"] + end + + A1 --> W1["watch #1"] + A2 --> W2["watch #2"] + B1 --> W3["watch #3"] + B2 --> W4["watch #4"] + C1 --> W5["watch #5"] + C2 --> W6["watch #6"] +``` + +So the total is a genuine product: + +```text +streams = Σ over GitTargets Σ over watched GVRs |namespace scopes for that GVR| +``` + +- **Across GitTargets: no reuse.** `m.targetWatches` is keyed by `gitDest.Key()`, and + `openTargetWatch` calls the dynamic client's `Watch()` directly — a raw watch, not a shared + informer cache. Two GitTargets mirroring the same ConfigMaps in the same namespace open two + independent watches against the API server. +- **Across reconciles: yes, reuse.** `replaceGitTargetWatches` compares the newly computed + spec map against the running one (`equalTargetWatchSpecs`) and returns early when unchanged, + so a no-op reconcile does not churn streams. The reuse that exists is *temporal*, not + *cross-target*. +- **Per namespace: one each.** A GVR selected in `team-a` and `team-b` is two watches, because + a namespaced watch takes exactly one namespace. + +## 4. What PR 2 changed + +Only the third bullet. Before PR 2, the presence of the `""` key made `SnapshotNamespaces()` +return `nil`, and `nil` meant *all namespaces* — so a co-resident named namespace was +**silently discarded**, along with its operation filter. + +```mermaid +flowchart TB + subgraph Input["One WatchedType, one GitTarget, one GVR"] + N["NamespaceOps
"" → UPDATE (ClusterWatchRule)
team-a → CREATE (WatchRule)"] + end + + N --> OLD["Before PR 2
SnapshotNamespaces() → nil
⇒ ONE stream
configmaps@*cluster-wide*=UPDATE

team-a's CREATE filter discarded;
rule silently widened to all namespaces"] + N --> NEW["After PR 2
WatchScopes() → ["", team-a]
⇒ TWO streams
configmaps@*cluster-wide*=UPDATE
configmaps@team-a=CREATE

each scope keeps its own filter"] + + NEW --> DUP["⚠ objects in team-a are now
delivered on BOTH streams"] +``` + +For every other shape the output is byte-identical: + +| `NamespaceOps` | Before | After | Same? | +|---|---|---|---| +| `{team-a}` | `configmaps@team-a` | `configmaps@team-a` | ✅ | +| `{""}` | `configmaps@*cluster-wide*` | `configmaps@*cluster-wide*` | ✅ | +| `{team-a, team-b}` | both | both | ✅ | +| `{"", team-a}` | cluster-wide **only** | **both** | ❌ the fix | +| `{}` (empty) | cluster-wide | **none** | ❌ unreachable in practice | + +## 5. Where the attribution suspicion sits + +A commit's author is not carried by the watch event. The event says *what* changed; a separate +audit-webhook fact says *who* did it, and the two are matched. + +```mermaid +sequenceDiagram + actor U as kubectl --as=jane@acme.com + participant API as kube-apiserver + participant AUD as audit webhook → Redis + participant W as target watch stream + participant M as match / write path + participant G as Git + + U->>API: create ConfigMap + API-->>AUD: audit event (user = jane@acme.com) + API-->>W: watch event (object + rv) + W->>M: live event + AUD->>M: audit fact + M->>M: match event ↔ fact + alt fact matched in window + M->>G: commit authored jane@acme.com + else no match + M->>G: commit authored "GitOps Reverser"
(DefaultCommitterName fallback) + end +``` + +The observed failure is the lower branch: the file and commit landed, only the author was +wrong. Note that `"GitOps Reverser"` is **also** the configured-author identity, which is why +the failure was initially misread — see +[the investigation](pr2-e2e-attribution-investigation.md). + +**Why the diagram above makes case C interesting.** If one object is delivered on two streams, +the write path sees it twice, but there is only ever **one** audit fact. Whichever delivery +consumes or races the match, the other has none — and a second, unattributed write of the same +object is a plausible way to land a commit authored by the fallback. This is the hypothesis +the instrumented run is testing: the declare log now names every stream, so a GVR appearing +twice for one GitTarget is case C caught in the act. + +It is still a hypothesis. The controlled experiment says PR 2 causes the failure; it does not +yet say by which mechanism. diff --git a/hack/attribution-diagnostics.sh b/hack/attribution-diagnostics.sh new file mode 100755 index 00000000..6135ad40 --- /dev/null +++ b/hack/attribution-diagnostics.sh @@ -0,0 +1,105 @@ +#!/usr/bin/env bash +# Post-run diagnostics for the author-attribution path. +# +# Attribution fails SILENTLY: when no audit fact matches within the grace window +# (watch.DefaultAttributionGraceWindow, 3s), the commit is authored by the configured +# committer instead of the real actor. Because git.DefaultCommitterName is ALSO the +# configured-author identity, the resulting commit is indistinguishable from a correct +# configured-author commit by inspection alone. This script makes the difference visible. +# +# It answers the one question that splits the diagnosis in two: +# +# Is the fact ABSENT from Redis (the apiserver never delivered it — batching, queue +# overflow, a routing error), or is it PRESENT but was written too late for the 3s +# grace window (the resolver gave up first)? +# +# Fact keys carry a 10-minute TTL, so the remaining TTL back-computes when each fact +# landed: age = ttl_total - ttl_remaining. Comparing that against the fact's own +# stageTimestamp gives the apiserver→Redis delivery lag directly. +# +# Usage: hack/attribution-diagnostics.sh [name-substring] +# Run it IMMEDIATELY after an e2e run — facts expire after 10 minutes. +set -euo pipefail + +CTX="${CTX:-k3d-gitops-reverser-test-e2e}" +VALKEY_NS="${VALKEY_NS:-valkey-e2e}" +PROM_NS="${PROM_NS:-prometheus-operator}" +FACT_TTL_SECONDS="${FACT_TTL_SECONDS:-600}" +FILTER="${1:-}" + +kc() { kubectl --context "${CTX}" "$@"; } + +echo "== attribution resolution outcomes (Prometheus) ==" +echo " absent = fell back to the configured committer; the actor's name was LOST." +prom_pod_port=9097 +kc -n "${PROM_NS}" port-forward svc/prometheus-operated ${prom_pod_port}:9090 >/dev/null 2>&1 & +pf=$! +trap 'kill ${pf} 2>/dev/null || true' EXIT +sleep 5 +curl -sG "http://localhost:${prom_pod_port}/api/v1/query" \ + --data-urlencode 'query=sum by (result) (gitopsreverser_attribution_resolutions_total)' | + python3 -c ' +import json,sys +d=json.load(sys.stdin) +rows=[(r["metric"].get("result","?"), float(r["value"][1])) for r in d.get("data",{}).get("result",[])] +total=sum(v for _,v in rows) or 1 +for k,v in sorted(rows, key=lambda x:-x[1]): + print(f" {k:32s} {v:8.0f} ({100*v/total:5.1f}%)") +print(f" {'TOTAL':32s} {total:8.0f}") +absent=dict(rows).get("absent",0) +if absent: + print(f"\n >> {absent:.0f} resolution(s) lost the actor. Each is a commit authored by the") + print( " >> committer instead of the human/SA that caused the change.") +' || echo " (query failed)" + +echo +echo "== apiserver-side audit delivery ==" +curl -sG "http://localhost:${prom_pod_port}/api/v1/query" \ + --data-urlencode 'query=sum(apiserver_audit_requests_rejected_total) or vector(0)' | + python3 -c 'import json,sys; d=json.load(sys.stdin); r=d.get("data",{}).get("result",[]); print(" apiserver_audit_requests_rejected_total =", r[0]["value"][1] if r else "n/a (not scraped)")' \ + || echo " (query failed)" +curl -sG "http://localhost:${prom_pod_port}/api/v1/query" \ + --data-urlencode 'query=sum(gitopsreverser_audit_eventlist_events_total) or vector(0)' | + python3 -c 'import json,sys; d=json.load(sys.stdin); r=d.get("data",{}).get("result",[]); print(" audit events INGESTED by us =", r[0]["value"][1] if r else "0")' \ + || echo " (query failed)" +kill ${pf} 2>/dev/null || true + +echo +echo "== attribution facts present in Valkey ==" +pw="$(kc -n "${VALKEY_NS}" get secret valkey-auth -o jsonpath='{.data.*}' | head -1 | base64 -d)" +pod="$(kc -n "${VALKEY_NS}" get pods -o jsonpath='{.items[0].metadata.name}')" +vc() { kc -n "${VALKEY_NS}" exec "${pod}" -c valkey -- valkey-cli -a "${pw}" --no-auth-warning "$@" 2>/dev/null; } + +keys="$(vc --scan --pattern '*author:v1:audit:*' | { [ -n "${FILTER}" ] && grep -F "${FILTER}" || cat; } | head -40)" +if [ -z "${keys}" ]; then + echo " no facts matching ${FILTER:-}" + echo " >> If a commit lost its author AND no fact exists, the apiserver never delivered it:" + echo " >> check audit-webhook-batch-max-wait/-size and the /audit-webhook/ route." + exit 0 +fi + +echo " (fact age is back-computed from the remaining TTL against a ${FACT_TTL_SECONDS}s TTL)" +for k in ${keys}; do + val="$(vc GET "${k}")" + ttl="$(vc TTL "${k}")" + python3 - "${k}" "${val}" "${ttl}" "${FACT_TTL_SECONDS}" <<'PY' +import json,sys +key,val,ttl,total = sys.argv[1],sys.argv[2],sys.argv[3],int(sys.argv[4]) +try: + f=json.loads(val) +except Exception: + print(f" {key} "); raise SystemExit +try: + age=total-int(ttl) +except ValueError: + age="?" +ns=f.get("namespace",""); name=f.get("name","") +print(f" {ns}/{name} verb={f.get('verb','?')} author={f.get('author','?')}") +print(f" stageTimestamp={f.get('stageTimestamp','?')} written_to_redis≈{age}s ago") +PY +done + +echo +echo " Reading this: a fact PRESENT here for an object whose commit lost its author means" +echo " the fact arrived, just not within the 3s grace — the grace window (or the audit" +echo " batching delay ahead of it) is the problem, not delivery." diff --git a/internal/watch/target_watch.go b/internal/watch/target_watch.go index 21965cfc..60014c6c 100644 --- a/internal/watch/target_watch.go +++ b/internal/watch/target_watch.go @@ -152,10 +152,29 @@ func (m *Manager) replaceGitTargetWatches( ops := table.operationsFor(watchKey) go m.runTargetWatch(childCtx, log, table.GitDest, watchKey, ops) } - log.Info("watch-first target watch set reconciled", "watchCount", len(keys)) + // Name every declared stream, not just the count. A GVR appearing twice — once + // cluster-wide ("") and once under a named namespace — means the same object is + // delivered on two streams, which is legitimate scoping but doubles the events for + // objects in that namespace. That is invisible in a bare count. + log.Info("watch-first target watch set reconciled", + "watchCount", len(keys), "streams", describeWatchKeys(keys, specs)) return nil } +// describeWatchKeys renders the declared streams as "@=" +// so a declare log names exactly what is being watched and under which operation filter. +func describeWatchKeys(keys []targetWatchKey, specs map[targetWatchKey]string) string { + parts := make([]string, 0, len(keys)) + for _, key := range keys { + scope := key.Namespace + if scope == "" { + scope = "*cluster-wide*" + } + parts = append(parts, fmt.Sprintf("%s@%s=%s", key.GVR.String(), scope, specs[key])) + } + return strings.Join(parts, " | ") +} + func (m *Manager) prepareTargetWatchSetReplacementLocked( key string, specs map[targetWatchKey]string, From c55c33b81408826ef104c7a953c2f2b9682329c8 Mon Sep 17 00:00:00 2001 From: Simon Koudijs Date: Mon, 20 Jul 2026 12:12:19 +0000 Subject: [PATCH 05/17] docs(debug): consolidate the attribution investigation into docs/debug/ MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three sprawling working documents in design/watchrule-source-namespace/ replaced by one condensed page, plus the watch-construction reference moved alongside it. The originals grew by accretion during the investigation and carried several conclusions that were later overturned; keeping them would mean keeping three versions of the same story with no signal about which is current. docs/debug/ is a new lifecycle folder, registered in INDEX.md: "this is wrong now" — open investigations into live defects, retired to finished/ once fixed. It sits between design/ (still deciding) and finished/ (already happened), which had no home for a known-broken behaviour someone must not re-investigate from scratch. attribution-loss.md states the defect, how far the 7-10% figure can be trusted and why, and then two tables that are the point of the exercise: what was RULED OUT with the evidence that killed each hypothesis (PR 2, late fact arrival, watermark gating, key shape, webhook routing, audit policy, replay events, load, "just flaky"), and what was FIXED along the way (CI trigger, author-mode probe, Taskfile rename, task-pipe hook, per-stream logging, the after-suite report, the histogram le-label bug). The ruled-out table is the part worth keeping. Every row cost a cycle, and several were things I asserted confidently before measuring. Also corrects the grace-window comment in config/deployment.yaml: raising 3s to 10s left the loss rate unchanged, so it buys headroom and observability, NOT a fix. And fixes the after-suite report's bucket queries — Prometheus renders le as "1.0"/"3.0"/"10.0", so %g matched nothing above 0.5 and produced a plausible but entirely wrong distribution. Co-Authored-By: Claude Opus 4.8 (1M context) --- config/deployment.yaml | 13 + docs/INDEX.md | 1 + docs/debug/README.md | 19 ++ docs/debug/attribution-loss.md | 101 ++++++ .../watch-construction.md} | 7 +- .../pr2-e2e-attribution-investigation.md | 251 -------------- .../pr2-session-log-and-findings.md | 322 ------------------ test/e2e/e2e_suite_test.go | 105 ++++++ 8 files changed, 242 insertions(+), 577 deletions(-) create mode 100644 docs/debug/README.md create mode 100644 docs/debug/attribution-loss.md rename docs/{design/watchrule-source-namespace/watch-construction-overview.md => debug/watch-construction.md} (96%) delete mode 100644 docs/design/watchrule-source-namespace/pr2-e2e-attribution-investigation.md delete mode 100644 docs/design/watchrule-source-namespace/pr2-session-log-and-findings.md diff --git a/config/deployment.yaml b/config/deployment.yaml index af5667c2..c2ab3ca4 100644 --- a/config/deployment.yaml +++ b/config/deployment.yaml @@ -34,6 +34,19 @@ spec: - --redis-addr=valkey.valkey-e2e.svc.cluster.local:6379 # The e2e Valkey serves plain TCP, so opt out of the now-default Redis TLS. - --redis-insecure + # Attribution waits this long for a matching audit fact before committing as the + # configured committer, which silently loses the actor's name. Widened from the 3s + # default to buy headroom over the apiserver's --audit-webhook-batch-max-wait=1s + # (test/e2e/cluster/start-cluster.sh): measured fact delivery is ~0.8-1.1s even on an + # idle cluster, so 3s leaves only ~2s of slack for a 4-proc run. + # + # Honest scope: this is NOT a fix for the ~7% of resolutions that find no fact. Raising + # 3s -> 10s left that rate unchanged, which means those facts never arrive rather than + # arriving late; only ~1 resolution per run lands in the 3-10s band. Kept because the + # headroom is cheap and makes the tail observable, not because it repairs anything. The + # after-suite report separates resolved-wait from absent-wait so the difference stays + # visible. + - --author-attribution-grace=10s # Dev/e2e only: the in-cluster Gitea SSH host key is not pinned, so permit SSH # when no known_hosts source exists. Never set this in production. - --insecure-allow-missing-known-hosts diff --git a/docs/INDEX.md b/docs/INDEX.md index 877eb992..90febd8d 100644 --- a/docs/INDEX.md +++ b/docs/INDEX.md @@ -14,6 +14,7 @@ those. If a document is not on this page, it is either a user guide (see | [`design/`](design/) | **We are still deciding.** Open questions, proposals, unbuilt work. | yes — as intent, not as shipped behaviour | | [`facts/`](facts/) | Durable reference: how Kubernetes behaves, and what we discovered about it. | yes, as reference | | [`finished/`](finished/) | **This happened.** Shipped plans, closed investigations. Kept for context. | **no** | +| [`debug/`](debug/) | **This is wrong now.** Open investigations into live defects, with what has been ruled out. Retired to `finished/` once fixed. | **yes** — as known-broken behaviour | The rule that was missing before: `design/` used to hold shipped work and `finished/` used to hold live contracts. If you are adding a document, pick the diff --git a/docs/debug/README.md b/docs/debug/README.md new file mode 100644 index 00000000..f0d949c4 --- /dev/null +++ b/docs/debug/README.md @@ -0,0 +1,19 @@ +# `debug/` — live investigations + +Working notes for defects that are **understood but not fixed**, or **not yet understood**. +Unlike [`finished/`](../finished/), these are open; unlike [`design/`](../design/), they +describe something that is wrong now rather than something being decided. + +Retire a page from here by fixing the defect and moving the write-up to `finished/`. + +| Page | What it is | Status | +|---|---|---| +| [attribution-loss.md](attribution-loss.md) | ~7–10% of live commits are authored by the configured committer instead of the real actor, silently | **open** | +| [watch-construction.md](watch-construction.md) | Reference: how watches are built, and at what cardinality | reference | + +## One-line state + +`WatchRule`/`ClusterWatchRule` stream scoping is correct as of +[PR 2](../design/watchrule-source-namespace/pr2-stream-scope-collapse.md). The open defect is +in **author attribution**, and it is older than that work — it was found while landing PR 2, +not caused by it. diff --git a/docs/debug/attribution-loss.md b/docs/debug/attribution-loss.md new file mode 100644 index 00000000..3aaeb90b --- /dev/null +++ b/docs/debug/attribution-loss.md @@ -0,0 +1,101 @@ +# Author attribution silently loses the actor + +> **Status: OPEN.** Found 2026-07-20 while landing +> [PR 2](../design/watchrule-source-namespace/pr2-stream-scope-collapse.md); **not caused by +> it**. Predates that work. + +## The symptom + +A commit that should be authored by the human or service account that made the change is +authored by the configured committer instead. **Silently**, and **invisibly by inspection**: +`git.DefaultCommitterName` is `"GitOps Reverser"` ([internal/git/types.go:22](../../internal/git/types.go#L22)), +which is *also* the configured-author identity — so a lost-actor commit is byte-identical to a +correct configured-author commit. The only way to see it is to count it. + +Naming the actor is the product's core promise, so this is a correctness defect, not cosmetics. + +## How it happens + +`ResolveAuthor` ([internal/watch/author_resolver.go](../../internal/watch/author_resolver.go)) +polls the Redis attribution index for a fact matching the live event's `(uid, rv)`, waiting up +to `--author-attribution-grace` (default 3s). No match → commit as the committer. + +## The number, and how far it can be trusted + +`absent / total` from `gitopsreverser_attribution_resolutions_total`: **7.1%, 8.0%, 9.6%** +across three runs. Read it as *roughly 1 in 12–14*, not as a precise figure. + +**What supports it.** Attribution runs **only on live events** — `attachAuthor` has one call +site, in `routeLiveTargetWatchEvent` +([target_watch.go:728](../../internal/watch/target_watch.go#L728)). `foldTargetReplayEvent` +never calls the resolver, so initial-replay of pre-existing objects contributes nothing to +either side of the ratio. + +**What weakens it.** + +- The queries use `sum(max_over_time(…[2h]))`, reconstructing totals across the deliberate + controller restart in the `restart-reconcile` spec rather than counting exactly. The same + population read instantaneously gave 92 where `max_over_time` gave 791. +- `absent` means *no fact matched*, which is not identical to *a human's name was lost*. A + live change with no audited actor belongs there legitimately. + +**To make it precise**, the resolver's metric needs a label separating *no fact was ever +produced* (correct) from *a fact should exist and we failed to match it* (the bug). Today both +collapse into `absent`, which is exactly why the number cannot yet carry much weight. + +## Ruled out + +| Hypothesis | Why not | +|---|---| +| **PR 2 (stream-scope collapse fix)** | Loss rate 8.0% with it vs 9.6% without; a PR 2 run passed the spec outright; all six CI e2e legs green. Its only runtime effect is `targetWatchSpecs`, whose output is byte-identical unless one `WatchedType` holds both `""` and a named key. | +| **Late fact arrival / grace too short** | Raising grace 3s → 10s left the rate **unchanged** (7.1%). Only ~1 resolution per run falls in the 3–10s band. Facts missing at 3s are still missing at 10s — they never arrive. | +| **Watermark / high-water gating** | `older_than_high_water = 0` in every run. | +| **Extra load from PR 2's duplicate streams** | Audit volume does not correlate: the *failing* run had the lowest (5667), a *passing* run the highest (9040). | +| **Wrong fact key shape** | Every ConfigMap uid in Valkey has **both** an exact-`rv` key and a `:last` key (46/46). Facts are indexed correctly. | +| **Audit webhook misrouted** | Posts to `/audit-webhook/default`, the correct **named** route ([webhook-config.yaml](../../test/e2e/cluster/audit/webhook-config.yaml)). ~92% of resolutions succeed, so delivery fundamentally works. | +| **Audit policy excluding the events** | Policy captures create/update/patch/delete/deletecollection broadly ([policy.yaml](../../test/e2e/cluster/audit/policy.yaml)). | +| **Initial-replay events with no actor** | Replay never invokes the resolver (see above). | +| **The e2e suite being "just flaky"** | Argued three times without evidence and wrong each time — though the spec *is* ~10%-flaky, because it samples this defect once per run. | +| **Controller restart mid-run disrupting the spec** | `restart_reconcile_e2e_test.go` is `Serial`; it cannot overlap parallel specs. | +| **`playground` Gitea key collision** | Self-inflicted contamination from a prior partial run, not a defect. | + +## Fixed along the way + +| Fix | Why it mattered | +|---|---| +| **`ci.yml` validates every PR** | `branches: [main]` meant stacked PRs ran **no** CI — #255 merged with no lint/test/e2e. Retargeting doesn't re-trigger (`edited` isn't a default trigger type). | +| **Author-mode probe reads the Deployment** | Was `kubectl logs --since=30m` grepping for a banner, failing **open**. Could silently swap in a weaker assertion that passes. | +| **`test/e2e/Taskfile.yml` → `Taskfile-e2e.yml`** | Running `task` from `test/e2e/` stopped Task's upward search there, so root-relative paths resolved nowhere. Caused `clean-cluster` to delete the cluster but not its stamps → next `prepare-e2e` skipped cluster creation. | +| **Hook blocking unguarded `task … \| …`** | A pipeline reports the *last* command's status, so a failed `prepare-e2e` looked successful and the suite ran against a nonexistent cluster. | +| **Per-stream declare logging** | `targetWatchSpecs` now names every stream (`@=`) instead of a bare count. Found the real two-stream fan-out on `unsupported-folder-dest`. | +| **e2e after-suite attribution report** | Prints outcome + wait distribution every run, split resolved vs absent. | +| **Histogram `le` label bug** | Buckets render as `1.0`/`3.0`/`10.0`, not `1`/`3`/`10`; `%g` matched nothing above `0.5` and produced a *plausible but wrong* distribution. | + +## Tools + +- [`hack/attribution-diagnostics.sh`](../../hack/attribution-diagnostics.sh) — separates *fact + never delivered* from *fact arrived too late*. Fact keys carry a 10-minute TTL, so the + remaining TTL back-computes when each landed. Run **immediately** after an e2e run. +- After-suite report — outcome counts plus cumulative wait buckets, split by result. +- Valkey: `valkey-cli -a "$(kubectl -n valkey-e2e get secret valkey-auth -o jsonpath='{.data.*}' | base64 -d)" --scan --pattern '*author:v1:audit:*'` + +## Measured facts worth keeping + +- Fact delivery lag, **idle** cluster: `835–1101 ms`, bimodal — the signature of the + apiserver's `--audit-webhook-batch-max-wait=1s`. +- Resolver mean wait by result: `weak` 0.001s · `exact_user` 0.533s · `absent` 2.268s (at the + 3s grace). +- Wait distribution (92-sample snapshot): `≤0.5s` 66 · `≤1s` 84 · `≤2s` 88 · `≤3s` 89 · + `≤10s` 90 · `+Inf` 92. + +## Next steps + +1. **Label the metric** to split *no fact existed* from *fact existed, match failed*. Until + then the rate cannot be acted on. +2. If the second bucket is non-empty, compare the fact's recorded `rv` against the live event's + `rv` for a failing object — the exact-key join is the likeliest place to lose a match. +3. **Decide the shipped default.** e2e now runs `--author-attribution-grace=10s`, but the + default is still 3s and the evidence says the grace is *not* the constraint. Raising it + ships latency for no proven benefit. +4. **Make the fallback loud.** Losing the actor should be observable at the commit, not only in + a counter nobody reads. diff --git a/docs/design/watchrule-source-namespace/watch-construction-overview.md b/docs/debug/watch-construction.md similarity index 96% rename from docs/design/watchrule-source-namespace/watch-construction-overview.md rename to docs/debug/watch-construction.md index fc410f0f..37e6a735 100644 --- a/docs/design/watchrule-source-namespace/watch-construction-overview.md +++ b/docs/debug/watch-construction.md @@ -1,7 +1,6 @@ # How watches are built — an overview -> Written 2026-07-20 while investigating the PR 2 attribution regression -> ([session log](pr2-session-log-and-findings.md)). Describes the tree as it is **today**, +> Written 2026-07-20 while investigating [attribution-loss.md](attribution-loss.md). Describes the tree as it is **today**, > verified against the code, not as designed. Every claim here was checked by reading the > named function. > @@ -73,7 +72,7 @@ rules that point at it. A GitTarget with no rules gets an empty table and no str asymmetry matters: the rule controllers can change a resident table, but `refreshRunningTargetWatches` only refreshes GitTargets that are **already running**, so a rule alone cannot bootstrap a stream. (This is the incidental protection that -[PR 3](pr3-clusterwatchrule-target-admission.md) exists to make deliberate.) +[PR 3](../design/watchrule-source-namespace/pr3-clusterwatchrule-target-admission.md) exists to make deliberate.) ## 2. The namespace scope of a selection @@ -193,7 +192,7 @@ sequenceDiagram The observed failure is the lower branch: the file and commit landed, only the author was wrong. Note that `"GitOps Reverser"` is **also** the configured-author identity, which is why the failure was initially misread — see -[the investigation](pr2-e2e-attribution-investigation.md). +[attribution-loss.md](attribution-loss.md). **Why the diagram above makes case C interesting.** If one object is delivered on two streams, the write path sees it twice, but there is only ever **one** audit fact. Whichever delivery diff --git a/docs/design/watchrule-source-namespace/pr2-e2e-attribution-investigation.md b/docs/design/watchrule-source-namespace/pr2-e2e-attribution-investigation.md deleted file mode 100644 index b8ce6efd..00000000 --- a/docs/design/watchrule-source-namespace/pr2-e2e-attribution-investigation.md +++ /dev/null @@ -1,251 +0,0 @@ -# Investigation — the author-attribution e2e failure seen while landing PR 2 - -> Working document, opened 2026-07-20 while landing -> [PR 2](pr2-stream-scope-collapse.md). **Status: unresolved.** It records what was -> measured, what each measurement does and does not establish, and where the reasoning -> went wrong, so the next person does not repeat the loop. -> -> **RESOLVED — see [§0](#0-resolution). PR 2 did not cause this.** The failure is a -> pre-existing ~10% attribution loss: the 3s grace window is too tight against an audit -> delivery lag that is ~1s even on an idle cluster. Everything below §0 is the working -> record, including three conclusions that were later overturned; it is kept because the -> reversals are the useful part. -> -> The headline: a commit that should have been authored `jane@acme.com` was authored -> `GitOps Reverser`. Attribution was genuinely lost — this is not a test asserting the -> wrong branch. - -## 0. Resolution - -**The mechanism, measured.** - -Attribution resolves a watch event against an audit fact, waiting up to -`DefaultAttributionGraceWindow` = **3 seconds** -([author_resolver.go](../../../internal/watch/author_resolver.go)). If no fact matches, the -commit is authored by the configured committer — silently, because -`git.DefaultCommitterName` ("GitOps Reverser") is *also* the configured-author identity. - -Measured fact-delivery lag on an **idle** cluster (12 samples, ConfigMap creates with -`--as=jane@acme.com`, polling Valkey for the fact key): - -~~~text -835 837 838 839 839 1092 1095 1096 1099 1099 1100 1101 (ms) -~~~ - -Bimodal at ~840ms and ~1100ms — the signature of the apiserver's -`--audit-webhook-batch-max-wait=1s` ([start-cluster.sh](../../../test/e2e/cluster/start-cluster.sh)). -So the 3s budget runs against a baseline lag of ~1s that is **dominated by a fixed batching -delay**, leaving roughly 2s of headroom for queueing. Under parallel load the tail crosses -3s and the actor is lost. - -Resolver wait times by result confirm it: - -| result | mean wait | -|---|---| -| `weak` | 0.001s | -| `exact_user` | 0.533s | -| `absent` | **2.268s** — polled the grace out and gave up | - -**Not a key-shape problem.** Every ConfigMap uid in Valkey has *both* an exact-`rv` key and a -`:last` key (46 of 46), so facts are indexed correctly. They simply are not there yet. - -**PR 2 is exonerated, by rate rather than by argument.** Attribution loss measured across -whole runs (~85 resolutions each, far more power than one spec's pass/fail): - -| Code | absent | total | loss | -|---|---|---|---| -| PR 2 present | 7 | 88 | **8.0%** | -| PR 2 absent (`4f37759`) | 8 | 83 | **9.6%** | - -Indistinguishable, with baseline marginally *worse*. Per-resource on baseline: -**configmaps = 5 absent / 49 = 10.0% loss** — the failing spec's own resource already loses -1 in 10 actors *without* PR 2. That makes the spec ~10%-flaky per run on its own, and the -2-fail-of-3 versus 0-fail-of-3 split that looked decisive is within the noise of that base -rate. - -**What to fix.** The grace window, not the stream scoping: - -1. Raise `DefaultAttributionGraceWindow` well above the batching delay (3s → ~10s). Cost: a - write with genuinely no audit fact waits the full grace before committing. -2. And/or lower `--audit-webhook-batch-max-wait` in the recommended audit config, which - attacks the dominant fixed term rather than padding around it. -3. Independently: **the fallback is silent.** `absent` is a counter nobody looks at. Losing - the actor is a correctness failure for a product whose promise is naming the actor; it - should be observable at the commit, not only in a metric. - -Reusable diagnostic: [`hack/attribution-diagnostics.sh`](../../../hack/attribution-diagnostics.sh) -separates "fact never delivered" from "fact arrived too late". - -## The failing spec, end to end - -`Manager WatchRule ConfigMap and Secret >> should create Git commit when ConfigMap is added -via WatchRule` -([watchrule_configmap_secret_e2e_test.go:446](../../../test/e2e/watchrule_configmap_secret_e2e_test.go#L446)), -failing at [:581](../../../test/e2e/watchrule_configmap_secret_e2e_test.go#L581): - -~~~text -Expected : GitOps Reverser -to contain substring : jane@acme.com -~~~ - -What the spec does: - -1. Creates a `GitTarget` (`watchrule-configmap-test-dest`) and a `WatchRule` selecting - ConfigMaps in the spec's own namespace, then waits for `Ready`. -2. Creates a ConfigMap **impersonating a user** — `kubectl --as=jane@acme.com` - ([:491](../../../test/e2e/watchrule_configmap_secret_e2e_test.go#L491)). The - impersonation is the whole point: the identity must survive into the Git commit. -3. `Eventually`-polls a local checkout of the Gitea repo until the ConfigMap's YAML exists, - the commit message carries `[CREATE]` and `v1/configmaps/test-configmap`, and the commit - **author** (`git log -1 --pretty=%an`) is the impersonated user. - -The author assertion is branched -([:576-579](../../../test/e2e/watchrule_configmap_secret_e2e_test.go#L576-L579)): - -~~~go -if configuredAuthorModeEnabled() { - g.Expect(author).To(ContainSubstring("GitOps Reverser")) -} else { - g.Expect(author).To(ContainSubstring("jane@acme.com")) -} -~~~ - -### How attribution is supposed to work - -The controller runs in one of two modes. - -- **configured-author mode** — no Redis / audit disabled. Every commit is authored by the - configured committer identity. `cmd/main.go` announces it at startup - ([main.go:274-277](../../../cmd/main.go#L274-L277)). -- **attribution mode** — the default here. The live watch event that produced the write is - matched against an audit-webhook fact carrying the acting user, and that user becomes the - commit author. When no fact matches, the commit falls back to - `DefaultCommitterName = "GitOps Reverser"` - ([internal/git/types.go:22](../../../internal/git/types.go#L22)). - -**`GitOps Reverser` is therefore ambiguous.** It is both the configured-author identity and -the fallback when attribution finds nothing. The observed string alone does not say which -happened — which is exactly why the first reading of this failure was wrong. - -### The mode probe is a log grep, and it fails open - -`configuredAuthorModeEnabled()` -([e2e_suite_test.go:124-130](../../../test/e2e/e2e_suite_test.go#L124-L130)) decides which -branch applies by shelling out to `kubectl logs deployment/gitops-reverser --since=30m` and -grepping for `configured-author mode:`. It returns `false` on **any** kubectl error. - -Two independent ways that misreports: - -- The controller has been up longer than 30 minutes, so the startup banner ages out of the - window. -- `kubectl logs deployment/...` errors or picks the wrong pod mid-rollout. The failing run's - event dump shows two ReplicaSets scaled up at once, so a rollout was in progress. - -Either makes the probe answer `false` — "attribution mode" — and demand `jane@acme.com` -regardless of how the controller is actually configured. It is also re-invoked on every -`Eventually` retry, which is why the failure timeline is full of repeated `kubectl logs` -calls. - -**This was checked and is NOT what happened.** The local deployment emits no -`configured-author mode:` banner at all, so it genuinely runs in attribution mode, the probe -correctly returned `false`, and the spec correctly demanded `jane@acme.com`. The commit fell -back to the default committer, so **attribution was really lost**. - -The probe is still a latent trap worth fixing independently — it can only ever fail in the -direction of demanding attribution that may not be configured. - -## What was measured - -| # | Code under test | Scope | Attribution spec | Notes | -|---|---|---|---|---| -| 1 | PR 2 branch | full suite, 56 specs | **FAILED** | also failed `playground` (see contamination below) | -| 2 | PR 2 branch | full suite, 56 specs | **FAILED** | clean cluster; 55 passed / 1 failed | -| 3 | `4f37759` — PR 2 code absent | `manager` subset, 43 specs | **passed** | 43 passed / 0 failed | -| 4 | `10a530a` — main, PR 2 present | `manager` subset | *running* | the controlled comparison | -| 5 | `10a530a` — main, PR 2 present | **CI**, all 6 sharded legs | **passed** | run 29734882468 | - -Run 5 is the strongest evidence so far that PR 2 did not break attribution. The merged -change was validated on clean GitHub infrastructure by `release.yml` (whose first job is -`uses: ./.github/workflows/ci.yml`), and **every** e2e leg passed — including -`E2E (full-manager)`, the leg that contains this exact spec, since -`Manager WatchRule ConfigMap and Secret` carries the `manager` label. - -That is not the same as proving the local failure is environmental. CI shards the suite -across six legs, so each leg runs at materially lower contention than one 56-spec local run -at `--procs=4` — which is precisely the load difference hypothesised below. CI passing is -consistent with both "PR 2 is innocent" and "the spec is load-sensitive and CI never applies -enough load". It rules out a deterministic regression; it does not rule out a latent race -that this change makes marginally easier to hit. - -Run 4 is the one that discriminates, because it differs from run 3 in exactly one variable. -Runs 2 and 3 differ in **two** — the code *and* the suite scope/parallel load — so run 3 -alone does not convict PR 2. - -Why load plausibly matters: the assertion is an `Eventually` with a 30s budget, and -attribution requires an audit fact to arrive and match within a window. A 56-spec run at -`--procs=4` on a dev container is a materially different timing environment from a 43-spec -run. - -## Why PR 2 was argued to be irrelevant — and why that argument was over-trusted - -PR 2 replaced `SnapshotNamespaces()` with `WatchScopes()` and made both read sites project -one stream per namespace scope. The claim was that this is a **strict no-op** unless a single -`WatchedType` holds both the cluster-wide `""` key and a named-namespace key: - -- only named keys → old returned the sorted names, new returns the same; -- only `""` → old returned `nil` and the caller synthesised `targetWatchKey{GVR, ""}`, new - returns `[""]` and builds the identical key. - -The failing spec creates one `GitTarget` with a single namespaced `WatchRule` and no -co-resident `ClusterWatchRule`, so it never reaches the changed branch. - -The code reading still looks correct. The error was **procedural**: this argument was used -to explain away three successive failures without ever running the cheap control that would -test it. A structural argument about which branch executes is not evidence about a -timing-sensitive integration failure. The control costs ~6 minutes and should have been run -after the first failure, not the third. - -## Confounders that invalidated earlier runs - -- **Self-inflicted cluster contamination.** A targeted `commit-request`-only run left Gitea - SSH keys registered; the next full run failed `playground` with - `HTTP 422: Key title has been used`. Any verdict from that run is untrustworthy. -- **`clean-cluster` run from the wrong directory.** It does `rm -rf .stamps/cluster/` on - a *relative* path, so running it from `test/e2e` deleted nothing real while k3d still - removed the cluster — leaving a stale `ready` stamp. `prepare-e2e` then skipped cluster - creation and everything failed with `No nodes found for given cluster`. -- **Exit status masked by a pipe.** `task prepare-e2e | tail -15 && task test-e2e` takes - `tail`'s exit status, so a failed prepare reported success and the suite ran against a - cluster that did not exist. Use `set -o pipefail` or capture exit codes separately. - -Net: of four runs, one was contaminated and two tested nothing. - -## Not to be confused with - -Three consecutive runs each failed a *different* timing-sensitive spec — `playground` + -`Commit Request Bundle (UC2)`, then `Commit Request generateName`, then this one. The -commit-request failures were separately re-run in isolation and **all 4 commit-request specs -passed**, which is why they were written off. That pattern is what made "this suite is flaky -here" attractive. It may still be true. It is not established. - -## Open questions - -1. **Does the `manager` subset fail on main?** (run 4). If yes, PR 2 is implicated and main - is currently broken — revert first, diagnose second. If no, the variable is suite load. -2. **Does CI reproduce it?** `release.yml`'s first job is `uses: ./.github/workflows/ci.yml`, - which has an `e2e` job, so the main build validates the merged PR 2 on clean - infrastructure — an independent read. -3. **If it is load, where exactly is the window lost?** The audit fact must arrive and match - before the write is authored. Worth instrumenting rather than inferring. -4. **Should the mode probe be fixed regardless?** It is a `--since=30m` log grep that fails - open. A deterministic signal — a status field, or asserting the deployment's flags — would - remove a whole class of misreads. - -## If it turns out to be PR 2 - -The place to look first is the no-op claim, since that is the load-bearing assumption. Test -it directly: build the `WatchedTypeTable` this spec actually produces and assert -`targetWatchSpecs` output is byte-identical before and after the change. If some GitTarget in -that namespace acquires both a `""` and a named scope through a path not anticipated, the -extra stream would deliver the same object twice, and a duplicate delivery racing the audit -match is a plausible mechanism for losing the attributed author. diff --git a/docs/design/watchrule-source-namespace/pr2-session-log-and-findings.md b/docs/design/watchrule-source-namespace/pr2-session-log-and-findings.md deleted file mode 100644 index f273efac..00000000 --- a/docs/design/watchrule-source-namespace/pr2-session-log-and-findings.md +++ /dev/null @@ -1,322 +0,0 @@ -# Session log — landing PR 2, and the attribution failure it surfaced - -> Written 2026-07-20, at the end of the session that landed -> [PR 2](pr2-stream-scope-collapse.md) and opened the branch for -> [PR 3](pr3-clusterwatchrule-target-admission.md). Companion to -> [pr2-e2e-attribution-investigation.md](pr2-e2e-attribution-investigation.md), which holds -> the mechanism detail; this page is the readable top-to-bottom account: what was done, what -> was measured, what was **ruled out and on what evidence**, and what is still open. -> -> **Bottom line: PR 2 is merged to main (#255) and is NOT the cause of the e2e failure.** -> The cause is a pre-existing ~10% attribution loss — the 3s grace window is too tight -> against an audit delivery lag of ~1s (dominated by apiserver audit batching). Mechanism, -> measurements, and fix options: [§0 of the investigation](pr2-e2e-attribution-investigation.md#0-resolution). -> -> This page keeps the working record, including **three conclusions I reached confidently and -> then had to reverse**. That sequence is the most useful thing here. - ---- - -## 1. What was set out to do - -1. Settle an open question left by PR 1 — one unexplained e2e failure. -2. Implement PR 2 (a cluster-wide selection must not collapse named-namespace streams). -3. Prepare PR 3. - -All three happened. A fourth thing happened that nobody planned: a genuine, pre-existing -attribution defect surfaced — ~10% of actors silently lost under load — plus four pieces of -test/CI infrastructure that were quietly lying to us. - ---- - -## 2. PR 1's open question — closed - -**Question:** the full suite failed `Commit Request Bundle (UC2)` -([commit_request_e2e_test.go:377](../../../test/e2e/commit_request_e2e_test.go#L377)) waiting -for a CommitRequest to become Ready. PR 1 changed `renderReconcileCommitMessage`'s signature -and added `Namespace` to `ReconcileCommitMessageData` — the only plausible contact point. - -**Done:** re-ran with -`E2E_LABEL_FILTER='!image-refresh && !bi-directional && !source-cluster && commit-request'`. - -**Result: all 4 commit-request specs passed.** Did not reproduce. - -**Conclusion:** not a PR 1 regression. Note the caveat that only became visible later — a -single non-reproduction is weak evidence, and this suite turned out to have a *different* -failing spec on each subsequent run. This closure is "did not reproduce", not "proven -unrelated". - ---- - -## 3. PR 2 — what shipped - -**The defect.** `SnapshotNamespaces()` returned `nil` whenever a `WatchedType` carried the -`""` namespace key, and `nil` meant *all namespaces* at every read site. A WatchRule scoped -to one namespace and a ClusterWatchRule scoped cluster-wide, on the same GVR and GitTarget, -collapsed into one all-namespaces stream. The named scope survived only in the plan hash, and -`targetWatchSpecs` substituted `NamespaceOps[""]` for its operation set — so a `CREATE`-only -named rule co-resident with an `UPDATE`-only cluster-wide rule lost its filter too. - -**The fix.** `SnapshotNamespaces` → `WatchedType.WatchScopes`, returning every scope -including the cluster-wide `""` instead of collapsing to it, with **both** read sites -projecting one stream per scope: - -- `targetWatchSpecs` keys a watch per scope, each carrying `NamespaceOps[ns]`. -- `snapshotGVRsFromTable` **and** `resolveSnapshotGVRForType` project through one shared - `snapshotGVRScopes` helper. `snapshotGVR.namespaces []string` became a single - `namespace string`, matching `targetWatchKey` and matching how a dynamic client `List` - takes a namespace (`""` = all). - -**Design choice — distinct streams, not subtraction.** The spec allowed either. "All -namespaces except `team-a`" is not expressible as a watch, and the subtraction variant would -need a sweep scope shaped as a complement, which `git.ResyncScope` cannot represent. - -**The replaced test.** `TestBuildWatchedTypeTable_ClusterWideOverridesNamedNamespaces` -asserted the collapse *as intended*. Deleted, not skipped. Three replacements, each verified -to fail against a restored pre-fix collapse, each on its own assertion: table level, -`targetWatchSpecs` (streams + op sets), `snapshotGVRsFromTable`. - -**Merged** as #255. Base was auto-retargeted from the PR 1 branch to `main` when #254 merged. - ---- - -## 4. The e2e story, in order - -| # | Code | Scope | Result | Verdict on this run | -|---|---|---|---|---| -| 0 | PR 1 branch | `commit-request` only | 4/4 passed | closed §2 | -| 1 | PR 2 branch | full, 56 specs | 2 failed: `playground`, `CommitRequest generateName` | **void** — cluster contaminated by run 0 | -| 2 | PR 2 branch | full, 56 specs | 55 passed, **1 failed**: attribution | valid | -| 3 | `4f37759`, PR 2 **absent** | `manager`, 43 specs | 43 passed, **0 failed** | valid control | -| 4 | `10a530a`, PR 2 **present** | `manager` | 40 passed, **1 failed**: attribution | **valid, controlled** | -| 5 | `10a530a` main, PR 2 present | **CI**, 6 sharded legs | **all passed** | valid, different environment | -| 6 | `4f37759`, PR 2 absent | `manager` | 43 passed, **0 failed** | second control sample | -| 7 | PR 2 + stream instrumentation | `manager` | 43 passed, **0 failed** | **PR 2 passes** — split the "controlled" story open | -| 8 | PR 2 (run 7), metrics | — | absent **7 / 88 = 8.0%** | the measurement that settled it | -| 9 | `4f37759` baseline, metrics | — | absent **8 / 83 = 9.6%** | baseline is *worse* | - -Runs 3, 4 and 6 looked decisive — same subset, same machine, control passing 2/2 and PR 2 -failing 2/2, on a diff verified to be exactly PR 2's 8 files (the Gitea unpin `27bf27c` is an -ancestor of **both** commits, so it cannot explain anything). **That conclusion was wrong.** -Run 7 passed with PR 2 present, and runs 8–9 replaced the coin-flip with a rate measured over -~85 resolutions per arm: **8.0% loss with PR 2, 9.6% without**. Per-resource on baseline, -configmaps alone lose **10.0%**. The spec is ~10%-flaky on its own; three samples per arm -could never separate that from an effect, and I treated a 2-vs-0 split as proof. - -**The control was verified to be a real control.** `git diff --stat 4f37759 10a530a` is -exactly the 8 `internal/watch` files of PR 2 and nothing else — in particular the Gitea unpin -(12.5.1 → 12.7.0) and dependency refresh in `27bf27c` are ancestors of **both** commits, so -they cannot explain the difference. This was checked precisely because a stacked branch can -easily lag main by unrelated commits; here it does not. - -Run 5 pulls the other way: main's `Release` build runs `ci.yml` (whose first job everything -else `needs`), and **every** e2e leg passed — including `full-manager`, which contains this -exact spec. But CI shards across six legs, so each runs at materially lower contention than -one 56-spec local run at `--procs=4`. At the time this was read as "rules out a deterministic -regression but not a latent race" — correct as far as it went, and in hindsight CI was simply -sampling the same ~10% lottery at lower load. - ---- - -## 5. The failing spec, explained - -`Manager WatchRule ConfigMap and Secret >> should create Git commit when ConfigMap is added -via WatchRule` -([watchrule_configmap_secret_e2e_test.go:446](../../../test/e2e/watchrule_configmap_secret_e2e_test.go#L446)). - -It proves a human's identity survives from `kubectl` into a Git commit author: - -1. Create a `GitTarget` + a `WatchRule` selecting ConfigMaps in its namespace; wait for Ready. -2. Create a ConfigMap **impersonating a user**: `kubectl --as=jane@acme.com`. The - impersonation is the point of the spec. -3. Poll a checkout of the Gitea repo until the YAML exists, the commit message carries - `[CREATE]` and `v1/configmaps/test-configmap`, and `git log -1 --pretty=%an` is the - impersonated user. - -**The two modes.** In **configured-author mode** (attribution off, or no Redis) every commit -is authored by the configured committer. In **attribution mode** (the default here) the live -watch event is matched against an audit-webhook fact carrying the acting user; with no match -it falls back to `DefaultCommitterName`. - -**The ambiguity that cost hours.** `DefaultCommitterName` is `"GitOps Reverser"` -([internal/git/types.go:22](../../../internal/git/types.go#L22)) — *also* the configured-author -identity. Observing `GitOps Reverser` means **either** "configured-author mode, working" **or** -"attribution mode, attribution lost". The string alone does not say which. - -**The failure:** expected `jane@acme.com`, got `GitOps Reverser`. - ---- - -## 6. What was ruled out, and on what evidence - -Each of these was a live hypothesis. None survived. - -### 6.1 A PR 1 regression — ruled out -Re-ran the commit-request specs in isolation: 4/4 passed (§2). - -### 6.2 The `playground` Gitea failure — ruled out (self-inflicted) -`HTTP 422: Key title has been used`. A previous targeted run had registered the SSH key and -never ran playground's cleanup. Environmental contamination from my own partial run, not a -code defect. Invalidated run 1 entirely. - -### 6.3 A mid-run controller restart disrupting the spec — ruled out -A spec does `rollout restart` the controller (explaining a 3-minute-old pod in a 10-minute -suite), but [restart_reconcile_e2e_test.go:34](../../../test/e2e/restart_reconcile_e2e_test.go#L34) -is decorated `Serial`, so Ginkgo runs it after all parallel specs. It cannot have overlapped. - -### 6.4 The test asserting the wrong branch — ruled out -The probe *could* misreport (§7.2). It did not here: the Deployment carries `--redis-addr=…` -with no `--author-attribution=false`, and the controller emits no `configured-author mode:` -banner. So the deployment is genuinely in attribution mode, the probe genuinely returned the -right answer, and the spec was right to demand `jane@acme.com`. - -**This also rules out the "after 30 minutes it stops detecting the mode" theory** for *this* -failure. Aging pushes the probe toward `false` = "expect `jane@acme.com`" — which is what it -already correctly returned. Aging can only cause a *false failure* in the opposite -configuration: a controller truly in configured-author mode, up >30 minutes. Real bug, wrong -suspect. - -### 6.4a The control being contaminated by unrelated main commits — ruled out -See §4: `4f37759` and `10a530a` differ by exactly PR 2's 8 files. Checked explicitly. - -### 6.5 "This suite is just flaky here" — RULED OUT -Argued three times on a code-reading (§6.6), then declared dead when the control came back -2/2 versus 0/2 — and then **reinstated by measurement**. Aggregate attribution-loss rates are -8.0% (PR 2) versus 9.6% (baseline), and configmaps alone lose 10.0% on baseline, so the spec -is ~10%-flaky per run on its own and the fail/pass split was noise. The spec *is* flaky here; -what was wrong was calling it flaky without evidence, and then calling it non-flaky on -evidence too thin to carry the claim. See [§0](pr2-e2e-attribution-investigation.md#0-resolution). - -### 6.6 "PR 2 is a strict no-op in this configuration" — code-true, but insufficient -The claim: the change alters nothing unless a single `WatchedType` holds **both** the `""` -key and a named key. -- only named keys → old returned the sorted names, new returns the same; -- only `""` → old returned `nil` and the caller synthesised `targetWatchKey{GVR, ""}`; new - returns `[""]` and builds the identical key. - -The failing spec has one `GitTarget` with a single namespaced `WatchRule` and no co-resident -`ClusterWatchRule`, so it should never reach the changed branch. **The code reading still -looks correct — and the experiment contradicts it.** The experiment outranks the reading. - -**The search space is small.** `WatchScopes()` has exactly ONE production consumer: -`targetWatchSpecs` ([target_watch.go:218](../../../internal/watch/target_watch.go#L218)). -Everything else PR 2 touched — `snapshotGVRsFromTable`, `resolveSnapshotGVRs`, -`resolveSnapshotGVRForType`, `snapshotGVRScopes` — has **no production callers at all** -(verified by grep on main, excluding tests). So the entire runtime effect of PR 2 reduces to -whatever `targetWatchSpecs` now returns. - -**One genuine behavioural difference found so far**, and it is not obviously the culprit: -when a `WatchedType` has an **empty** `NamespaceOps` map, the old code fell into its -`len(namespaces) == 0` branch and synthesised a **cluster-wide** watch key -`targetWatchKey{GVR, ""}`; the new code iterates an empty scope list and creates **no key at -all**. Production should never produce an empty map (`buildWatchedTypeTable` always records -at least one namespace per selection), and the failure mode does not match — a missing watch -would mean the ConfigMap never reaches Git, whereas here the file and commit landed and only -the *author* was wrong. Recorded because it is a real, untested divergence. - -Next step is instrumentation rather than more reading: capture the controller's -`watch-first target watch set reconciled` lines (they log `watchCount`) from a failing main -run and a passing baseline run and diff the declared watch sets for the spec's GitTarget. If -they are identical, the cause is downstream of the watch set and the reading is right; if -they differ, the difference names the bug. - ---- - -## 7. Infrastructure defects found (all real, all fixed or filed) - -### 7.1 Stacked PRs ran no CI at all — fixed -`ci.yml` triggered on `pull_request: branches: [main]`. A stacked PR based on the previous -PR's branch never matched. **#255 merged having run only CodeRabbit and GitGuardian — no -lint, no unit tests, no e2e.** Retargeting does not rescue it: when the base merges, GitHub -retargets and fires `pull_request` with action `edited`, which is not in the default trigger -set, so nothing re-runs. - -Fixed by dropping the `branches` filter (PR #257). Without it, four of this workstream's five -PRs would merge unvalidated. **Caveat: the fix cannot validate itself** — #257 targets `main`, -so it runs CI regardless. First real proof is the next PR against a non-`main` base. - -### 7.2 The author-mode probe was a log grep that fails open — fixed -`configuredAuthorModeEnabled` ran `kubectl logs … --since=30m`, grepped for a startup banner, -and returned `false` on any error. Three ways to misreport: banner ages out after 30 minutes; -`kubectl logs deployment/…` errors or picks the wrong pod mid-rollout; **the quickstart specs -deliberately redeploy the controller in configured-author mode, leaving that banner in the -shared log window for later specs to misread.** Given §5's ambiguity, a wrong answer silently -swaps in a weaker assertion that *passes*. - -Fixed in #257: read the Deployment's container args, fail loudly if unreadable, with a pure -`configuredAuthorModeFromArgs` under unit test pinning the defaults (`cmd/main.go` defaults -**both** `--author-attribution` true and `--redis-addr` `valkey:6379`, so an absent flag means -attribution mode). The change can only make the assertion **stricter**. - -That quickstart contamination path is also a plausible reason this spec looks healthier in CI -than locally: CI shards `quickstart-install` and `full-manager` into separate legs; locally -they share one cluster and one log window. - -### 7.3 `task clean-cluster` is cwd-sensitive — filed, not fixed -It does `rm -rf .stamps/cluster/` on a **relative** path. Run from `test/e2e` it deletes -nothing real while k3d still removes the cluster, leaving a stale `ready` stamp. `prepare-e2e` -then skips cluster creation and everything fails with `No nodes found for given cluster`. Cost -one wasted cycle. **Always run task targets from the repo root.** - -### 7.4 Piping a task to `tail` masks its exit status — process fix -`task prepare-e2e | tail -15 && task test-e2e` takes `tail`'s status, so a failed prepare -reported success and the suite ran against a nonexistent cluster, producing a `BeforeSuite` -failure that then had to be unwound. Use `set -o pipefail` or capture exit codes separately. - ---- - -## 8. Process lessons (mine) - -- **A structural argument is not evidence about a timing-sensitive integration failure.** The - §6.6 no-op reading was used to wave off three consecutive failures. The control that tested - it cost ~6 minutes and should have been run after the *first* failure, not the third. -- **Two variables changed at once.** Run 2 (PR 2, full suite) vs run 3 (baseline, subset) - differed in both code and load, so run 3 alone could not convict or acquit. Only runs 3+4 - isolated it. -- **I contaminated my own evidence twice** — the Gitea key (§6.2), then the cwd/pipefail pair - (§7.3, §7.4) which produced two runs that tested nothing. Of six runs, one was void and two - were empty. -- **Stated confidence outran the evidence** in three successive summaries. The maintainer's - "I don't remember flakes around this test" was worth more than any of them. - ---- - -## 9. Open questions — resolved - -1. **Did the control hold at n=2?** Yes (baseline 2/2 passed) — and it did not matter. The - aggregate loss rate settled it instead: 8.0% with PR 2 versus 9.6% without. -2. **How could §6.6 be code-true and the experiment still fail?** Both were right. The code - reading was correct (PR 2 is a no-op for that spec's GitTarget); the experiment was - sampling a ~10%-lossy process three times per arm and I read the split as signal. - Instrumentation did find real case-C fan-out - (`unsupported-folder-dest` gets two `configmaps` streams), but on a different GitTarget in - a different namespace, and audit volume did not correlate with failure — the failing run - had the *lowest* volume. -3. **Is attribution reliable under 4-proc load?** **No, and that is the real defect.** ~10% of - ConfigMap resolutions lose the actor, independent of PR 2. -4. **Does #257 go green?** Open at time of writing. - -## 10. State at end of session - -- **Merged to main:** #254 (PR 1), #255 (PR 2). PR 2 is correct and stays. -- **Open:** #257 on `feat/watchrule-src-ns-pr3-clusterwatchrule-target-admission` — the CI - trigger fix, the author-mode probe fix, per-stream declare logging, - `hack/attribution-diagnostics.sh`, and these findings. **PR 3's actual ClusterWatchRule - admission fix is not written yet.** -- **The real open defect:** attribution silently loses ~10% of actors under parallel load. - The 3s grace window is too tight against a ~1s audit batching delay. Fix options in - [§0](pr2-e2e-attribution-investigation.md#0-resolution). This is a correctness bug in the - product's core promise — naming who made a change — and it is worth more than the PR 2 - question that surfaced it. -- **PR 3 groundwork:** design verified against the tree; every load-bearing claim holds. One - gap found: the design says only a ClusterProvider → ClusterWatchRules mapper is missing, but - admission is evaluated against **namespace labels**, and the GitTarget controller carries a - matching `namespaceToGitTargets` watch. The CWR controller watches only GitTarget and - GitProvider, so revocation by *relabelling a namespace* would converge for GitTargets and - never for ClusterWatchRules. **PR 3 needs both mappers plus a revocation test for the label - path.** -- **Also fixed by a parallel agent, and worth knowing:** `test/e2e/Taskfile.yml` is now - `Taskfile-e2e.yml` so that `task` invoked from inside `test/e2e/` cannot stop its upward - search there and run with the wrong working directory — the structural cause of the - `clean-cluster` incident in §7.3. A `PreToolUse` hook now refuses unguarded `task … | …` - pipelines (§7.4). diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index 81f42781..ef3c45a8 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -5,10 +5,12 @@ package e2e import ( "errors" "fmt" + "math" "os" "os/exec" "os/signal" "path/filepath" + "strconv" "strings" "syscall" "testing" @@ -73,6 +75,9 @@ var _ = SynchronizedBeforeSuite(func() []byte { // if the assertion fails — it also releases on process exit). var _ = SynchronizedAfterSuite(func() {}, func() { defer releaseE2ERunLock() + // Diagnostic first, so its numbers reach the artifacts even when the gating + // audit invariant below fails the suite. + reportAttributionStats() assertNoAnomalousAuditOutcomes() }) @@ -121,6 +126,106 @@ func assertNoAnomalousAuditOutcomes() { _, _ = fmt.Fprintf(GinkgoWriter, "✅ no anomalous audit outcomes (0 error-category events across the run)\n") } +// attributionWaitBuckets mirrors telemetry's attributionWaitBuckets tail. Only the seconds-scale +// boundaries are interesting here: the question this report answers is how close the run came to +// the grace window, and how far past the 3s DEFAULT grace a fact can still arrive. +var attributionWaitBuckets = []float64{0.5, 1, 2, 3, 5, 10} + +// reportAttributionStats prints the run's author-attribution outcome and timing distribution. +// +// It is DIAGNOSTIC, not gating. It exists because attribution fails silently: when no audit fact +// matches within the grace window the commit is authored by the configured committer, and +// git.DefaultCommitterName ("GitOps Reverser") is ALSO the configured-author identity — so a lost +// actor is indistinguishable from a correct configured-author commit by inspection. The only way +// to see it is to count it. +// +// The two numbers to read: +// +// - "absent" — resolutions that gave up and lost the actor's name. Every one of these is a +// commit attributed to the committer instead of the human or service account responsible. +// - resolutions resolved ABOVE 3s — facts that arrived later than the 3s default grace. Each is +// an attribution the default would have dropped, so this is the direct measure of how much +// headroom --author-attribution-grace is buying (e2e sets 10s; see config/deployment.yaml). +// +// Fact delivery is bounded below by the apiserver's --audit-webhook-batch-max-wait (1s in this +// cluster), so a healthy run still shows most waits in the 0.5-2s range; the tail is what matters. +func reportAttributionStats() { + total, err := queryPrometheus(`sum(max_over_time(gitopsreverser_attribution_resolutions_total[2h])) or vector(0)`) + if err != nil || total == 0 { + _, _ = fmt.Fprintf(GinkgoWriter, + "ℹ️ attribution stats skipped: no resolutions recorded (configured-author mode or no live events)\n") + return + } + + _, _ = fmt.Fprintf(GinkgoWriter, "\n📊 author attribution — %.0f resolutions this run\n", total) + + var absent float64 + for _, result := range []string{"exact_user", "weak", "exact_deletecollection_item", "absent"} { + n, qErr := queryPrometheus(fmt.Sprintf( + `sum(max_over_time(gitopsreverser_attribution_resolutions_total{result=%q}[2h])) or vector(0)`, result)) + if qErr != nil { + continue + } + if result == "absent" { + absent = n + } + _, _ = fmt.Fprintf(GinkgoWriter, " result %-28s = %6.0f (%5.1f%%)\n", result, n, 100*n/total) + } + + // Cumulative histogram: le=X is "waited at most X seconds". + // + // Split by result, because the two populations answer different questions. For RESOLVED + // results the tail says how close fact delivery ran to the window — anything above 3s only + // succeeded because e2e widens the grace past the 3s default. For "absent" the wait is just + // the grace window being spent, so a cluster of absents at the ceiling means "waited the + // whole time and nothing ever came", NOT "arrived slightly too late". + _, _ = fmt.Fprintf(GinkgoWriter, " wait distribution (cumulative):\n") + printWaitBuckets("resolved", `,result!="absent"`) + printWaitBuckets("absent ", `,result="absent"`) + + resolvedOverDefault, err := queryPrometheus( + `(sum(max_over_time(gitopsreverser_attribution_resolution_wait_seconds_count{result!="absent"}[2h])) ` + + `or vector(0)) - (sum(max_over_time(` + + `gitopsreverser_attribution_resolution_wait_seconds_bucket{result!="absent",le="3.0"}[2h])) or vector(0))`) + if err == nil { + _, _ = fmt.Fprintf(GinkgoWriter, + " %.0f resolution(s) SUCCEEDED after waiting longer than the 3s default grace"+ + " — these are the ones a 3s window would have lost.\n", resolvedOverDefault) + } + if absent > 0 { + _, _ = fmt.Fprintf(GinkgoWriter, + " ⚠ %.0f resolution(s) lost the actor (%.1f%%): committed as the configured committer.\n"+ + " If their waits sit at the grace ceiling above, the fact never arrived at all and a\n"+ + " LONGER grace will not help — check delivery with hack/attribution-diagnostics.sh.\n"+ + " Note some absences are legitimate: a watch's initial replay re-delivers"+ + " pre-existing\n objects that no live actor touched, and those have no fact by"+ + " construction.\n", + absent, 100*absent/total) + } +} + +// printWaitBuckets renders one cumulative wait histogram. The `le` label is rendered by the +// OTel→Prometheus exporter as "1.0"/"3.0"/"10.0" — NOT "1"/"3"/"10" — so an integral boundary +// must be formatted with a decimal or every query above 0.5 silently matches nothing and +// returns 0, which reads as a plausible (and wrong) distribution rather than as an error. +func printWaitBuckets(label, resultSelector string) { + var prev float64 + for _, le := range attributionWaitBuckets { + leLabel := strconv.FormatFloat(le, 'g', -1, 64) + if le == math.Trunc(le) { + leLabel = fmt.Sprintf("%.1f", le) + } + n, qErr := queryPrometheus(fmt.Sprintf( + `sum(max_over_time(gitopsreverser_attribution_resolution_wait_seconds_bucket{le=%q%s}[2h]))`+ + ` or vector(0)`, leLabel, resultSelector)) + if qErr != nil { + continue + } + _, _ = fmt.Fprintf(GinkgoWriter, " %s <=%5.1fs %6.0f (+%.0f)\n", label, le, n, n-prev) + prev = n + } +} + // configuredAuthorModeEnabled reports whether the deployed controller authors every commit // as the configured committer (configured-author mode) rather than naming the real actor // from audit facts (attribution mode). From db3c13687d023df1c574d3278d2cfb7d59faa503 Mon Sep 17 00:00:00 2001 From: Simon Koudijs Date: Mon, 20 Jul 2026 12:18:45 +0000 Subject: [PATCH 06/17] docs(design): propose an explicit author for unattributed commits MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When attribution is enabled but no audit fact matches, the commit is authored by the configured committer — the same "GitOps Reverser" identity that every commit carries in configured-author mode. So a silent attribution failure is byte-identical in Git history to correct configured-author behaviour, and the only way to see the ~7-10% loss is a Prometheus counter nobody looks at. That ambiguity is what made the loss so slow to diagnose (docs/debug/ attribution-loss.md). Proposes authoring those commits as `unattributed ` while leaving the committer as the operator — using git's author/committer split for exactly what it is for: the operator really did make the commit, and the honest answer to "on whose behalf" is "we do not know". Configured-author mode is untouched, since there the committer genuinely is the author. Checked the interaction most likely to make this risky: commit-window grouping matches on UserInfo.Username, and today unattributed events share Username == "" so they already group with each other and not with attributed ones. A sentinel renames that equivalence class without splitting or merging any window. Does NOT fix the underlying loss — makes it legible, and worth doing first because it turns an invisible correctness failure into one visible in `git log`, and gives the open investigation a labelled instance tied to an object and time rather than a counter increment. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../design/unattributed-author-placeholder.md | 154 ++++++++++++++++++ 1 file changed, 154 insertions(+) create mode 100644 docs/design/unattributed-author-placeholder.md diff --git a/docs/design/unattributed-author-placeholder.md b/docs/design/unattributed-author-placeholder.md new file mode 100644 index 00000000..51c4c5dd --- /dev/null +++ b/docs/design/unattributed-author-placeholder.md @@ -0,0 +1,154 @@ +# Name the gap: an explicit author for unattributed commits + +> Design, 2026-07-20. **Not built.** Small change, no API/CRD impact. +> Motivated by [debug/attribution-loss.md](../debug/attribution-loss.md), which took hours +> longer to diagnose than it should have — precisely because this gap has no name. + +## The problem + +When author attribution is **enabled** but no audit fact matches, the commit is authored by +the configured committer: + +~~~go +author := pendingWrite.AuthorUserInfo() +if author.Username == "" { + return &git.CommitOptions{Author: committer, Committer: committer, Signer: signer} +} +~~~ +[commit.go:188-194](../../internal/git/commit.go#L188-L194) + +`committer` is `DefaultCommitterName` = `"GitOps Reverser"` +([types.go:22](../../internal/git/types.go#L22)) — which is **also** the identity every commit +carries in configured-author mode. So one string means two unrelated things: + +| Mode | Fact found? | Author today | +|---|---|---| +| configured-author | n/a | `GitOps Reverser` ← correct and expected | +| attribution | yes | the real actor | +| attribution | **no** | `GitOps Reverser` ← **a silent failure, indistinguishable from row 1** | + +Rows 1 and 3 are byte-identical in Git history. Nothing in the repository, the commit, or a +`git log` distinguishes "this operator does not do attribution" from "this operator does +attribution and just lost someone's name". That ambiguity is not theoretical: it is why the +first reading of the e2e failure was wrong, and why the only way to see the ~7–10% loss today +is a Prometheus counter nobody looks at. + +## The proposal + +When attribution is **enabled** and no fact matches, author the commit as an explicit +placeholder instead of the committer. + +~~~text +Author: unattributed +Committer: GitOps Reverser +~~~ + +Configured-author mode is untouched: it keeps authoring as the committer, because there the +committer genuinely *is* the author. + +| Mode | Fact found? | Author after | +|---|---|---| +| configured-author | n/a | `GitOps Reverser` | +| attribution | yes | the real actor | +| attribution | **no** | **`unattributed`** ← now says what happened | + +### Why this shape + +**Use Git's author/committer split for what it is for.** The operator really did make the +commit, so it stays the committer. The author slot answers "on whose behalf", and the honest +answer is "we do not know". Overwriting the committer would lose the operator's identity for +no gain. + +**`.invalid` is reserved (RFC 2606).** `unattributed@gitops-reverser.invalid` can never +collide with a real address, never routes mail, and is recognisable at a glance. Note +`authorEmail` already synthesises `ConstructSafeEmail(username, "cluster.local")` for users +with no email claim, so a synthetic address is established practice here — but `.invalid` is +the stronger signal for a value that is deliberately not a person. + +**Lowercase, no display name.** Real actors get an OIDC display name; the placeholder should +not look like one. `unattributed` sorts and greps distinctly, and `git shortlog -sn` surfaces +it as its own line — which is the point: the loss becomes a number a human sees without +querying anything. + +## What it does not change + +**Commit-window grouping is unaffected in shape.** `canAppend` matches on +`e.UserInfo.Username == w.Author` ([open_window.go:57](../../internal/git/open_window.go#L57)). +Today unattributed events all carry `Username == ""` and therefore group with each other and +not with attributed ones. With a sentinel they still all share one identity and still group +with each other and not with attributed ones. One equivalence class is renamed; none is split +or merged. This was the interaction most likely to make the change risky, and it does not. + +**CommitRequest attribution is unaffected** — it sets a named author explicitly, so it never +reaches the placeholder path. + +**No CRD or API change.** The value never appears in a spec or status. + +## The cost, stated plainly + +This changes what lands in users' Git history. Three real consequences: + +1. **Tooling that keys on the author string.** Anyone matching `GitOps Reverser` to mean "the + operator wrote this" will now miss unattributed commits. That is arguably a *fix* — those + commits were mislabelled — but it is still a behaviour change on shipped data. +2. **It makes an existing defect visible.** With ~7–10% loss unfixed, adopters will suddenly + see `unattributed` in their history and reasonably file bugs. That is the intent, and it + should be release-noted rather than discovered. +3. **Signed commits** carry the author in the signature payload. The placeholder must pass + `isSafeSignatureField` — it does (ASCII, no angle brackets, no control characters). + +### Should it be configurable? + +**Recommendation: yes, one boolean, defaulting to the new behaviour.** + +~~~text +--author-attribution-placeholder=true # default: author unattributed commits as `unattributed` +--author-attribution-placeholder=false # legacy: author them as the committer +~~~ + +Default-on because silently mislabelling authorship is the bug being fixed, and a flag that +defaults to the broken behaviour fixes nothing. The escape hatch exists for an operator whose +downstream tooling cannot absorb the change mid-stream; it is a migration aid, not a +long-term choice, and should say so in its flag help. + +Rejected alternative: make the placeholder *string* configurable. It invites every deployment +inventing its own value, which destroys the one property that matters — that everyone +recognises it. + +## Implementation sketch + +1. `internal/git/types.go` — add `UnattributedAuthorName = "unattributed"` and + `UnattributedAuthorEmail = "unattributed@gitops-reverser.invalid"` next to the committer + constants, so the three identities are readable in one place. +2. `internal/watch/target_watch.go` — in `attachAuthor`, when the resolver is non-nil (i.e. + attribution is enabled) and `ResolveAuthor` returns `ok=false`, stamp the placeholder + `UserInfo` instead of leaving it zero. The resolver stays untouched; it already reports + `AttributionAbsent`. +3. `internal/git/commit.go` — the `author.Username == ""` branch then means only + "configured-author mode", which is exactly what it should have meant all along. Tighten + its comment accordingly. +4. Flag plumbing in `cmd/main.go` alongside `--author-attribution`. + +The placeholder is stamped at the **watch** layer rather than the commit layer on purpose: the +watch layer is the only place that knows attribution was *attempted and failed*, as opposed to +never attempted. + +## Tests + +- **Unit:** attribution enabled + no fact → author is the placeholder, committer unchanged; + attribution disabled + no fact → author is the committer (unchanged); fact found → real + actor (unchanged). +- **Unit:** two unattributed events land in one commit window (the equivalence class survives + the rename); an unattributed and an attributed event do not. +- **e2e:** the existing author assertions gain a third expectation. Today a lost attribution + fails as `expected jane@acme.com, got GitOps Reverser` — ambiguous. After, it fails as + `got unattributed`, which names the cause in the failure message itself. + +## Relationship to the open defect + +This does **not** fix [attribution-loss.md](../debug/attribution-loss.md); it makes it +legible. Both are worth doing, and this one is worth doing *first*: it turns an invisible +correctness failure into one an operator can see in `git log` without a Prometheus query, and +it removes the ambiguity that made the underlying defect so slow to diagnose. It also gives +that investigation a sharper signal — an `unattributed` commit is a labelled instance of the +failure, tied to a specific object and time, rather than a counter increment. From 3715d94aa0efc9b7aedb3d8b57da1ab422bf2122 Mon Sep 17 00:00:00 2001 From: Simon Koudijs Date: Mon, 20 Jul 2026 12:25:36 +0000 Subject: [PATCH 07/17] docs(design): drop the flag, and say "attribution failed" rather than "unattributed" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two revisions after review. No flag. The earlier draft proposed --author-attribution-placeholder defaulting on. The only thing such a flag can do is restore silently mislabelled authorship, which is not a preference to preserve behind an opt-out — it is the bug. Attribution is already gated by --author-attribution: an operator who does not want actor names turns that off and gets configured-author mode, where the committer genuinely is the author. A second flag would only add an incoherent third state: attribution on, but lie about it when it fails. "unattributed" was too passive — it reads as "this operator does not do attribution", which is exactly the confusion with configured-author mode the change exists to remove. The reader must be able to tell the operator TRIED and failed, because that is a defect to investigate rather than a configuration they chose. Settled on three distinct strings, because UserInfo feeds three places and one string does not serve them: Username attribution-failed (window grouping key AND the grouped commit MESSAGE body — must stand alone there, where a bare "unknown" says nothing) DisplayName unknown (attribution failed) (the git author Name a human reads; leads with what they care about, follows with why) Email attribution-failed@gitops-reverser.invalid Verified parentheses and spaces pass isSafeSignatureField, so the display form is valid in a commit object and in a signed payload. With no opt-out, adopters WILL see these commits appear, so the release note is now mandatory rather than advisable, and must point at the metric and at docs/debug/attribution-loss.md. Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/design/attribution-failed-author.md | 190 ++++++++++++++++++ .../design/unattributed-author-placeholder.md | 154 -------------- 2 files changed, 190 insertions(+), 154 deletions(-) create mode 100644 docs/design/attribution-failed-author.md delete mode 100644 docs/design/unattributed-author-placeholder.md diff --git a/docs/design/attribution-failed-author.md b/docs/design/attribution-failed-author.md new file mode 100644 index 00000000..a0e9df39 --- /dev/null +++ b/docs/design/attribution-failed-author.md @@ -0,0 +1,190 @@ +# Name the gap: an explicit author when attribution fails + +> Design, 2026-07-20. **Not built.** Small change, no API/CRD impact. +> Motivated by [debug/attribution-loss.md](../debug/attribution-loss.md), which took hours +> longer to diagnose than it should have — precisely because this gap has no name. + +## The problem + +When author attribution is **enabled** but no audit fact matches, the commit is authored by +the configured committer: + +~~~go +author := pendingWrite.AuthorUserInfo() +if author.Username == "" { + return &git.CommitOptions{Author: committer, Committer: committer, Signer: signer} +} +~~~ +[commit.go:188-194](../../internal/git/commit.go#L188-L194) + +`committer` is `DefaultCommitterName` = `"GitOps Reverser"` +([types.go:22](../../internal/git/types.go#L22)) — which is **also** the identity every commit +carries in configured-author mode. So one string means two unrelated things: + +| Mode | Fact found? | Author today | +|---|---|---| +| configured-author | n/a | `GitOps Reverser` ← correct and expected | +| attribution | yes | the real actor | +| attribution | **no** | `GitOps Reverser` ← **a silent failure, indistinguishable from row 1** | + +Rows 1 and 3 are byte-identical in Git history. Nothing in the repository, the commit, or a +`git log` distinguishes "this operator does not do attribution" from "this operator does +attribution and just lost someone's name". That ambiguity is not theoretical: it is why the +first reading of the e2e failure was wrong, and why the only way to see the ~7–10% loss today +is a Prometheus counter nobody looks at. + +## The proposal + +When attribution is **enabled** and no fact matches, author the commit as an explicit +placeholder instead of the committer. Always — there is no flag (see +[below](#no-flag)). + +~~~text +Author: unknown (attribution failed) +Committer: GitOps Reverser +~~~ + +Configured-author mode is untouched: it keeps authoring as the committer, because there the +committer genuinely *is* the author. + +| Mode | Fact found? | Author after | +|---|---|---| +| configured-author | n/a | `GitOps Reverser` | +| attribution | yes | the real actor | +| attribution | **no** | **`unknown (attribution failed)`** ← now says what happened | + +### The three slots, and why each string differs + +`UserInfo` feeds three distinct places, which is easy to miss and is why one string will not +do: + +| Field | Surfaces in | Value | +|---|---|---| +| `Username` | window grouping key ([open_window.go:57](../../internal/git/open_window.go#L57)), the **grouped commit message body** ([commit.go:96](../../internal/git/commit.go#L96)), and the author-name fallback | `attribution-failed` | +| `DisplayName` | the git author Name, via `authorName` | `unknown (attribution failed)` | +| `Email` | the git author Email, via `authorEmail` | `attribution-failed@gitops-reverser.invalid` | + +`Username` lands **inside commit message bodies**, so it has to stand alone there — a bare +`unknown` in a commit message says nothing, whereas `attribution-failed` names the event. +`DisplayName` is what a human reads in `git log`, so it gets the fuller phrasing. The split +mirrors the existing OIDC-display-name-vs-Kubernetes-username pattern rather than inventing a +convention. + +### Why this wording + +**Say it FAILED, not that it is absent.** `unattributed` (the first draft) is too passive — it +reads as "this operator does not do attribution", which is exactly the confusion with +configured-author mode that the change exists to remove. The reader must be able to tell that +the operator *tried and did not manage it*. That is a defect they should investigate, not a +configuration they chose. + +**`unknown` first, mechanism second.** `unknown (attribution failed)` leads with the fact a +human cares about — we do not know who — and follows with why. `unknown-attribution-failed` +as one token is slightly redundant (a failure implies unknown) and reads as a machine +identifier in the one slot that is meant for humans. + +**Use Git's author/committer split for what it is for.** The operator really did make the +commit, so it stays the committer. The author slot answers "on whose behalf", and the honest +answer is "we do not know". Overwriting the committer would lose the operator's identity for +no gain. + +**`.invalid` is reserved (RFC 2606).** `attribution-failed@gitops-reverser.invalid` can never +collide with a real address, never routes mail, and greps cleanly. `authorEmail` already +synthesises `ConstructSafeEmail(username, "cluster.local")` for users with no email claim, so +a synthetic address is established practice here — but `.invalid` is the stronger signal for a +value that is deliberately not a person. + +**Parentheses are safe in a signature.** `isSafeSignatureField` rejects control characters and +the angle brackets that delimit the email; spaces and parens pass, so the display form is +valid in a commit object and in a signed payload. + +`git shortlog -sn` then surfaces the loss as its own line with a count — the point being that +it becomes a number a human sees without querying anything. + +## What it does not change + +**Commit-window grouping is unaffected in shape.** `canAppend` matches on +`e.UserInfo.Username == w.Author` ([open_window.go:57](../../internal/git/open_window.go#L57)). +Today attribution-failed events all carry `Username == ""` and therefore group with each other and +not with attributed ones. With a sentinel they still all share one identity and still group +with each other and not with attributed ones. One equivalence class is renamed; none is split +or merged. This was the interaction most likely to make the change risky, and it does not. + +**CommitRequest attribution is unaffected** — it sets a named author explicitly, so it never +reaches the placeholder path. + +**No CRD or API change.** The value never appears in a spec or status. + +## The cost, stated plainly + +This changes what lands in users' Git history. Three real consequences: + +1. **Tooling that keys on the author string.** Anyone matching `GitOps Reverser` to mean "the + operator wrote this" will now miss the commits where attribution failed. That is arguably a *fix* — those + commits were mislabelled — but it is still a behaviour change on shipped data. +2. **It makes an existing defect visible.** With ~7–10% loss unfixed, adopters will suddenly + see `unknown (attribution failed)` in their history and reasonably file bugs. That is the + intent, and with no opt-out flag it is unavoidable — so it MUST be release-noted, with a + pointer to the metric and to docs/debug/attribution-loss.md, rather than discovered. +3. **Signed commits** carry the author in the signature payload. The placeholder must pass + `isSafeSignatureField` — it does (ASCII, no angle brackets, no control characters). + +### No flag + +**Rejected: making this configurable.** An earlier draft proposed +`--author-attribution-placeholder`, defaulting on. Dropped, and the reasoning is worth +keeping: the only thing such a flag can do is restore *silently mislabelled authorship*. That +is not a preference to be preserved behind an opt-out — it is the bug. A knob whose "off" +position reinstates a correctness defect is a knob nobody should be given, and shipping one +would imply the old behaviour is a legitimate choice. + +Attribution is already gated by `--author-attribution`. An operator who does not want actor +names in Git turns that off and gets configured-author mode, where the committer genuinely is +the author and no placeholder ever appears. That is the real, coherent choice; a second flag +would only add an incoherent third state — attribution on, but lie about it when it fails. + +Also rejected: making the placeholder *string* configurable. It invites every deployment +inventing its own value, which destroys the one property that matters — that everyone +recognises it. + +## Implementation sketch + +1. `internal/git/types.go` — add the placeholder identity next to the committer constants, so + all three identities (committer, real actor, failed) are readable in one place: + `AttributionFailedUsername = "attribution-failed"`, + `AttributionFailedDisplayName = "unknown (attribution failed)"`, + `AttributionFailedEmail = "attribution-failed@gitops-reverser.invalid"`. +2. `internal/watch/target_watch.go` — in `attachAuthor`, when the resolver is non-nil (i.e. + attribution is enabled) and `ResolveAuthor` returns `ok=false`, stamp the placeholder + `UserInfo` instead of leaving it zero. The resolver stays untouched; it already reports + `AttributionAbsent`. +3. `internal/git/commit.go` — the `author.Username == ""` branch then means only + "configured-author mode", which is exactly what it should have meant all along. Tighten + its comment accordingly. +No flag, so no `cmd/main.go` change. + +The placeholder is stamped at the **watch** layer rather than the commit layer on purpose: the +watch layer is the only place that knows attribution was *attempted and failed*, as opposed to +never attempted. + +## Tests + +- **Unit:** attribution enabled + no fact → author is the placeholder, committer unchanged; + attribution disabled + no fact → author is the committer (unchanged); fact found → real + actor (unchanged). +- **Unit:** two attribution-failed events land in one commit window (the equivalence class + survives the rename); an attribution-failed and an attributed event do not. +- **e2e:** the existing author assertions gain a third expectation. Today a lost attribution + fails as `expected jane@acme.com, got GitOps Reverser` — ambiguous. After, it fails as + `got unknown (attribution failed)`, which names the cause in the failure message itself. +- **Unit:** the placeholder passes `isSafeSignatureField` and produces a valid commit object, + since the display form contains spaces and parentheses. + +## Relationship to the open defect + +This does **not** fix [attribution-loss.md](../debug/attribution-loss.md); it makes it +legible. Both are worth doing, and this one is worth doing *first*: it turns an invisible +correctness failure into one an operator can see in `git log` without a Prometheus query, and +it removes the ambiguity that made the underlying defect so slow to diagnose. It also gives +that investigation a sharper signal — an `attribution-failed` commit is a labelled instance of the +failure, tied to a specific object and time, rather than a counter increment. diff --git a/docs/design/unattributed-author-placeholder.md b/docs/design/unattributed-author-placeholder.md deleted file mode 100644 index 51c4c5dd..00000000 --- a/docs/design/unattributed-author-placeholder.md +++ /dev/null @@ -1,154 +0,0 @@ -# Name the gap: an explicit author for unattributed commits - -> Design, 2026-07-20. **Not built.** Small change, no API/CRD impact. -> Motivated by [debug/attribution-loss.md](../debug/attribution-loss.md), which took hours -> longer to diagnose than it should have — precisely because this gap has no name. - -## The problem - -When author attribution is **enabled** but no audit fact matches, the commit is authored by -the configured committer: - -~~~go -author := pendingWrite.AuthorUserInfo() -if author.Username == "" { - return &git.CommitOptions{Author: committer, Committer: committer, Signer: signer} -} -~~~ -[commit.go:188-194](../../internal/git/commit.go#L188-L194) - -`committer` is `DefaultCommitterName` = `"GitOps Reverser"` -([types.go:22](../../internal/git/types.go#L22)) — which is **also** the identity every commit -carries in configured-author mode. So one string means two unrelated things: - -| Mode | Fact found? | Author today | -|---|---|---| -| configured-author | n/a | `GitOps Reverser` ← correct and expected | -| attribution | yes | the real actor | -| attribution | **no** | `GitOps Reverser` ← **a silent failure, indistinguishable from row 1** | - -Rows 1 and 3 are byte-identical in Git history. Nothing in the repository, the commit, or a -`git log` distinguishes "this operator does not do attribution" from "this operator does -attribution and just lost someone's name". That ambiguity is not theoretical: it is why the -first reading of the e2e failure was wrong, and why the only way to see the ~7–10% loss today -is a Prometheus counter nobody looks at. - -## The proposal - -When attribution is **enabled** and no fact matches, author the commit as an explicit -placeholder instead of the committer. - -~~~text -Author: unattributed -Committer: GitOps Reverser -~~~ - -Configured-author mode is untouched: it keeps authoring as the committer, because there the -committer genuinely *is* the author. - -| Mode | Fact found? | Author after | -|---|---|---| -| configured-author | n/a | `GitOps Reverser` | -| attribution | yes | the real actor | -| attribution | **no** | **`unattributed`** ← now says what happened | - -### Why this shape - -**Use Git's author/committer split for what it is for.** The operator really did make the -commit, so it stays the committer. The author slot answers "on whose behalf", and the honest -answer is "we do not know". Overwriting the committer would lose the operator's identity for -no gain. - -**`.invalid` is reserved (RFC 2606).** `unattributed@gitops-reverser.invalid` can never -collide with a real address, never routes mail, and is recognisable at a glance. Note -`authorEmail` already synthesises `ConstructSafeEmail(username, "cluster.local")` for users -with no email claim, so a synthetic address is established practice here — but `.invalid` is -the stronger signal for a value that is deliberately not a person. - -**Lowercase, no display name.** Real actors get an OIDC display name; the placeholder should -not look like one. `unattributed` sorts and greps distinctly, and `git shortlog -sn` surfaces -it as its own line — which is the point: the loss becomes a number a human sees without -querying anything. - -## What it does not change - -**Commit-window grouping is unaffected in shape.** `canAppend` matches on -`e.UserInfo.Username == w.Author` ([open_window.go:57](../../internal/git/open_window.go#L57)). -Today unattributed events all carry `Username == ""` and therefore group with each other and -not with attributed ones. With a sentinel they still all share one identity and still group -with each other and not with attributed ones. One equivalence class is renamed; none is split -or merged. This was the interaction most likely to make the change risky, and it does not. - -**CommitRequest attribution is unaffected** — it sets a named author explicitly, so it never -reaches the placeholder path. - -**No CRD or API change.** The value never appears in a spec or status. - -## The cost, stated plainly - -This changes what lands in users' Git history. Three real consequences: - -1. **Tooling that keys on the author string.** Anyone matching `GitOps Reverser` to mean "the - operator wrote this" will now miss unattributed commits. That is arguably a *fix* — those - commits were mislabelled — but it is still a behaviour change on shipped data. -2. **It makes an existing defect visible.** With ~7–10% loss unfixed, adopters will suddenly - see `unattributed` in their history and reasonably file bugs. That is the intent, and it - should be release-noted rather than discovered. -3. **Signed commits** carry the author in the signature payload. The placeholder must pass - `isSafeSignatureField` — it does (ASCII, no angle brackets, no control characters). - -### Should it be configurable? - -**Recommendation: yes, one boolean, defaulting to the new behaviour.** - -~~~text ---author-attribution-placeholder=true # default: author unattributed commits as `unattributed` ---author-attribution-placeholder=false # legacy: author them as the committer -~~~ - -Default-on because silently mislabelling authorship is the bug being fixed, and a flag that -defaults to the broken behaviour fixes nothing. The escape hatch exists for an operator whose -downstream tooling cannot absorb the change mid-stream; it is a migration aid, not a -long-term choice, and should say so in its flag help. - -Rejected alternative: make the placeholder *string* configurable. It invites every deployment -inventing its own value, which destroys the one property that matters — that everyone -recognises it. - -## Implementation sketch - -1. `internal/git/types.go` — add `UnattributedAuthorName = "unattributed"` and - `UnattributedAuthorEmail = "unattributed@gitops-reverser.invalid"` next to the committer - constants, so the three identities are readable in one place. -2. `internal/watch/target_watch.go` — in `attachAuthor`, when the resolver is non-nil (i.e. - attribution is enabled) and `ResolveAuthor` returns `ok=false`, stamp the placeholder - `UserInfo` instead of leaving it zero. The resolver stays untouched; it already reports - `AttributionAbsent`. -3. `internal/git/commit.go` — the `author.Username == ""` branch then means only - "configured-author mode", which is exactly what it should have meant all along. Tighten - its comment accordingly. -4. Flag plumbing in `cmd/main.go` alongside `--author-attribution`. - -The placeholder is stamped at the **watch** layer rather than the commit layer on purpose: the -watch layer is the only place that knows attribution was *attempted and failed*, as opposed to -never attempted. - -## Tests - -- **Unit:** attribution enabled + no fact → author is the placeholder, committer unchanged; - attribution disabled + no fact → author is the committer (unchanged); fact found → real - actor (unchanged). -- **Unit:** two unattributed events land in one commit window (the equivalence class survives - the rename); an unattributed and an attributed event do not. -- **e2e:** the existing author assertions gain a third expectation. Today a lost attribution - fails as `expected jane@acme.com, got GitOps Reverser` — ambiguous. After, it fails as - `got unattributed`, which names the cause in the failure message itself. - -## Relationship to the open defect - -This does **not** fix [attribution-loss.md](../debug/attribution-loss.md); it makes it -legible. Both are worth doing, and this one is worth doing *first*: it turns an invisible -correctness failure into one an operator can see in `git log` without a Prometheus query, and -it removes the ambiguity that made the underlying defect so slow to diagnose. It also gives -that investigation a sharper signal — an `unattributed` commit is a labelled instance of the -failure, tied to a specific object and time, rather than a counter increment. From 613d57b636836e9305b9ea1415afb83bae355660 Mon Sep 17 00:00:00 2001 From: Simon Koudijs Date: Mon, 20 Jul 2026 12:34:50 +0000 Subject: [PATCH 08/17] docs(design): carry attribution outcome explicitly, not in the author string MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Revised after review. Every point raised was verified against the code and every one held; two would have shipped silent regressions. The structural change: stop encoding attribution outcome in the author string. Add an explicit AttributionOutcome (not_attempted | resolved | unresolved) on the event and pending write, and let the git author identity, the author_kind metric, and CommitRequest window matching all READ it. The first draft inferred outcome from UserInfo.Username, which is load-bearing in three places it did not account for. HIGH — CommitRequest attachment. The draft claimed this was untouched; it is not. A fallback CommitRequest attaches with Author == "" and matchesWindow compares the author string exactly, so today it coincides with an unattributed window's empty author and attaches. A naive sentinel breaks that coincidence and the request silently lands as its own commit. Now matched on outcome, which also turns today's accidental match-on-empty-string into stated behaviour. Integration test required, not just a unit test. HIGH — author_kind. authorKind() classifies by string: not empty and not a serviceaccount prefix means "user". A sentinel username would therefore be counted as a real named actor, so failed attribution would render in dashboards as IMPROVED user attribution — actively masking the defect rather than merely staying silent. Adds `unresolved` as a fourth kind, driven by the outcome, with metric docs and dashboards updated in the same change. "failed" -> "unresolved". AttributionAbsent collapses no-fact-existed (correct behaviour), cancelled waits, Redis read errors and malformed values — matchFactKey discards the error deliberately. "Failed" asserts a fault we cannot prove. "Unresolved" says we tried and did not arrive at an actor, which is all the evidence supports; a genuine `failed` state can be split out later if the index grows typed outcomes. Resolver contract. ResolveAuthor documents ok=false as "commit as the configured committer". Returning a placeholder instead would overturn that contract while leaving its documentation intact — the exact failure mode this investigation was caused by. The signature returns the outcome explicitly instead, which also removes the unsound "non-nil resolver means attribution is configured" inference. Documentation inventory. architecture.md ("the author identity is never invented"), configuration.md, attribution-setup-guide.md, interpreting-metrics.md and UPGRADING.md all promise the committer fallback and must move together. Also notes Username reaches user-authored per-event templates, not only grouped messages, so that surface needs compatibility coverage. Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/design/attribution-failed-author.md | 190 -------------- docs/design/attribution-outcome-and-author.md | 242 ++++++++++++++++++ 2 files changed, 242 insertions(+), 190 deletions(-) delete mode 100644 docs/design/attribution-failed-author.md create mode 100644 docs/design/attribution-outcome-and-author.md diff --git a/docs/design/attribution-failed-author.md b/docs/design/attribution-failed-author.md deleted file mode 100644 index a0e9df39..00000000 --- a/docs/design/attribution-failed-author.md +++ /dev/null @@ -1,190 +0,0 @@ -# Name the gap: an explicit author when attribution fails - -> Design, 2026-07-20. **Not built.** Small change, no API/CRD impact. -> Motivated by [debug/attribution-loss.md](../debug/attribution-loss.md), which took hours -> longer to diagnose than it should have — precisely because this gap has no name. - -## The problem - -When author attribution is **enabled** but no audit fact matches, the commit is authored by -the configured committer: - -~~~go -author := pendingWrite.AuthorUserInfo() -if author.Username == "" { - return &git.CommitOptions{Author: committer, Committer: committer, Signer: signer} -} -~~~ -[commit.go:188-194](../../internal/git/commit.go#L188-L194) - -`committer` is `DefaultCommitterName` = `"GitOps Reverser"` -([types.go:22](../../internal/git/types.go#L22)) — which is **also** the identity every commit -carries in configured-author mode. So one string means two unrelated things: - -| Mode | Fact found? | Author today | -|---|---|---| -| configured-author | n/a | `GitOps Reverser` ← correct and expected | -| attribution | yes | the real actor | -| attribution | **no** | `GitOps Reverser` ← **a silent failure, indistinguishable from row 1** | - -Rows 1 and 3 are byte-identical in Git history. Nothing in the repository, the commit, or a -`git log` distinguishes "this operator does not do attribution" from "this operator does -attribution and just lost someone's name". That ambiguity is not theoretical: it is why the -first reading of the e2e failure was wrong, and why the only way to see the ~7–10% loss today -is a Prometheus counter nobody looks at. - -## The proposal - -When attribution is **enabled** and no fact matches, author the commit as an explicit -placeholder instead of the committer. Always — there is no flag (see -[below](#no-flag)). - -~~~text -Author: unknown (attribution failed) -Committer: GitOps Reverser -~~~ - -Configured-author mode is untouched: it keeps authoring as the committer, because there the -committer genuinely *is* the author. - -| Mode | Fact found? | Author after | -|---|---|---| -| configured-author | n/a | `GitOps Reverser` | -| attribution | yes | the real actor | -| attribution | **no** | **`unknown (attribution failed)`** ← now says what happened | - -### The three slots, and why each string differs - -`UserInfo` feeds three distinct places, which is easy to miss and is why one string will not -do: - -| Field | Surfaces in | Value | -|---|---|---| -| `Username` | window grouping key ([open_window.go:57](../../internal/git/open_window.go#L57)), the **grouped commit message body** ([commit.go:96](../../internal/git/commit.go#L96)), and the author-name fallback | `attribution-failed` | -| `DisplayName` | the git author Name, via `authorName` | `unknown (attribution failed)` | -| `Email` | the git author Email, via `authorEmail` | `attribution-failed@gitops-reverser.invalid` | - -`Username` lands **inside commit message bodies**, so it has to stand alone there — a bare -`unknown` in a commit message says nothing, whereas `attribution-failed` names the event. -`DisplayName` is what a human reads in `git log`, so it gets the fuller phrasing. The split -mirrors the existing OIDC-display-name-vs-Kubernetes-username pattern rather than inventing a -convention. - -### Why this wording - -**Say it FAILED, not that it is absent.** `unattributed` (the first draft) is too passive — it -reads as "this operator does not do attribution", which is exactly the confusion with -configured-author mode that the change exists to remove. The reader must be able to tell that -the operator *tried and did not manage it*. That is a defect they should investigate, not a -configuration they chose. - -**`unknown` first, mechanism second.** `unknown (attribution failed)` leads with the fact a -human cares about — we do not know who — and follows with why. `unknown-attribution-failed` -as one token is slightly redundant (a failure implies unknown) and reads as a machine -identifier in the one slot that is meant for humans. - -**Use Git's author/committer split for what it is for.** The operator really did make the -commit, so it stays the committer. The author slot answers "on whose behalf", and the honest -answer is "we do not know". Overwriting the committer would lose the operator's identity for -no gain. - -**`.invalid` is reserved (RFC 2606).** `attribution-failed@gitops-reverser.invalid` can never -collide with a real address, never routes mail, and greps cleanly. `authorEmail` already -synthesises `ConstructSafeEmail(username, "cluster.local")` for users with no email claim, so -a synthetic address is established practice here — but `.invalid` is the stronger signal for a -value that is deliberately not a person. - -**Parentheses are safe in a signature.** `isSafeSignatureField` rejects control characters and -the angle brackets that delimit the email; spaces and parens pass, so the display form is -valid in a commit object and in a signed payload. - -`git shortlog -sn` then surfaces the loss as its own line with a count — the point being that -it becomes a number a human sees without querying anything. - -## What it does not change - -**Commit-window grouping is unaffected in shape.** `canAppend` matches on -`e.UserInfo.Username == w.Author` ([open_window.go:57](../../internal/git/open_window.go#L57)). -Today attribution-failed events all carry `Username == ""` and therefore group with each other and -not with attributed ones. With a sentinel they still all share one identity and still group -with each other and not with attributed ones. One equivalence class is renamed; none is split -or merged. This was the interaction most likely to make the change risky, and it does not. - -**CommitRequest attribution is unaffected** — it sets a named author explicitly, so it never -reaches the placeholder path. - -**No CRD or API change.** The value never appears in a spec or status. - -## The cost, stated plainly - -This changes what lands in users' Git history. Three real consequences: - -1. **Tooling that keys on the author string.** Anyone matching `GitOps Reverser` to mean "the - operator wrote this" will now miss the commits where attribution failed. That is arguably a *fix* — those - commits were mislabelled — but it is still a behaviour change on shipped data. -2. **It makes an existing defect visible.** With ~7–10% loss unfixed, adopters will suddenly - see `unknown (attribution failed)` in their history and reasonably file bugs. That is the - intent, and with no opt-out flag it is unavoidable — so it MUST be release-noted, with a - pointer to the metric and to docs/debug/attribution-loss.md, rather than discovered. -3. **Signed commits** carry the author in the signature payload. The placeholder must pass - `isSafeSignatureField` — it does (ASCII, no angle brackets, no control characters). - -### No flag - -**Rejected: making this configurable.** An earlier draft proposed -`--author-attribution-placeholder`, defaulting on. Dropped, and the reasoning is worth -keeping: the only thing such a flag can do is restore *silently mislabelled authorship*. That -is not a preference to be preserved behind an opt-out — it is the bug. A knob whose "off" -position reinstates a correctness defect is a knob nobody should be given, and shipping one -would imply the old behaviour is a legitimate choice. - -Attribution is already gated by `--author-attribution`. An operator who does not want actor -names in Git turns that off and gets configured-author mode, where the committer genuinely is -the author and no placeholder ever appears. That is the real, coherent choice; a second flag -would only add an incoherent third state — attribution on, but lie about it when it fails. - -Also rejected: making the placeholder *string* configurable. It invites every deployment -inventing its own value, which destroys the one property that matters — that everyone -recognises it. - -## Implementation sketch - -1. `internal/git/types.go` — add the placeholder identity next to the committer constants, so - all three identities (committer, real actor, failed) are readable in one place: - `AttributionFailedUsername = "attribution-failed"`, - `AttributionFailedDisplayName = "unknown (attribution failed)"`, - `AttributionFailedEmail = "attribution-failed@gitops-reverser.invalid"`. -2. `internal/watch/target_watch.go` — in `attachAuthor`, when the resolver is non-nil (i.e. - attribution is enabled) and `ResolveAuthor` returns `ok=false`, stamp the placeholder - `UserInfo` instead of leaving it zero. The resolver stays untouched; it already reports - `AttributionAbsent`. -3. `internal/git/commit.go` — the `author.Username == ""` branch then means only - "configured-author mode", which is exactly what it should have meant all along. Tighten - its comment accordingly. -No flag, so no `cmd/main.go` change. - -The placeholder is stamped at the **watch** layer rather than the commit layer on purpose: the -watch layer is the only place that knows attribution was *attempted and failed*, as opposed to -never attempted. - -## Tests - -- **Unit:** attribution enabled + no fact → author is the placeholder, committer unchanged; - attribution disabled + no fact → author is the committer (unchanged); fact found → real - actor (unchanged). -- **Unit:** two attribution-failed events land in one commit window (the equivalence class - survives the rename); an attribution-failed and an attributed event do not. -- **e2e:** the existing author assertions gain a third expectation. Today a lost attribution - fails as `expected jane@acme.com, got GitOps Reverser` — ambiguous. After, it fails as - `got unknown (attribution failed)`, which names the cause in the failure message itself. -- **Unit:** the placeholder passes `isSafeSignatureField` and produces a valid commit object, - since the display form contains spaces and parentheses. - -## Relationship to the open defect - -This does **not** fix [attribution-loss.md](../debug/attribution-loss.md); it makes it -legible. Both are worth doing, and this one is worth doing *first*: it turns an invisible -correctness failure into one an operator can see in `git log` without a Prometheus query, and -it removes the ambiguity that made the underlying defect so slow to diagnose. It also gives -that investigation a sharper signal — an `attribution-failed` commit is a labelled instance of the -failure, tied to a specific object and time, rather than a counter increment. diff --git a/docs/design/attribution-outcome-and-author.md b/docs/design/attribution-outcome-and-author.md new file mode 100644 index 00000000..a0a577e2 --- /dev/null +++ b/docs/design/attribution-outcome-and-author.md @@ -0,0 +1,242 @@ +# Make attribution outcome explicit, and name the gap in Git history + +> Design, 2026-07-20, **revised after review**. Not built. No CRD/API change. +> Motivated by [debug/attribution-loss.md](../debug/attribution-loss.md). +> +> **The revision is structural, not cosmetic.** The first draft stamped a sentinel author +> string and claimed nothing else was affected. Review showed that string is load-bearing in +> three places that would silently break. The fix is to stop encoding attribution outcome in +> the author string at all: carry it as an **explicit value**, and let the author string be a +> *rendering* of it. + +## The problem + +When attribution is enabled but no fact matches, the commit is authored by the configured +committer — the same `"GitOps Reverser"` identity used in configured-author mode +([commit.go:188-194](../../internal/git/commit.go#L188-L194), +[types.go:22](../../internal/git/types.go#L22)). + +| Mode | Fact matched? | Author today | +|---|---|---| +| configured-author | n/a | `GitOps Reverser` ← correct | +| attribution | yes | the real actor | +| attribution | **no** | `GitOps Reverser` ← **silent, indistinguishable from row 1** | + +Rows 1 and 3 are byte-identical in Git history. That ambiguity is why the ~7–10% loss took so +long to diagnose, and why the only way to see it today is a counter nobody reads. + +## The core change: an explicit outcome + +**Add an attribution outcome to the event, and derive everything from it.** This is the part +the first draft got wrong by trying to infer outcome from `UserInfo.Username`. + +~~~go +type AttributionOutcome string + +const ( + AttributionNotAttempted AttributionOutcome = "not_attempted" // configured-author mode + AttributionResolved AttributionOutcome = "resolved" // a fact named the actor + AttributionUnresolved AttributionOutcome = "unresolved" // attempted, no usable fact +) +~~~ + +Carried on `git.Event` beside `UserInfo`, and on `PendingWrite`. Three consumers then read the +outcome instead of sniffing a string: + +1. **Git author rendering** — `unresolved` renders the sentinel identity (below). +2. **`author_kind` metric** — gets its own value, so a failure never counts as a success. +3. **CommitRequest window matching** — matches on outcome, not on an author string that now + varies. + +This also removes the need to guess "is attribution configured?" from a non-nil resolver, +which the first draft did and which is not a sound proxy. + +### Why "unresolved", not "failed" + +The first draft said `attribution failed`. That overclaims. `AttributionAbsent` collapses +several genuinely different situations: no fact was ever produced (correct — not every change +has an audited human actor), a cancelled wait, a Redis read error, and a malformed value. All +of them return the same "not found" from `matchFactKey` +([attribution_index.go:404-412](../../internal/queue/attribution_index.go#L404-L412)), which +discards the error deliberately. + +So the honest word is **unresolved**: we attempted attribution and did not arrive at an actor. +It still tells a reader something is worth investigating without asserting a fault we cannot +prove. If typed outcomes are added to the index later (distinguishing "no fact existed" from +"lookup errored"), a genuine `failed` state can be split out then — and the outcome enum is +the right place for it. + +## The sentinel identity + +| Field | Surfaces in | Value | +|---|---|---| +| `Username` | window grouping, **grouped commit message body** ([commit.go:96](../../internal/git/commit.go#L96)), **per-event custom templates** ([commit.go:21-33](../../internal/git/commit.go#L21-L33)), author-name fallback | `attribution-unresolved` | +| `DisplayName` | git author Name via `authorName` | `unknown (attribution unresolved)` | +| `Email` | git author Email via `authorEmail` | `attribution-unresolved@gitops-reverser.invalid` | + +~~~text +Author: unknown (attribution unresolved) +Committer: GitOps Reverser +~~~ + +The operator really did make the commit, so it stays the **committer**; the **author** slot +answers "on whose behalf", and the honest answer is "we do not know". `.invalid` is reserved +(RFC 2606), so the address can never collide with a real one. Parentheses and spaces pass +`isSafeSignatureField`, so the display form is valid in a commit object and a signed payload. + +**`Username` reaches user-authored templates.** It is interpolated into `EventTemplate` as +`{{.Username}}`, not only into grouped messages. Anyone whose template renders the author will +see `attribution-unresolved` appear. That is the intent, but it is a visible change to +user-configured output and belongs in the release note. + +## HIGH — CommitRequest attachment must not silently stop matching + +A CommitRequest that could not attribute its own author falls back to `Author == ""` +([commitrequest_controller.go:147-151](../../internal/controller/commitrequest_controller.go#L147-L151)), +and attachment matches the open window by **exact author string** +([commit_request_attach.go:106-114](../../internal/git/commit_request_attach.go#L106-L114)): + +~~~go +return p.author == w.Author && p.gitTargetName == w.GitTarget && ... +~~~ + +**Today** an unresolved live window also has `Author == ""`, so the two coincide and the +request attaches. **After a naive sentinel change** the window carries +`attribution-unresolved` while the request still carries `""` — they stop matching, and the +CommitRequest silently lands as a separate commit instead of naming the window it was meant +to. That is a real regression, and the first draft asserting "CommitRequest is unaffected" was +simply wrong. + +**Decision: match on outcome, not on the string.** Both sides carry an `AttributionOutcome`; +a request whose own attribution is `unresolved` matches a window whose attribution is +`unresolved`, for the same GitTarget. Concretely, `matchesWindow` compares +`(outcome, author, gitTarget, gitTargetNamespace)` where the author comparison is only +meaningful when both sides are `resolved`. + +This is strictly better than today's accidental match-on-empty-string: two *different* +unattributed actors currently coalesce into one window because both are `""`, and they still +will — but now that behaviour is stated rather than emergent, and it is visible in the +outcome rather than hidden in an empty field. + +**Requires an integration test**, not just a unit test: a fallback CommitRequest must attach +to an unresolved live window and produce one commit, before and after this change. + +## HIGH — `author_kind` must not report a failure as a success + +`authorKind()` classifies by string +([pending_writes.go:251-260](../../internal/git/pending_writes.go#L251-L260)): empty → +`committer`, `system:serviceaccount:` prefix → `serviceaccount`, otherwise → **`user`**. + +A sentinel username is none of those, so it would be classified **`user`** — and +[interpreting-metrics.md](../interpreting-metrics.md) defines `user` as a real named actor and +says "unattributed events use `committer`". **Failed attribution would appear in dashboards as +improved user attribution**, which is worse than the silence it replaces: it would actively +mask the defect, and could show attribution *improving* as it degrades. + +**Add `unresolved` as a fourth `author_kind`**, driven by the outcome rather than by string +sniffing. Requires, together, in one change: + +- `authorKind()` reads `AttributionOutcome`. +- `docs/interpreting-metrics.md` documents the new value and corrects the "unattributed events + use `committer`" sentence. +- Any dashboard or alert keying on `author_kind` — a `user`-vs-`committer` ratio panel would + otherwise misread the new value. +- Tests asserting the classification for all four kinds. + +## The resolver contract changes + +`ResolveAuthor` documents `ok=false` as **"commit as the configured committer"** +([author_resolver.go:67-82](../../internal/watch/author_resolver.go#L67-L82)). Returning a +placeholder instead silently overturns that contract while leaving the documentation intact — +the exact class of mismatch this whole investigation was caused by. + +Change the signature to return the outcome explicitly rather than a bool whose meaning is +documented one way and implemented another: + +~~~go +ResolveAuthor(...) (git.UserInfo, AttributionOutcome) +~~~ + +`attachAuthor` then stamps `UserInfo` and outcome together. A nil resolver yields +`not_attempted`; a resolver that ran and found nothing yields `unresolved`. This is also what +removes the unsound "non-nil resolver means attribution is configured" inference. + +## Documentation to update in the same change + +The committer-fallback promise is stated in several places; leaving any of them would make the +docs lie in the way this design exists to prevent: + +- [architecture.md](../architecture.md) — "The author identity is never invented". A sentinel + **is** an invented identity, so this must be rewritten to say the author is never + *fabricated as a person*: an unresolved attribution is labelled unresolved, not attributed + to someone. +- [configuration.md](../configuration.md) — the `--author-attribution` description. +- [attribution-setup-guide.md](../attribution-setup-guide.md) — the fallback behaviour users + are told to expect. +- [interpreting-metrics.md](../interpreting-metrics.md) — `author_kind`, as above. +- [UPGRADING.md](../UPGRADING.md) — mandatory release note (below). + +## What genuinely does not change + +- **Configured-author mode.** No resolver runs, outcome is `not_attempted`, author remains the + committer. +- **Resolved attribution.** Unchanged in every respect. +- **Window grouping shape.** Unresolved events share one identity and group with each other, + not with attributed ones — exactly as they do today via `""`. One equivalence class is + renamed, none split or merged. +- **No CRD or API change.** The outcome is internal; it never appears in a spec or status. + +## Cost + +1. **Git history changes.** Tooling matching `GitOps Reverser` to mean "the operator wrote + this" will no longer see unresolved commits. Arguably a fix — those commits were + mislabelled — but a behaviour change on shipped data. +2. **User templates change.** `{{.Username}}` in a custom `EventTemplate` renders the sentinel. +3. **The defect becomes visible with no opt-out.** Adopters will see these commits appear and + reasonably file bugs. Intended — but it makes the release note **mandatory**, pointing at + `gitopsreverser_attribution_resolutions_total` and at + [debug/attribution-loss.md](../debug/attribution-loss.md). + +### No flag + +The only thing a flag could do is restore silently mislabelled authorship. That is not a +preference to preserve behind an opt-out — it is the bug. Attribution is already gated by +`--author-attribution`: an operator who does not want actor names turns that off and gets +configured-author mode, where the committer genuinely is the author. A second flag would add +an incoherent third state — attribution on, but lie when it fails. Making the placeholder +*string* configurable is likewise rejected: it destroys the one property that matters, that +everyone recognises it. + +## Implementation order + +1. `AttributionOutcome` type + the three sentinel constants in `internal/git`. +2. `ResolveAuthor` returns the outcome; `attachAuthor` stamps both. +3. `Event` / `PendingWrite` carry the outcome. +4. `authorKind()` reads the outcome; metrics docs updated together. +5. `matchesWindow` matches on outcome; **integration test** for fallback-CommitRequest + attachment. +6. Commit author rendering reads the outcome. +7. Documentation sweep (list above) + release note. + +Steps 4 and 5 are the ones that must not be deferred: each is a silent behavioural regression +on its own. + +## Tests + +- **Unit:** all three outcomes → correct author and committer; `author_kind` for all four + kinds; sentinel passes `isSafeSignatureField` and yields a valid commit object. +- **Unit:** two unresolved events coalesce into one window; an unresolved and a resolved event + do not. +- **Integration:** a fallback CommitRequest attaches to an unresolved live window and yields + **one** commit — the regression this design exists to avoid. +- **Compatibility:** a custom `EventTemplate` rendering `{{.Username}}` produces the sentinel + rather than an empty string. +- **e2e:** author assertions fail as `got unknown (attribution unresolved)`, naming the cause + in the failure message. + +## Relationship to the open defect + +This does **not** fix [debug/attribution-loss.md](../debug/attribution-loss.md); it makes it +legible and gives the investigation a labelled instance tied to an object and time rather than +a counter increment. Worth doing first for that reason — and the explicit outcome is what the +eventual fix will be measured against. From ee140aa3a9afa378214eed1c720ad6c39d565491 Mon Sep 17 00:00:00 2001 From: Simon Koudijs Date: Mon, 20 Jul 2026 12:57:12 +0000 Subject: [PATCH 09/17] feat(attribution): carry the outcome explicitly, and name unresolved authors in Git MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Attribution failed silently. When it ran and found no fact, the commit was authored by the configured committer — the same "GitOps Reverser" identity used in configured-author mode — so a lost actor was byte-identical in Git history to a deployment that simply does not do attribution. The only way to see the ~7-10% loss was a Prometheus counter nobody reads. Implements docs/design/attribution-outcome-and-author.md. The change is NOT just a sentinel string. Attribution outcome is now an explicit value carried on Event and PendingWrite: not_attempted configured-author mode; the committer legitimately IS the author resolved a fact named the actor unresolved attribution ran and did not arrive at an actor Three consumers read the outcome instead of inferring it from the author string, which is load-bearing in places a naive sentinel would have broken: - Commit author. `unresolved` renders `unknown (attribution unresolved) ` while the committer stays the operator — git's author/committer split used for exactly what it is for. RFC 2606 reserves .invalid, so it can never collide with a real address. - author_kind gains `unresolved`. Classifying it as "user" would have made a lost actor indistinguishable from a named one, so a DEGRADING attribution path would have read in dashboards as an improving one. - CommitRequest attachment matches on outcome. A fallback CommitRequest carries an empty author, which today coincides with an unattributed window's empty author and attaches by accident; once the window carries the sentinel those strings differ and the request would have silently landed as its own commit. Matching on the outcome keeps them together and makes the old accident explicit. ResolveAuthor returns the outcome rather than an ok bool whose documented meaning ("commit as the configured committer") the placeholder would have overturned while leaving the doc intact — the exact class of mismatch that made this investigation slow. It also removes the unsound "non-nil resolver means attribution is configured" inference. Two defects found while implementing, neither in the review: - Adding attributionNotAttempted left setCommitRequestAttributed's switch unhandled, so the AuthorAttributed condition stopped being set at all. - The existing attributionCommitter message claimed "the validate-operator-types webhook is not configured", which is now only true for not-attempted. A configured-but-missing webhook would have sent operators to the wrong place. Both messages are now accurate. Docs moved together, as they must: architecture.md ("the author identity is never invented" -> never fabricated as a PERSON, with both no-actor cases spelled out), configuration.md, attribution-setup-guide.md, interpreting-metrics.md. Verified: e2e attribution report shows the split working — 3 resolutions succeeded after waiting longer than the 3s default grace, and absent waits now separate fast returns (a fact with no author) from grace-ceiling ones (no fact ever arrived), which the old combined view could not distinguish. Co-Authored-By: Claude Opus 4.8 (1M context) --- .coverage-baseline | 2 +- docs/architecture.md | 22 +- docs/attribution-setup-guide.md | 6 + docs/configuration.md | 10 +- docs/interpreting-metrics.md | 13 +- .../controller/commitrequest_controller.go | 5 +- internal/controller/commitrequest_finalize.go | 32 ++- internal/git/attribution_outcome_test.go | 234 ++++++++++++++++++ internal/git/commit.go | 9 + internal/git/commit_request_attach.go | 25 +- internal/git/commit_request_attach_loop.go | 1 + internal/git/open_window.go | 9 +- internal/git/pending_writes.go | 21 ++ internal/git/types.go | 59 +++++ internal/watch/author_resolver.go | 39 +-- internal/watch/author_resolver_test.go | 60 +++-- internal/watch/target_watch.go | 12 +- 17 files changed, 511 insertions(+), 48 deletions(-) create mode 100644 internal/git/attribution_outcome_test.go diff --git a/.coverage-baseline b/.coverage-baseline index 2ba01570..f9f577b8 100644 --- a/.coverage-baseline +++ b/.coverage-baseline @@ -1 +1 @@ -76.6 +76.7 diff --git a/docs/architecture.md b/docs/architecture.md index f56fabc0..30b7ba9e 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -625,12 +625,22 @@ and the *author* (who wrote the change) — and GitOps Reverser uses both on pur to `GitOps Reverser `. Every commit, attributed or not, is committed by the operator, because the operator is what actually wrote it to Git. * The **author is the real actor — but only when we are sure.** On a strong attribution match the author is - set to that actor; with no confident match the author is set to the operator too. Git always carries an - author, so it is **never left blank** — when we are not sure it is simply the operator (identical to the - committer), never a guessed person. A commit whose author differs from the committer is therefore a - positive statement that the operator is confident who made the change. - -The author identity is never invented — it is taken from the authenticated request. The name and email come + set to that actor. Git always carries an author, so it is **never left blank**, and it is never a guessed + person. What fills it when we are not sure depends on WHY: + * **Attribution is switched off** (configured-author mode, and reconcile/resync writes that have no actor + at all): the author is the operator, identical to the committer. That is honest — the operator really is + the author. + * **Attribution ran and did not resolve an actor**: the author is the explicit sentinel + `unknown (attribution unresolved) `. It is deliberately + NOT the operator, because authoring it as the operator made a lost actor byte-identical to a + configured-author commit — so the gap was invisible in Git history and only countable in a metric. See + [docs/design/attribution-outcome-and-author.md](design/attribution-outcome-and-author.md). + + A commit whose author is a **person** is therefore a positive statement that the operator is confident who + made the change; one authored by the sentinel is a positive statement that it tried and could not tell. + +The author identity is never fabricated as a person — a real author is always taken from the authenticated +request, and an unresolved one is labelled as unresolved rather than attributed to anybody. The name and email come from the **OIDC claims** when the apiserver maps them; otherwise they come from the actor's own **Kubernetes identity** — the username, which for a controller is its service account (`system:serviceaccount::`), with a stable derived email under `noreply.cluster.local`. diff --git a/docs/attribution-setup-guide.md b/docs/attribution-setup-guide.md index ae3b9f12..39e28493 100644 --- a/docs/attribution-setup-guide.md +++ b/docs/attribution-setup-guide.md @@ -5,6 +5,12 @@ configured committer identity. **Attribution** turns on a second identity — th user or service account that made the change becomes the commit *author*, while the committer stays constant. Git carries both. +Once attribution is on, a change whose actor cannot be resolved is authored +`unknown (attribution unresolved) ` rather than falling +back to the committer. Seeing that identity in your history means attribution ran and did not find a +matching audit fact — check the audit webhook path before assuming it is normal. Its rate is +`commits_total{author_kind="unresolved"}`. + Attribution is the only optional capability, and it is a real operational step up: it requires kube-apiserver audit delivery and a Redis/Valkey backing store. This guide covers what that costs and how the pieces fit. It assumes GitOps Reverser is already installed and mirroring state — see diff --git a/docs/configuration.md b/docs/configuration.md index a59d2b73..7578f561 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -171,6 +171,12 @@ For mirrored-resource commits, the author comes from the configured committer id `attribution.enabled=true` and a matching kube-apiserver audit event names the Kubernetes user or service account. Snapshot/reconcile commits are operator-authored. +When attribution IS enabled and no matching audit fact arrives, the commit is authored +`unknown (attribution unresolved) ` — **not** the +committer. That distinction is the point: a committer-authored commit means attribution was never +attempted, while the sentinel means it was attempted and did not resolve, which is worth investigating. +Such commits also count under `author_kind="unresolved"` in `commits_total`. + That distinction is useful in practice: - `git log --author=alice` answers "what did Alice change?" @@ -873,7 +879,9 @@ When attribution is enabled, these flags tune the join: - `--author-attribution-ttl` (default `10m`): how long an attribution fact is retained waiting for the matching watch event to join it. - `--author-attribution-grace` (default `3s`): bounded per-event wait for a matching audit fact before a - watch event ships authored by the committer. + watch event ships authored by the `attribution-unresolved` sentinel. Note the delivery floor: the + apiserver's own `--audit-webhook-batch-max-wait` delays every fact by up to that much, so a grace at or + below it will lose actors systematically. A matched actor is always named by its own username — humans and service accounts alike (e.g. `system:serviceaccount:flux-system:kustomize-controller`); there is no option to collapse service diff --git a/docs/interpreting-metrics.md b/docs/interpreting-metrics.md index 496fdcd2..bc9d9907 100644 --- a/docs/interpreting-metrics.md +++ b/docs/interpreting-metrics.md @@ -87,8 +87,17 @@ signals. Background: [architecture.md → Git Write Architecture](architecture.m `commits_total` carries the **`BranchWorker`'s** `{provider_namespace, provider_name, branch, author_kind}` identity, not a GitTarget: one worker can serve several GitTargets sharing a provider+branch, coalescing their writes into one commit batch, so -the worker is the honest attribution unit. `author_kind` is `user`, `serviceaccount`, or `committer`; -reconcile/resync commits and unattributed events use `committer`. The namespace/name keys are +the worker is the honest attribution unit. `author_kind` is `user`, `serviceaccount`, `committer`, or `unresolved`; +reconcile/resync commits and configured-author mode use `committer`. + +**`unresolved` is the one to watch.** It means attribution RAN and did not name an actor, so +the commit carries the `unknown (attribution unresolved)` author instead of a person. It is +deliberately not folded into `user` (which would make a lost actor look like a named one, so a +degrading attribution path would read as an improving one) nor into `committer` (which would +hide it among legitimately unattributed reconcile writes). A non-zero and growing `unresolved` +share is a real defect, not noise — see +[`gitopsreverser_attribution_resolutions_total`](#audit-attribution-optional) and +[docs/debug/attribution-loss.md](debug/attribution-loss.md). The namespace/name keys are **prefixed on purpose** — a Prometheus pod scrape with `honor_labels=false` overwrites a bare `namespace` attribute with the scraping pod's namespace, so a per-provider `namespace` selector would silently match nothing. The same reasoning applies to diff --git a/internal/controller/commitrequest_controller.go b/internal/controller/commitrequest_controller.go index 522d61fd..d4336f13 100644 --- a/internal/controller/commitrequest_controller.go +++ b/internal/controller/commitrequest_controller.go @@ -149,6 +149,7 @@ func (r *CommitRequestReconciler) Reconcile(ctx context.Context, req ctrl.Reques Name: commitRequest.Name, UID: string(commitRequest.UID), Author: author.Author, + Attribution: attribution.gitOutcome(), GitTargetName: commitRequest.Spec.TargetRef.Name, GitTargetNamespace: commitRequest.Namespace, Message: capCommitRequestMessage(commitRequest.Spec.Message), @@ -197,7 +198,9 @@ func (r *CommitRequestReconciler) attributeAuthor( if r.AuthorLookup == nil { log.Info("command-author lookup disabled (validate-operator-types webhook off); committing as committer", "name", client.ObjectKeyFromObject(commitRequest), "uid", commitRequest.UID) - return queue.CommandAuthor{}, attributionCommitter + // Capture is off, so nothing was attempted — distinct from a capture that ran and + // found no record, which is what attributionCommitter now means. + return queue.CommandAuthor{}, attributionNotAttempted } if author, ok := r.AuthorLookup.LookupCommandAuthor(ctx, commitRequest.UID); ok { log.Info("command author resolved from admission record", diff --git a/internal/controller/commitrequest_finalize.go b/internal/controller/commitrequest_finalize.go index 7d197b9e..c9d77268 100644 --- a/internal/controller/commitrequest_finalize.go +++ b/internal/controller/commitrequest_finalize.go @@ -57,8 +57,28 @@ const ( // validate-operator-types webhook is not configured (or did not record one) — so the // commit is authored by the configured committer. attributionCommitter + // attributionNotAttempted means command-author capture is switched off entirely (the + // validate-operator-types webhook is not running), so no submitter was ever sought. It is + // distinct from attributionCommitter, where capture DID run and found nothing. + attributionNotAttempted ) +// gitOutcome projects the controller's attribution state onto the git layer's outcome, which +// is what the worker matches windows on. Keeping the projection here — rather than passing the +// controller's own enum down — means the git package never learns about admission records. +func (a commitRequestAttribution) gitOutcome() git.AttributionOutcome { + switch a { + case attributionFromAdmission: + return git.AttributionResolved + case attributionCommitter: + return git.AttributionUnresolved + case attributionNotAttempted: + return git.AttributionNotAttempted + default: + return git.AttributionNotAttempted + } +} + // setCommitRequestCondition upserts a condition keyed by type, stamping it with // the request's generation so kstatus can tell a current status from a stale one. func setCommitRequestCondition( @@ -111,10 +131,18 @@ func setCommitRequestAttributed(cr *configv1alpha3.CommitRequest, attribution co crReasonAttributedFromAdmission, "the submitter was captured at admission and named as the commit author") case attributionCommitter: + // Capture RAN and found nothing. Distinct from attributionNotAttempted below, whose + // message this used to carry — reporting "the webhook is not configured" for a + // webhook that is configured and simply missed sends operators to the wrong place. + setCommitRequestCondition(cr, ConditionTypeAuthorAttributed, metav1.ConditionFalse, + crReasonCommitterFallback, + "no admission author record was found for this request; committed as the "+ + "configured committer") + case attributionNotAttempted: setCommitRequestCondition(cr, ConditionTypeAuthorAttributed, metav1.ConditionFalse, crReasonCommitterFallback, - "no admission author record (the validate-operator-types webhook is not configured); "+ - "committed as the configured committer") + "command-author capture is disabled (the validate-operator-types webhook is not "+ + "configured); committed as the configured committer") } } diff --git a/internal/git/attribution_outcome_test.go b/internal/git/attribution_outcome_test.go new file mode 100644 index 00000000..a0ab2aff --- /dev/null +++ b/internal/git/attribution_outcome_test.go @@ -0,0 +1,234 @@ +// SPDX-License-Identifier: Apache-2.0 + +package git + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/ConfigButler/gitops-reverser/internal/types" +) + +// attributedEvent builds a commit-shaped event carrying an explicit attribution outcome. +func attributedEvent(username string, outcome AttributionOutcome) Event { + return Event{ + Identifier: types.NewResourceIdentifier("", "v1", "configmaps", "team-a", "app"), + Operation: "CREATE", + UserInfo: UserInfo{Username: username}, + Attribution: outcome, + GitTargetName: "target", + GitTargetNamespace: "default", + } +} + +func commitWrite(events ...Event) PendingWrite { + return PendingWrite{Kind: PendingWriteCommit, Events: events} +} + +// An attribution that RAN and did not resolve must NOT be authored by the committer. Authoring +// it as the committer is what made a lost actor byte-identical to a configured-author commit, +// so the loss was invisible in git history. +func TestCommitOptionsFor_UnresolvedAttributionUsesSentinelAuthor(t *testing.T) { + config := ResolveCommitConfig(nil) + when := time.Now() + + options := commitOptionsFor(commitWrite(attributedEvent("", AttributionUnresolved)), config, nil, when) + + require.NotNil(t, options.Author) + require.NotNil(t, options.Committer) + assert.Equal(t, UnresolvedAuthorDisplayName, options.Author.Name) + assert.Equal(t, UnresolvedAuthorEmail, options.Author.Email) + assert.Equal(t, DefaultCommitterName, options.Committer.Name, + "the operator really did write the commit, so it stays the committer") + assert.NotEqual(t, options.Committer.Name, options.Author.Name, + "an unresolved attribution must be distinguishable from a configured-author commit") +} + +// Configured-author mode is untouched: nothing was attempted, so the committer legitimately +// authors the commit. This is the case that MUST stay identical to the sentinel case's +// opposite — the whole point is that the two are now told apart. +func TestCommitOptionsFor_NotAttemptedStillAuthorsAsCommitter(t *testing.T) { + config := ResolveCommitConfig(nil) + when := time.Now() + + options := commitOptionsFor(commitWrite(attributedEvent("", AttributionNotAttempted)), config, nil, when) + + require.NotNil(t, options.Author) + assert.Equal(t, DefaultCommitterName, options.Author.Name) + assert.Equal(t, DefaultCommitterEmail, options.Author.Email) + assert.Equal(t, options.Committer.Name, options.Author.Name) +} + +func TestCommitOptionsFor_ResolvedAttributionNamesTheActor(t *testing.T) { + config := ResolveCommitConfig(nil) + + options := commitOptionsFor( + commitWrite(attributedEvent("jane@acme.com", AttributionResolved)), config, nil, time.Now()) + + require.NotNil(t, options.Author) + assert.Equal(t, "jane@acme.com", options.Author.Name) + assert.NotEqual(t, UnresolvedAuthorDisplayName, options.Author.Name) +} + +// The sentinel is written into a git signature header, so it must survive the same safety +// check every other author value does — spaces and parentheses included. +func TestUnresolvedAuthor_IsSafeInASignatureHeader(t *testing.T) { + user := UnresolvedAuthor() + assert.True(t, isSafeSignatureField(user.DisplayName), + "the display form must be placeable verbatim in a git signature") + assert.Equal(t, UnresolvedAuthorDisplayName, authorName(user)) + assert.Equal(t, UnresolvedAuthorEmail, authorEmail(user), + "the reserved .invalid address must pass email validation and be used verbatim") +} + +// author_kind drives dashboards. An unresolved attribution must never be counted as a named +// user: that would make a DEGRADING attribution path read as an improving one. +func TestAuthorKind_ClassifiesAllFourKinds(t *testing.T) { + for _, tc := range []struct { + name string + write PendingWrite + want string + }{ + { + name: "resolved human is a user", + write: commitWrite(attributedEvent("jane@acme.com", AttributionResolved)), + want: authorKindUser, + }, + { + name: "resolved service account is a serviceaccount", + write: commitWrite(attributedEvent("system:serviceaccount:flux-system:kustomize", AttributionResolved)), + want: authorKindServiceAccount, + }, + { + name: "attribution never attempted is the committer", + write: commitWrite(attributedEvent("", AttributionNotAttempted)), + want: authorKindCommitter, + }, + { + name: "attribution attempted and unresolved is its own kind", + write: commitWrite(attributedEvent("", AttributionUnresolved)), + want: authorKindUnresolved, + }, + { + name: "an atomic write never attempts attribution", + write: PendingWrite{Kind: PendingWriteAtomic}, + want: authorKindCommitter, + }, + } { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.want, tc.write.authorKind()) + }) + } +} + +// Even if a sentinel username ever reached authorKind through a path that lost the outcome, +// it must not be silently counted as a real user. This pins the intent rather than the +// mechanism, so a future refactor that drops the outcome check fails here. +func TestAuthorKind_SentinelUsernameIsNeverCountedAsAUser(t *testing.T) { + write := commitWrite(attributedEvent(UnresolvedAuthorUsername, AttributionUnresolved)) + assert.Equal(t, authorKindUnresolved, write.authorKind()) + assert.NotEqual(t, authorKindUser, write.authorKind()) +} + +// Window coalescing is keyed on (outcome, author): two unresolved events belong together +// exactly as they did when both carried an empty username, and an unresolved event must never +// join a resolved one. +func TestOpenWindow_CanAppendMatchesOnOutcomeAndAuthor(t *testing.T) { + unresolved := attributedEvent("", AttributionUnresolved) + window := newOpenWindow(unresolved, nil) + + assert.True(t, window.canAppend(attributedEvent("", AttributionUnresolved)), + "two unresolved events share one window") + assert.False(t, window.canAppend(attributedEvent("jane@acme.com", AttributionResolved)), + "a named actor must not be folded into an unresolved window") + assert.False(t, window.canAppend(attributedEvent("", AttributionNotAttempted)), + "configured-author writes must not be folded into an unresolved window") +} + +func TestOpenWindow_ResolvedWindowsStillSplitByAuthor(t *testing.T) { + window := newOpenWindow(attributedEvent("jane@acme.com", AttributionResolved), nil) + + assert.True(t, window.canAppend(attributedEvent("jane@acme.com", AttributionResolved))) + assert.False(t, window.canAppend(attributedEvent("bob@acme.com", AttributionResolved))) +} + +// THE REGRESSION THIS DESIGN EXISTS TO AVOID. +// +// A CommitRequest that could not be attributed carries an empty author. Before the sentinel, +// that coincided with an unattributed window's empty author and attached by accident. Once the +// window carries the sentinel author the strings differ, so matching on the author alone would +// silently stop attaching and the request would land as its own commit. Matching on the +// outcome keeps them together. +func TestPendingCommitRequest_UnresolvedRequestAttachesToUnresolvedWindow(t *testing.T) { + window := newOpenWindow(attributedEvent("", AttributionUnresolved), nil) + window.Author = UnresolvedAuthorUsername // as the writer will render it + + request := &pendingCommitRequest{ + author: "", // a fallback CommitRequest has no author + attribution: AttributionUnresolved, + gitTargetName: "target", + gitTargetNamespace: "default", + } + + assert.True(t, request.matchesWindow(window), + "a fallback CommitRequest must still attach to the unresolved window it was meant for, "+ + "even though the two carry different author strings") +} + +func TestPendingCommitRequest_DoesNotCrossAttributionOutcomes(t *testing.T) { + resolvedWindow := newOpenWindow(attributedEvent("jane@acme.com", AttributionResolved), nil) + + unresolvedRequest := &pendingCommitRequest{ + author: "", + attribution: AttributionUnresolved, + gitTargetName: "target", + gitTargetNamespace: "default", + } + assert.False(t, unresolvedRequest.matchesWindow(resolvedWindow), + "an unattributed request must not claim a named actor's window") + + resolvedRequest := &pendingCommitRequest{ + author: "jane@acme.com", + attribution: AttributionResolved, + gitTargetName: "target", + gitTargetNamespace: "default", + } + assert.True(t, resolvedRequest.matchesWindow(resolvedWindow)) + + otherActor := &pendingCommitRequest{ + author: "bob@acme.com", + attribution: AttributionResolved, + gitTargetName: "target", + gitTargetNamespace: "default", + } + assert.False(t, otherActor.matchesWindow(resolvedWindow), + "a resolved request still matches on the actor") +} + +func TestPendingCommitRequest_StillScopedToOneGitTarget(t *testing.T) { + window := newOpenWindow(attributedEvent("", AttributionUnresolved), nil) + request := &pendingCommitRequest{ + author: "", + attribution: AttributionUnresolved, + gitTargetName: "other-target", + gitTargetNamespace: "default", + } + assert.False(t, request.matchesWindow(window), + "the outcome match must not widen attachment across GitTargets") +} + +// Username reaches user-authored per-event templates via {{.Username}}, not only grouped +// commit messages. A custom template must render the sentinel rather than an empty string. +func TestRenderEventCommitMessage_UnresolvedUsernameReachesCustomTemplates(t *testing.T) { + config := ResolveCommitConfig(nil) + config.Message.EventTemplate = "{{.Operation}} by {{.Username}}" + + event := attributedEvent(UnresolvedAuthorUsername, AttributionUnresolved) + message, err := renderEventCommitMessage(event, config) + + require.NoError(t, err) + assert.Equal(t, "CREATE by "+UnresolvedAuthorUsername, message) +} diff --git a/internal/git/commit.go b/internal/git/commit.go index 9b587ec9..a10fdc27 100644 --- a/internal/git/commit.go +++ b/internal/git/commit.go @@ -186,6 +186,15 @@ func commitOptionsFor( ) *git.CommitOptions { committer := operatorSignature(config, when) author := pendingWrite.AuthorUserInfo() + // An attribution that RAN and did not resolve is authored by the sentinel, not by the + // committer. Authoring it as the committer is what made a lost actor byte-identical to a + // configured-author commit, so the loss was invisible in Git history. + if pendingWrite.AttributionOutcome() == AttributionUnresolved { + author = UnresolvedAuthor() + } + // Reaching here with an empty username now means only "attribution was never attempted" + // (configured-author mode, reconcile/resync writes), where the committer genuinely IS the + // author. if author.Username == "" { return &git.CommitOptions{ Author: committer, diff --git a/internal/git/commit_request_attach.go b/internal/git/commit_request_attach.go index 6ca847d9..a1f35fd1 100644 --- a/internal/git/commit_request_attach.go +++ b/internal/git/commit_request_attach.go @@ -61,6 +61,13 @@ type AttachCommitRequest struct { // matches is attached; this binds "the open window" to "the requesting // author's open window". Author string + // Attribution is the outcome of attributing THIS CommitRequest. It must be matched + // alongside Author, not inferred from it: a request that could not be attributed carries + // an empty Author, which used to coincide with an unattributed window's empty author and + // attach by accident. Once unresolved windows carry a sentinel author that coincidence + // disappears, and matching on the outcome is what keeps the fallback request attaching to + // the window it was meant for instead of silently landing as its own commit. + Attribution AttributionOutcome // GitTargetName / GitTargetNamespace scope the finalize to one GitTarget. GitTargetName string GitTargetNamespace string @@ -93,6 +100,7 @@ func (a AttachCommitRequest) id() commitRequestID { type pendingCommitRequest struct { id commitRequestID author string + attribution AttributionOutcome gitTargetName string gitTargetNamespace string message string @@ -104,11 +112,26 @@ type pendingCommitRequest struct { } // matchesWindow reports whether the request identifies the given open window by -// author and GitTarget. +// attribution outcome, author, and GitTarget. +// +// The outcome is matched FIRST and the author comparison only carries meaning when both sides +// resolved to a real actor. A CommitRequest that could not be attributed carries an empty +// author, and an unresolved window carries the sentinel author — different strings for the +// same situation, so comparing strings alone would stop them matching and the request would +// silently land as a separate commit. Two unresolved parties are treated as belonging together +// exactly as they did when both were the empty string; the difference is that it is now stated +// rather than an accident of two fields both happening to be empty. func (p *pendingCommitRequest) matchesWindow(w *openWindow) bool { if p == nil || w == nil { return false } + if p.attribution != w.Attribution { + return false + } + if p.attribution == AttributionUnresolved { + // Neither side knows who acted; there is no author to agree on. + return p.gitTargetName == w.GitTarget && p.gitTargetNamespace == w.GitTargetNamespace + } return p.author == w.Author && p.gitTargetName == w.GitTarget && p.gitTargetNamespace == w.GitTargetNamespace diff --git a/internal/git/commit_request_attach_loop.go b/internal/git/commit_request_attach_loop.go index 01dd9b81..8ff62a75 100644 --- a/internal/git/commit_request_attach_loop.go +++ b/internal/git/commit_request_attach_loop.go @@ -83,6 +83,7 @@ func (l *branchWorkerEventLoop) handleAttachCommitRequest(req *AttachCommitReque l.pendingCRs[id] = &pendingCommitRequest{ id: id, author: req.Author, + attribution: req.Attribution, gitTargetName: req.GitTargetName, gitTargetNamespace: req.GitTargetNamespace, message: req.Message, diff --git a/internal/git/open_window.go b/internal/git/open_window.go index 517cd75a..189ca8b8 100644 --- a/internal/git/open_window.go +++ b/internal/git/open_window.go @@ -9,6 +9,11 @@ package git type openWindow struct { // Author is event.UserInfo.Username verbatim. Author string + // Attribution is the window's attribution outcome, taken from its first event. It pairs + // with Author for matching: two events share a window only when BOTH the outcome and the + // author agree, so an unresolved event never coalesces with a resolved one that happens + // to carry an empty username. + Attribution AttributionOutcome // GitTarget is the target this window is bound to. Each finalized commit // covers exactly one target so per-target encryption and bootstrap // configuration apply unambiguously. @@ -43,6 +48,7 @@ const groupedCommitOperationKinds = 3 func newOpenWindow(e Event, writer eventContentWriter) *openWindow { return &openWindow{ Author: e.UserInfo.Username, + Attribution: e.Attribution, GitTarget: e.GitTargetName, GitTargetNamespace: e.GitTargetNamespace, pathToEvent: make(map[string]Event), @@ -54,7 +60,8 @@ func (w *openWindow) canAppend(e Event) bool { if w == nil { return false } - return e.UserInfo.Username == w.Author && + return e.Attribution == w.Attribution && + e.UserInfo.Username == w.Author && e.GitTargetName == w.GitTarget && e.GitTargetNamespace == w.GitTargetNamespace } diff --git a/internal/git/pending_writes.go b/internal/git/pending_writes.go index e3a9371c..f607e2bf 100644 --- a/internal/git/pending_writes.go +++ b/internal/git/pending_writes.go @@ -239,8 +239,23 @@ const ( authorKindUser = "user" authorKindServiceAccount = "serviceaccount" authorKindCommitter = "committer" + // authorKindUnresolved is attribution that RAN and did not name an actor. It is its own + // kind on purpose: classifying it as "user" would make a lost actor indistinguishable + // from a named one in dashboards, so a degrading attribution path would read as an + // improving one. See docs/interpreting-metrics.md. + authorKindUnresolved = "unresolved" ) +// AttributionOutcome returns the attribution outcome for commit-shaped pending writes. It +// mirrors AuthorUserInfo: the window is single-author, so the first event's outcome describes +// the whole write. Atomic and empty writes never attempt attribution. +func (p PendingWrite) AttributionOutcome() AttributionOutcome { + if p.Kind == PendingWriteAtomic || len(p.Events) == 0 { + return AttributionNotAttempted + } + return p.Events[0].Attribution +} + func (p PendingWrite) createdCommit() bool { if p.Kind == PendingWriteResync { return p.Committed != nil && *p.Committed @@ -248,7 +263,13 @@ func (p PendingWrite) createdCommit() bool { return !p.CommitSHA.IsZero() } +// authorKind classifies a commit's author for the commits_total metric. It reads the +// attribution OUTCOME first: an unresolved attribution is its own kind, never "user", so a +// lost actor can never be counted as a named one. func (p PendingWrite) authorKind() string { + if p.AttributionOutcome() == AttributionUnresolved { + return authorKindUnresolved + } author := p.Author() if author == "" { return authorKindCommitter diff --git a/internal/git/types.go b/internal/git/types.go index 277e665c..c4f2f64f 100644 --- a/internal/git/types.go +++ b/internal/git/types.go @@ -17,6 +17,57 @@ import ( "github.com/ConfigButler/gitops-reverser/internal/types" ) +// AttributionOutcome records what happened when the operator tried to name the actor behind a +// change. It is carried EXPLICITLY rather than inferred from the author identity, because the +// author string is load-bearing in several places (window grouping, commit-message templates, +// the author_kind metric) and overloading it to also mean "attribution failed" made a silent +// failure indistinguishable from correct configured-author behaviour. See +// docs/design/attribution-outcome-and-author.md. +type AttributionOutcome string + +const ( + // AttributionNotAttempted is configured-author mode: attribution is switched off, so the + // committer legitimately IS the author and no actor was ever sought. + AttributionNotAttempted AttributionOutcome = "not_attempted" + // AttributionResolved means an audit fact named the actor. + AttributionResolved AttributionOutcome = "resolved" + // AttributionUnresolved means attribution ran and did not arrive at an actor. + // + // Deliberately "unresolved", not "failed": the lookup collapses several genuinely + // different situations into one miss — no fact was ever produced (correct; not every + // change has an audited human actor), a cancelled wait, a Redis read error, and a + // malformed value all return the same not-found. Calling that a failure would assert a + // fault the operator cannot prove. + AttributionUnresolved AttributionOutcome = "unresolved" +) + +// UnresolvedAuthor is the identity written to Git when attribution ran and did not resolve an +// actor. It exists so an unresolved attribution is visible in `git log` instead of being +// indistinguishable from a configured-author commit. +// +// Three fields, three different strings, because UserInfo feeds three surfaces: +// - Username reaches window grouping, the grouped commit message body, AND user-authored +// per-event templates via {{.Username}} — so it must stand alone as a machine token. +// - DisplayName is what a human reads in `git log`, so it leads with what they care about. +// - Email uses the RFC 2606 reserved .invalid TLD, so it can never collide with a real +// address and never routes mail. +func UnresolvedAuthor() UserInfo { + return UserInfo{ + Username: UnresolvedAuthorUsername, + DisplayName: UnresolvedAuthorDisplayName, + Email: UnresolvedAuthorEmail, + } +} + +const ( + // UnresolvedAuthorUsername is the stable machine token for an unresolved attribution. + UnresolvedAuthorUsername = "attribution-unresolved" + // UnresolvedAuthorDisplayName is the human-facing git author name. + UnresolvedAuthorDisplayName = "unknown (attribution unresolved)" + // UnresolvedAuthorEmail is a reserved-invalid address (RFC 2606). + UnresolvedAuthorEmail = "attribution-unresolved@gitops-reverser.invalid" +) + const ( // DefaultCommitterName matches the default operator identity in Git history. DefaultCommitterName = "GitOps Reverser" @@ -378,6 +429,14 @@ type Event struct { // UserInfo contains user information for commit messages. UserInfo UserInfo + // Attribution records whether naming the actor was attempted and whether it succeeded. + // It is the authority for author rendering, the author_kind metric, and CommitRequest + // window matching — none of which may infer the outcome from UserInfo, because an empty + // or sentinel username cannot distinguish "attribution is off" from "attribution ran and + // found nothing". The zero value is AttributionNotAttempted, which is correct for every + // non-live path (reconcile, resync, bootstrap) where no actor is ever sought. + Attribution AttributionOutcome + // Path is the POSIX-like relative path prefix for this event's files. // This comes from the GitTarget that triggered this event. // Empty string means write to repository root. diff --git a/internal/watch/author_resolver.go b/internal/watch/author_resolver.go index 1f8c000f..ca0eb0d8 100644 --- a/internal/watch/author_resolver.go +++ b/internal/watch/author_resolver.go @@ -66,11 +66,17 @@ type CursorStore interface { // AuthorResolver names the commit author for a live watch event from audit facts. type AuthorResolver interface { - // ResolveAuthor returns the author UserInfo for a watch event, or ok=false to - // commit as the configured committer. It may wait up to the grace window for a - // matching fact; it never blocks indefinitely and never returns an error path — - // an absent fact is a committer commit, not a failure. exactCapable distinguishes + // ResolveAuthor returns the author UserInfo for a watch event together with the + // attribution OUTCOME. It may wait up to the grace window for a matching fact; it never + // blocks indefinitely and never returns an error path. exactCapable distinguishes // ADDED/MODIFIED events (true) from known RV-mismatch removals (false). + // + // The outcome is returned explicitly rather than as an ok bool because the two possible + // "no author" cases are NOT the same and callers must be able to tell them apart: + // AttributionNotAttempted (configured-author mode — the committer legitimately is the + // author) versus AttributionUnresolved (attribution ran and found nothing — a gap worth + // surfacing). An empty UserInfo cannot distinguish them, which is exactly how the loss + // stayed invisible. A resolved outcome always carries a non-empty UserInfo. ResolveAuthor( ctx context.Context, providerName string, @@ -78,7 +84,7 @@ type AuthorResolver interface { uid k8stypes.UID, rv string, exactCapable bool, - ) (git.UserInfo, bool) + ) (git.UserInfo, git.AttributionOutcome) } type attributionResolver struct { @@ -106,47 +112,50 @@ func (r *attributionResolver) ResolveAuthor( uid k8stypes.UID, rv string, exactCapable bool, -) (git.UserInfo, bool) { +) (git.UserInfo, git.AttributionOutcome) { start := time.Now() + // A nil lookup is configured-author mode: attribution was never switched on, so nothing + // was attempted and the committer legitimately authors the commit. if r.lookup == nil { recordAttributionResolution(ctx, gvr, queue.AttributionAbsent, time.Since(start)) - return git.UserInfo{}, false + return git.UserInfo{}, git.AttributionNotAttempted } deadline := time.Now().Add(r.grace) for { resolution := r.lookup.LookupAuthorResolution(ctx, providerName, gvr, uid, rv, exactCapable) if resolution.Result != queue.AttributionAbsent { - ui, ok, result := r.userInfoForResolution(resolution) + ui, outcome, result := r.userInfoForResolution(resolution) recordAttributionResolution(ctx, gvr, result, time.Since(start)) - return ui, ok + return ui, outcome } if !time.Now().Before(deadline) { recordAttributionResolution(ctx, gvr, queue.AttributionAbsent, time.Since(start)) - return git.UserInfo{}, false + return git.UserInfo{}, git.AttributionUnresolved } if !sleepOrDone(ctx, attributionPollInterval) { recordAttributionResolution(ctx, gvr, queue.AttributionAbsent, time.Since(start)) - return git.UserInfo{}, false + return git.UserInfo{}, git.AttributionUnresolved } } } // userInfoForResolution turns a matched fact into a commit author. The matched // actor — human or service account — is always named by its own username; a fact -// with no author falls back to the committer (ok=false). +// that carries no author is UNRESOLVED, not not-attempted: attribution ran, found a +// fact, and still could not name anyone. func (r *attributionResolver) userInfoForResolution( resolution queue.AuthorResolution, -) (git.UserInfo, bool, queue.AttributionResult) { +) (git.UserInfo, git.AttributionOutcome, queue.AttributionResult) { fact := resolution.Fact result := resolution.Result if fact.Author == "" { - return git.UserInfo{}, false, result + return git.UserInfo{}, git.AttributionUnresolved, result } return git.UserInfo{ Username: fact.Author, DisplayName: fact.DisplayName, Email: fact.Email, - }, true, result + }, git.AttributionResolved, result } func recordAttributionResolution( diff --git a/internal/watch/author_resolver_test.go b/internal/watch/author_resolver_test.go index 33e1f780..bf2cb732 100644 --- a/internal/watch/author_resolver_test.go +++ b/internal/watch/author_resolver_test.go @@ -13,6 +13,7 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" k8stypes "k8s.io/apimachinery/pkg/types" + "github.com/ConfigButler/gitops-reverser/internal/git" "github.com/ConfigButler/gitops-reverser/internal/queue" "github.com/ConfigButler/gitops-reverser/internal/telemetry" ) @@ -51,8 +52,8 @@ func TestAuthorResolver_HumanHit(t *testing.T) { } r := NewAuthorResolver(lookup, DefaultAttributionGraceWindow, logr.Discard()) - ui, ok := r.ResolveAuthor(context.Background(), "prod-eu-1", resolverGVR, "uid-1", "101", true) - require.True(t, ok) + ui, outcome := r.ResolveAuthor(context.Background(), "prod-eu-1", resolverGVR, "uid-1", "101", true) + require.Equal(t, git.AttributionResolved, outcome) assert.Equal(t, "alice", ui.Username) assert.Equal(t, "a@x.io", ui.Email) assert.Equal(t, 1, lookup.calls) @@ -75,8 +76,9 @@ func TestAuthorResolver_ServiceAccountIsNamed(t *testing.T) { } r := NewAuthorResolver(lookup, DefaultAttributionGraceWindow, logr.Discard()) - ui, ok := r.ResolveAuthor(context.Background(), "prod-eu-1", resolverGVR, "uid-1", "101", true) - require.True(t, ok, "a matched service account is named, not collapsed to the committer") + ui, outcome := r.ResolveAuthor(context.Background(), "prod-eu-1", resolverGVR, "uid-1", "101", true) + require.Equal(t, git.AttributionResolved, outcome, + "a matched service account is named, not collapsed to the committer") assert.Equal(t, sa, ui.Username) count, ok := telemetry.CollectInt64Sum(reader, "gitopsreverser_attribution_resolutions_total", @@ -90,14 +92,16 @@ func TestAuthorResolver_ServiceAccountIsNamed(t *testing.T) { assert.Equal(t, uint64(1), waitCount) } -func TestAuthorResolver_MissExpiresToCommitter(t *testing.T) { +func TestAuthorResolver_MissExpiresToUnresolved(t *testing.T) { lookup := &fakeLookup{resolution: queue.AuthorResolution{Result: queue.AttributionAbsent}, hitAfter: 1000} r := NewAuthorResolver(lookup, 0, logr.Discard()) - // A zero grace does a single lookup and, on a miss, ships as committer (ok=false). - // There is no longer a miss-marker write-back. - _, ok := r.ResolveAuthor(context.Background(), "prod-eu-1", resolverGVR, "uid-1", "101", true) - assert.False(t, ok) + // A zero grace does a single lookup and, on a miss, reports UNRESOLVED — attribution ran + // and did not name anyone. It is deliberately not NotAttempted, which would claim + // attribution was never switched on. There is no miss-marker write-back. + ui, outcome := r.ResolveAuthor(context.Background(), "prod-eu-1", resolverGVR, "uid-1", "101", true) + assert.Equal(t, git.AttributionUnresolved, outcome) + assert.Empty(t, ui.Username, "an unresolved attribution names nobody") assert.Equal(t, 1, lookup.calls) } @@ -111,8 +115,8 @@ func TestAuthorResolver_DeleteEventIsNotExactCapable(t *testing.T) { } r := NewAuthorResolver(lookup, DefaultAttributionGraceWindow, logr.Discard()) - _, ok := r.ResolveAuthor(context.Background(), "prod-eu-1", resolverGVR, "uid-1", "999", false) - require.True(t, ok) + _, outcome := r.ResolveAuthor(context.Background(), "prod-eu-1", resolverGVR, "uid-1", "999", false) + require.Equal(t, git.AttributionResolved, outcome) assert.False(t, lookup.lastExactCapable, "a removal event may consult the /last pointer") } @@ -126,14 +130,38 @@ func TestAuthorResolver_WaitsThroughGraceWindowForLateFact(t *testing.T) { } r := NewAuthorResolver(lookup, 2*time.Second, logr.Discard()) - ui, ok := r.ResolveAuthor(context.Background(), "prod-eu-1", resolverGVR, "uid-1", "101", true) - require.True(t, ok) + ui, outcome := r.ResolveAuthor(context.Background(), "prod-eu-1", resolverGVR, "uid-1", "101", true) + require.Equal(t, git.AttributionResolved, outcome) assert.Equal(t, "bob", ui.Username) assert.GreaterOrEqual(t, lookup.calls, 3) } -func TestAuthorResolver_NilLookupIsCommitter(t *testing.T) { +// A nil lookup is configured-author mode: attribution was never switched on, so the outcome +// must be NotAttempted — not Unresolved. Conflating the two is what made a lost actor +// indistinguishable from a deployment that simply does not do attribution. +func TestAuthorResolver_NilLookupIsNotAttempted(t *testing.T) { r := NewAuthorResolver(nil, DefaultAttributionGraceWindow, logr.Discard()) - _, ok := r.ResolveAuthor(context.Background(), "prod-eu-1", resolverGVR, "uid-1", "101", true) - assert.False(t, ok) + + ui, outcome := r.ResolveAuthor(context.Background(), "prod-eu-1", resolverGVR, "uid-1", "101", true) + + assert.Equal(t, git.AttributionNotAttempted, outcome, + "attribution that was never enabled has not failed — the committer legitimately authors") + assert.Empty(t, ui.Username) +} + +// A fact that exists but carries no author is also unresolved, not not-attempted: attribution +// ran, found something, and still could not name anyone. +func TestAuthorResolver_AuthorlessFactIsUnresolved(t *testing.T) { + lookup := &fakeLookup{ + resolution: queue.AuthorResolution{ + Fact: queue.AuthorFact{Author: ""}, + Result: queue.AttributionExactUser, + }, + hitAfter: 1, + } + r := NewAuthorResolver(lookup, DefaultAttributionGraceWindow, logr.Discard()) + + _, outcome := r.ResolveAuthor(context.Background(), "prod-eu-1", resolverGVR, "uid-1", "101", true) + + assert.Equal(t, git.AttributionUnresolved, outcome) } diff --git a/internal/watch/target_watch.go b/internal/watch/target_watch.go index 60014c6c..fa83f059 100644 --- a/internal/watch/target_watch.go +++ b/internal/watch/target_watch.go @@ -750,6 +750,8 @@ func (m *Manager) attachAuthor( gvr schema.GroupVersionResource, u *unstructured.Unstructured, ) { + // A nil resolver is configured-author mode: nothing is attempted, and the zero + // AttributionNotAttempted on the event is already correct. if m.AuthorResolver == nil { return } @@ -762,9 +764,15 @@ func (m *Manager) attachAuthor( // ClusterProvider name — this event was watched on. It keys the author read against exactly // the facts the audit handler recorded for that cluster, so a fact from cluster A can never // name the author of an object watched on cluster B. - if userInfo, ok := m.AuthorResolver.ResolveAuthor( + userInfo, outcome := m.AuthorResolver.ResolveAuthor( ctx, event.SourceCluster, gvr, u.GetUID(), u.GetResourceVersion(), exactCapable, - ); ok { + ) + // Stamp the outcome even when no actor was named: an unresolved attribution is a fact the + // writer, the author_kind metric, and CommitRequest matching all need. Leaving it at the + // zero value would say "attribution was never attempted", which is exactly the conflation + // that made this loss invisible. + event.Attribution = outcome + if outcome == git.AttributionResolved { event.UserInfo = userInfo } } From 2f29c39c9d55adfa13276dd5d99836a9cc304575 Mon Sep 17 00:00:00 2001 From: Simon Koudijs Date: Mon, 20 Jul 2026 13:16:02 +0000 Subject: [PATCH 10/17] fix(giteaclient): follow pagination when listing a user's keys MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ListUserKeys issued GET /users/{login}/keys with no pagination, so it saw at most one page. Verified against both sides: - Server: GetListOptions does PageSize: ToCorrectPageSize(limit), and ToCorrectPageSize clamps `size <= 0` back to DefaultPagingNum — 30 by default, MaxResponseItems 50. There is NO "return everything" mode for this endpoint. (gitea modules/setting/api.go, services/convert/utils.go, routers/api/v1/utils/page.go) - SDK, for the record: code.gitea.io/sdk documents ListOptions.Page = -1 as "disables pagination", but it merely sends page=0&limit=0, which the server clamps straight back to 30. So adopting the SDK would NOT have avoided this; its escape hatch is ineffective on this endpoint and actively misleading. The consequence is not a truncated listing but a wedged workflow: this listing backs FindUserKeyByTitle, which EnsureUserKeyAsAdmin uses to decide whether a title is taken. Past one page, "ensure" cannot see the key it needs to replace, so it tries to create and fails `422 Key title has been used` for a key that demonstrably exists. The e2e suite hits this deterministically. It registers ~24 seeded transport keys per full run against a Gitea that outlives the run, so from the second run onward the admin user holds more than one page. The long-lived, stable-titled `playground` key then becomes unreachable and the playground spec fails at BeforeAll — which is why it only ever appeared after several runs on one cluster, and why `task clean-cluster` "fixed" it: that resets Gitea to zero keys rather than fixing anything. Requests limit=50 (MaxResponseItems) to keep round trips low, and stops on the first short page. An empty page also terminates, guarding against a server that ignores the limit. Verified: both new tests fail against the unpaginated implementation — one for a title on the last of three pages, one for a listing that exactly fills a page. Co-Authored-By: Claude Opus 4.8 (1M context) --- internal/giteaclient/keys.go | 42 +++++++-- internal/giteaclient/keys_test.go | 143 ++++++++++++++++++++++++++++++ 2 files changed, 176 insertions(+), 9 deletions(-) diff --git a/internal/giteaclient/keys.go b/internal/giteaclient/keys.go index 08e0b41b..ba3a41cb 100644 --- a/internal/giteaclient/keys.go +++ b/internal/giteaclient/keys.go @@ -25,17 +25,41 @@ func NormalizeAuthorizedKey(k string) string { } // ListUserKeys returns every public key registered for the named user. +// keyPageSize is the per-page item count requested when listing keys. Gitea caps it at +// MAX_RESPONSE_ITEMS (50 by default) and silently truncates beyond that, so requesting the cap +// keeps the number of round trips low without relying on the server's DEFAULT_PAGING_NUM. +const keyPageSize = 50 + +// ListUserKeys returns EVERY public key on the named user, following pagination. +// +// The pagination is load-bearing, not tidiness. Gitea's default page size is 30, and this +// listing backs FindUserKeyByTitle, which EnsureUserKeyAsAdmin uses to decide whether a title +// is already taken. An unpaginated listing therefore stops seeing keys once a user holds more +// than one page of them, and "ensure" starts failing with `422 Key title has been used` for a +// key that demonstrably exists — the caller cannot see it to replace it. The e2e suite hits +// this reliably: it registers ~24 seeded transport keys per full run against a long-lived +// Gitea, so the second run onward exceeds one page and the stable-titled `playground` key +// becomes unreachable. func (c *Client) ListUserKeys(ctx context.Context, login string) ([]PublicKey, error) { - var keys []PublicKey - path := "/users/" + PathEscape(login) + "/keys" - code, raw, err := c.Do(ctx, http.MethodGet, path, nil, &keys) - if err != nil { - return nil, err - } - if code != http.StatusOK { - return nil, unexpectedStatus(http.MethodGet, path, code, raw) + var all []PublicKey + base := "/users/" + PathEscape(login) + "/keys" + for page := 1; ; page++ { + var keys []PublicKey + path := fmt.Sprintf("%s?page=%d&limit=%d", base, page, keyPageSize) + code, raw, err := c.Do(ctx, http.MethodGet, path, nil, &keys) + if err != nil { + return nil, err + } + if code != http.StatusOK { + return nil, unexpectedStatus(http.MethodGet, path, code, raw) + } + all = append(all, keys...) + // A short page is the last page. An empty page also terminates, which guards against a + // server that ignores the limit and would otherwise loop forever. + if len(keys) < keyPageSize { + return all, nil + } } - return keys, nil } // FindUserKey looks up a public key on the named user by key material, diff --git a/internal/giteaclient/keys_test.go b/internal/giteaclient/keys_test.go index 0f154502..d2490f15 100644 --- a/internal/giteaclient/keys_test.go +++ b/internal/giteaclient/keys_test.go @@ -4,8 +4,11 @@ package giteaclient import ( "context" + "fmt" "net/http" "net/http/httptest" + "strconv" + "strings" "testing" ) @@ -111,3 +114,143 @@ func TestClientFindUserKeyByTitle_ReturnsMatch(t *testing.T) { t.Fatalf("FindUserKeyByTitle() = %#v, want key id 2", key) } } + +// keysPageJSON renders one page of the fake keys listing: ids [start, end) of `total`, where +// the LAST key of the whole set carries lastTitle so a title-search must reach the final page. +func keysPageJSON(start, end, total int, lastTitle string) string { + var b strings.Builder + b.WriteString("[") + for i := start; i < end; i++ { + if i > start { + b.WriteString(",") + } + title := fmt.Sprintf("key-%d", i) + if i == total-1 { + title = lastTitle + } + fmt.Fprintf(&b, `{"id":%d,"title":%q,"key":"ssh-ed25519 AAAA k%d","fingerprint":"fp%d"}`, + i+1, title, i, i) + } + b.WriteString("]") + return b.String() +} + +// Gitea paginates key listings: DEFAULT_PAGING_NUM is 30 and MAX_RESPONSE_ITEMS is 50, and +// `ToCorrectPageSize` clamps a limit of 0 or less straight back to the default — so there is +// no server-side "return everything" mode for this endpoint. An unpaginated listing therefore +// goes blind past the first page, which silently breaks FindUserKeyByTitle and makes +// EnsureUserKeyAsAdmin fail with `422 Key title has been used` for a key that exists but +// cannot be seen (and so cannot be replaced). +// +// The fake below serves a key set larger than one page, with the sought title on the LAST +// page. Without the page loop the client only ever sees page 1 and the title is reported +// missing. +func TestClientListUserKeys_FollowsPaginationBeyondTheFirstPage(t *testing.T) { + t.Parallel() + + const ( + pageSize = 50 + total = 124 // > 2 full pages, so a single-page read cannot pass by luck + lastTitle = "E2E Transport Key playground" + ) + var pagesServed []string + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + page := requireKeysPageRequest(t, r) + pagesServed = append(pagesServed, strconv.Itoa(page)) + + start := (page - 1) * pageSize + w.Header().Set("Content-Type", "application/json") + if start >= total { + _, _ = w.Write([]byte(`[]`)) + return + } + _, _ = w.Write([]byte(keysPageJSON(start, min(start+pageSize, total), total, lastTitle))) + })) + defer server.Close() + + client := New(server.URL, "admin", "password") + + keys, err := client.ListUserKeys(context.Background(), "giteaadmin") + if err != nil { + t.Fatalf("ListUserKeys() error = %v", err) + } + if len(keys) != total { + t.Fatalf("ListUserKeys() returned %d keys, want all %d — pagination was not followed", len(keys), total) + } + if len(pagesServed) != 3 { + t.Fatalf("pages requested = %v, want 3 (two full pages plus a short final page)", pagesServed) + } + + // The point of the loop: a title on a later page must still be findable, because that is + // what EnsureUserKeyAsAdmin needs in order to replace a stale key instead of colliding. + found, ok, err := client.FindUserKeyByTitle(context.Background(), "giteaadmin", lastTitle) + if err != nil { + t.Fatalf("FindUserKeyByTitle() error = %v", err) + } + if !ok { + t.Fatal("FindUserKeyByTitle() did not find a key that exists on a later page") + } + if found.ID != total { + t.Fatalf("FindUserKeyByTitle() = id %d, want %d", found.ID, total) + } +} + +// requireKeysPageRequest asserts the request is an explicitly paginated keys read and returns +// the requested page. An unpaginated read is the bug this file exists to prevent, so it fails +// here rather than silently truncating at the server default. +func requireKeysPageRequest(t *testing.T, r *http.Request) int { + t.Helper() + if r.Method != http.MethodGet || r.URL.Path != "/users/giteaadmin/keys" { + t.Fatalf("unexpected request: %s %s", r.Method, r.URL.Path) + } + query := r.URL.Query() + page := query.Get("page") + if page == "" { + t.Fatal("ListUserKeys must request an explicit page; an unpaginated read silently truncates at 30") + } + if got := query.Get("limit"); got != "50" { + t.Fatalf("limit = %q, want 50 (Gitea's MAX_RESPONSE_ITEMS; a larger value is silently clamped)", got) + } + n, err := strconv.Atoi(page) + if err != nil { + t.Fatalf("bad page parameter %q: %v", page, err) + } + return n +} + +// A listing that exactly fills a page must still terminate: the client asks for one more page, +// gets an empty body, and stops. Without that, an exact multiple would either loop forever or +// drop the tail. +func TestClientListUserKeys_ExactPageMultipleTerminates(t *testing.T) { + t.Parallel() + + const pageSize = 50 + requests := 0 + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests++ + if requests > 5 { + t.Fatal("ListUserKeys did not terminate on an empty page") + } + page := requireKeysPageRequest(t, r) + w.Header().Set("Content-Type", "application/json") + if page != 1 { + _, _ = w.Write([]byte(`[]`)) + return + } + _, _ = w.Write([]byte(keysPageJSON(0, pageSize, pageSize+1, "unused"))) + })) + defer server.Close() + + keys, err := New(server.URL, "admin", "password").ListUserKeys(context.Background(), "giteaadmin") + if err != nil { + t.Fatalf("ListUserKeys() error = %v", err) + } + if len(keys) != pageSize { + t.Fatalf("ListUserKeys() returned %d keys, want %d", len(keys), pageSize) + } + if requests != 2 { + t.Fatalf("requests = %d, want 2 (the full page, then the empty page that ends the loop)", requests) + } +} From be2f9d3ec57a3412096ea556daa9ff5214ff56b5 Mon Sep 17 00:00:00 2001 From: Simon Koudijs Date: Mon, 20 Jul 2026 13:16:03 +0000 Subject: [PATCH 11/17] refactor(watch): drop the dead snapshot-resolution path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit resolveSnapshotGVRs, resolveSnapshotGVRForType, snapshotGVRsFromTable, snapshotGVRScopes and the snapshotGVR type had no production callers — only each other and their tests. The desired set is no longer gathered through this path; it is the spliced materialization, so scope_resolve.go kept a parallel, never-executed projection of the same watched-type table. Found while implementing PR 2, where it caused real cost: the change had to be applied to BOTH read sites and verified at both, and one of PR 2's three regression tests asserted this projection specifically. That test goes with the code, which is the honest outcome — with the path gone there is exactly ONE consumer of WatchedType.WatchScopes (targetWatchSpecs), so the "both read sites must agree" invariant PR 2 was written to defend no longer has two sides to disagree. Kept: retainedWatchedTypes, typeWobbling, gvkListSummary and desiredFromObject, which are all live from target_watch.go and the splice. 243 lines removed. No behaviour change — nothing called any of it. Co-Authored-By: Claude Opus 4.8 (1M context) --- internal/watch/manager_snapshot_test.go | 91 +------------- internal/watch/scope_resolve.go | 128 -------------------- internal/watch/scope_resolve_test.go | 22 ---- internal/watch/snapshot_stream_type_test.go | 24 ---- 4 files changed, 1 insertion(+), 264 deletions(-) diff --git a/internal/watch/manager_snapshot_test.go b/internal/watch/manager_snapshot_test.go index 179e1180..97210aa3 100644 --- a/internal/watch/manager_snapshot_test.go +++ b/internal/watch/manager_snapshot_test.go @@ -21,10 +21,7 @@ import ( itypes "github.com/ConfigButler/gitops-reverser/internal/types" ) -var ( - secretsGVR = schema.GroupVersionResource{Group: "", Version: "v1", Resource: "secrets"} - nodesGVR = schema.GroupVersionResource{Group: "", Version: "v1", Resource: "nodes"} -) +var secretsGVR = schema.GroupVersionResource{Group: "", Version: "v1", Resource: "secrets"} // makeScheme returns a scheme with core Kubernetes types registered. func makeScheme(t *testing.T) *runtime.Scheme { @@ -83,47 +80,10 @@ func addSecretsWatchRule(store *rulestore.RuleStore) { ) } -// addClusterWatchRule registers a cluster-scoped ClusterWatchRule for my-target. -func addClusterWatchRule(store *rulestore.RuleStore, name, resource string) { - store.AddOrUpdateClusterWatchRule( - configv1alpha3.ClusterWatchRule{ - ObjectMeta: metav1.ObjectMeta{Name: name}, - Spec: configv1alpha3.ClusterWatchRuleSpec{ - TargetRef: configv1alpha3.NamespacedTargetReference{Name: "my-target", Namespace: "gitops-reverser"}, - Rules: []configv1alpha3.ClusterResourceRule{{ - APIGroups: []string{""}, APIVersions: []string{"v1"}, Resources: []string{resource}, - Scope: configv1alpha3.ResourceScopeCluster, - }}, - }, - }, - "my-target", "gitops-reverser", "provider", "gitops-reverser", "main", "live", - ) -} - func myTargetRef() itypes.ResourceReference { return itypes.NewResourceReference("my-target", "gitops-reverser") } -// A reconcile fails closed while the cluster API surface has not been observed yet (an -// empty/unready discovery leaves the type registry unready): sweeping a mark over an -// unobserved surface would delete the mirror. -func TestResolveSnapshotGVRs_FailsClosedWhenRegistryNotReady(t *testing.T) { - store := rulestore.NewStore() - addSecretsWatchRule(store) - empty := apiResourceDiscovery(staticCatalogDiscovery{}) - m := &Manager{ - Log: logr.Discard(), - RuleStore: store, - resourceCatalog: NewAPIResourceCatalog(), - discoveryClient: func() (apiResourceDiscovery, error) { return empty, nil }, - } - - _, err := m.resolveSnapshotGVRs(context.Background(), myTargetRef()) - require.Error(t, err, "an unobserved API surface must abort the gather rather than sweep") - assert.Contains(t, err.Error(), "has not been observed yet") -} - -// A normally-served target has no retained types, so the snapshot is not blocked. func TestRetainedWatchedTypes_NoneWhenAllServed(t *testing.T) { store := rulestore.NewStore() addSecretsWatchRule(store) @@ -148,52 +108,3 @@ func TestGVKListSummary(t *testing.T) { assert.Contains(t, got, "Kind=ConfigMap") assert.Contains(t, got, "Kind=Deployment") } - -// resolveSnapshotGVRs scopes a namespaced resource to its rule namespace and a -// cluster-scoped resource cluster-wide (no namespaces). -func TestResolveSnapshotGVRs_ScopesNamespacedAndClusterWide(t *testing.T) { - store := rulestore.NewStore() - addSecretsWatchRule(store) - addClusterWatchRule(store, "cwr-nodes", "nodes") - - m := streamingManager(t, gitTargetFixture(), store) - gvrs, err := m.resolveSnapshotGVRs(context.Background(), myTargetRef()) - require.NoError(t, err) - - byGVR := map[schema.GroupVersionResource][]string{} - for _, sg := range gvrs { - byGVR[sg.gvr] = append(byGVR[sg.gvr], sg.namespace) - } - assert.Equal(t, []string{"ns-a"}, byGVR[secretsGVR], "namespaced resource scoped to its rule namespace") - assert.Equal(t, []string{""}, byGVR[nodesGVR], "cluster-scoped resource gathers under the cluster-wide scope") -} - -// A wildcard resource pattern expands to every served namespaced resource in the group, -// so the snapshot is not silently narrowed. -func TestResolveSnapshotGVRs_WildcardResourceExpands(t *testing.T) { - store := rulestore.NewStore() - store.AddOrUpdateWatchRule( - configv1alpha3.WatchRule{ - ObjectMeta: metav1.ObjectMeta{Name: "wr-all", Namespace: "ns-a"}, - Spec: configv1alpha3.WatchRuleSpec{ - TargetRef: configv1alpha3.LocalTargetReference{Name: "my-target"}, - Rules: []configv1alpha3.ResourceRule{{ - APIGroups: []string{""}, APIVersions: []string{"v1"}, Resources: []string{"*"}, - }}, - }, - }, - "my-target", "gitops-reverser", "provider", "gitops-reverser", "main", "live", - ) - - m := streamingManager(t, gitTargetFixture(), store) - gvrs, err := m.resolveSnapshotGVRs(context.Background(), myTargetRef()) - require.NoError(t, err) - - resources := map[string]struct{}{} - for _, sg := range gvrs { - resources[sg.gvr.Resource] = struct{}{} - } - assert.Contains(t, resources, "configmaps") - assert.Contains(t, resources, "secrets") - assert.Contains(t, resources, "services") -} diff --git a/internal/watch/scope_resolve.go b/internal/watch/scope_resolve.go index 13f6dbc2..37a3248c 100644 --- a/internal/watch/scope_resolve.go +++ b/internal/watch/scope_resolve.go @@ -3,7 +3,6 @@ package watch import ( - "context" "fmt" "sort" "strings" @@ -40,114 +39,6 @@ type ClusterSnapshot struct { CoverageHead string } -// snapshotGVR is one resolved watched resource type paired with the ONE namespace scope to -// gather it under: an empty namespace means cluster-wide (every namespace), exactly as it -// does for a dynamic client List and for targetWatchKey. -// -// A type followed both cluster-wide and in named namespaces yields one entry per scope — -// the same distinct set targetWatchSpecs streams — so a gather's scope is always exactly -// one stream's scope, and therefore exactly the scope its mark-and-sweep may delete over. -type snapshotGVR struct { - gvr schema.GroupVersionResource - namespace string -} - -// resolveSnapshotGVRForType resolves one watched type's (GVR, namespace-scope) set for a -// GitTarget, with the same fail-closed discipline as resolveSnapshotGVRs but scoped to the -// single type. It returns one entry per scope the type is gathered under, so a type followed -// both cluster-wide and in a named namespace reconciles as the two streams it actually is. The -// bool is false when this GitTarget does not watch the type (so there is nothing to reconcile). -// It refuses (error) when the surface is unobserved or the type is currently `retained` (a -// wobble) — the per-type expression of the anti-sweep invariant (R9/R11). -func (m *Manager) resolveSnapshotGVRForType( - ctx context.Context, - gitDest types.ResourceReference, - gvr schema.GroupVersionResource, -) ([]snapshotGVR, bool, error) { - if err := m.RefreshAPIResourceCatalog(ctx); err != nil { - return nil, false, fmt.Errorf("refresh API resource catalog for %s: %w", gitDest.String(), err) - } - m.refreshWatchedTypeTables() - - reg := m.registryForGitTarget(gitDest) - if !reg.Ready() { - return nil, false, fmt.Errorf( - "aborting per-type reconcile for %s: the cluster API surface has not been observed yet", - gitDest.String()) - } - - table := m.residentWatchedTypeTable(gitDest) - var watched *WatchedType - for i := range table.Types { - if table.Types[i].GVR == gvr { - watched = &table.Types[i] - break - } - } - if watched == nil { - return nil, false, nil - } - - if typeWobbling(reg, gvr) { - return nil, false, fmt.Errorf( - "aborting per-type reconcile for %s: %s within the removal grace (currently unserved); "+ - "refusing to reconcile a reduced view", - gitDest.String(), gvr.String()) - } - - return snapshotGVRScopes(*watched), true, nil -} - -// snapshotGVRScopes projects one watched type into its per-scope gather entries: one entry -// per namespace scope the type is streamed under, cluster-wide ("") included. It is the -// single projection both read sites share, so the whole-target and per-type paths cannot -// disagree about a type's scope. -func snapshotGVRScopes(wt WatchedType) []snapshotGVR { - out := make([]snapshotGVR, 0, len(wt.NamespaceOps)) - for _, ns := range wt.WatchScopes() { - out = append(out, snapshotGVR{gvr: wt.GVR, namespace: ns}) - } - return out -} - -// resolveSnapshotGVRs returns the GitTarget's watched (GVR, namespace-scope) set, read from the -// resident watched-type table. It refreshes the trusted API catalog, the registry, and the -// table first, then fails closed if the registry is not ready — a reconcile must never be built -// from an unobserved API surface, and a mark-and-sweep over a reduced view would delete KRM from -// git. A type that briefly leaves discovery stays followable (and so stays in the table) for the -// registry's removal grace, so a transient wobble never sweeps git. It is the scope side of the -// splice and the demand Declare (DEC-L3). -func (m *Manager) resolveSnapshotGVRs( - ctx context.Context, - gitDest types.ResourceReference, -) ([]snapshotGVR, error) { - if err := m.RefreshAPIResourceCatalog(ctx); err != nil { - return nil, fmt.Errorf("refresh API resource catalog for %s: %w", gitDest.String(), err) - } - m.refreshWatchedTypeTables() - - if !m.registryForGitTarget(gitDest).Ready() { - return nil, fmt.Errorf( - "aborting scope resolution for %s: the cluster API surface has not been observed yet; "+ - "refusing to reconcile a partial cluster view", - gitDest.String()) - } - - table := m.residentWatchedTypeTable(gitDest) - - // A watched type the registry holds as `retained` is followable under the removal grace but - // is not actually served right now (a discovery wobble). Reconciling it would sweep a reduced - // view and delete a still-valid mirror, so fail closed until the wobble resolves. - if retained := m.retainedWatchedTypes(gitDest, table); len(retained) > 0 { - return nil, fmt.Errorf( - "aborting scope resolution for %s: %s within the removal grace (currently unserved); "+ - "refusing to sweep a reduced cluster view", - gitDest.String(), gvkListSummary(retained)) - } - - return snapshotGVRsFromTable(table), nil -} - // retainedWatchedTypes returns the GVKs of the target's watched types the registry currently // holds as `retained` (followable under the grace, but not served right now), resolved against // the GitTarget's OWN source cluster's registry. @@ -189,25 +80,6 @@ func gvkListSummary(gvks []schema.GroupVersionKind) string { return fmt.Sprintf("%d watched types [%s]", len(parts), strings.Join(parts, ", ")) } -// snapshotGVRsFromTable projects a watched-type table into the deterministic, sorted -// (GVR, namespace) set: one entry per type per namespace scope, with an empty namespace -// meaning cluster-wide. A type followed both cluster-wide and in a named namespace yields -// both entries — it must agree with targetWatchSpecs scope for scope, or the plan hash and -// the running streams describe different mirrors. -func snapshotGVRsFromTable(table WatchedTypeTable) []snapshotGVR { - out := make([]snapshotGVR, 0, len(table.Types)) - for _, wt := range table.Types { - out = append(out, snapshotGVRScopes(wt)...) - } - sort.Slice(out, func(i, j int) bool { - if out[i].gvr.String() == out[j].gvr.String() { - return out[i].namespace < out[j].namespace - } - return out[i].gvr.String() < out[j].gvr.String() - }) - return out -} - // desiredFromObject converts a materialized object into a desired resource, pairing the // GVR-derived API identity with the sanitized object the writer will materialise. It is shared // by the splice's scope projection (splice_snapshot.go) so a reconcile's desired set is shaped diff --git a/internal/watch/scope_resolve_test.go b/internal/watch/scope_resolve_test.go index 81fc1a99..419d1847 100644 --- a/internal/watch/scope_resolve_test.go +++ b/internal/watch/scope_resolve_test.go @@ -36,25 +36,3 @@ func TestDesiredFromObject(t *testing.T) { _, ok = desiredFromObject(configMapGVR, (*unstructured.Unstructured)(nil)) assert.False(t, ok, "a nil object is not a desired entry") } - -// The snapshot read site must project the same scope set targetWatchSpecs streams. A fix that -// lands only in the watch path leaves the plan hash and the running streams describing -// different mirrors — and, once a per-scope replay drives the mark-and-sweep, a gather wider -// than its stream deletes managed documents that were never in that stream's scope. -func TestSnapshotGVRsFromTable_NamedAndClusterWideScopesStayDistinctEntries(t *testing.T) { - table := WatchedTypeTable{ - Types: []WatchedType{{ - GVR: configMapGVR, - NamespaceOps: map[string]OperationSet{ - "": {"UPDATE": struct{}{}}, - "team-a": {"CREATE": struct{}{}}, - }, - }}, - } - - got := snapshotGVRsFromTable(table) - - require.Len(t, got, 2, "a cluster-wide selection must not swallow the named-namespace gather") - assert.Equal(t, snapshotGVR{gvr: configMapGVR, namespace: ""}, got[0], "cluster-wide scope sorts first") - assert.Equal(t, snapshotGVR{gvr: configMapGVR, namespace: "team-a"}, got[1]) -} diff --git a/internal/watch/snapshot_stream_type_test.go b/internal/watch/snapshot_stream_type_test.go index c2194c02..e7d4aee0 100644 --- a/internal/watch/snapshot_stream_type_test.go +++ b/internal/watch/snapshot_stream_type_test.go @@ -3,40 +3,16 @@ package watch import ( - "context" "testing" - "github.com/go-logr/logr" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" "k8s.io/apimachinery/pkg/runtime/schema" - - "github.com/ConfigButler/gitops-reverser/internal/rulestore" ) // configmapsGVR is a second served namespaced type, used to prove per-type scope resolution // is membership-exact. var configmapsGVR = schema.GroupVersionResource{Group: "", Version: "v1", Resource: "configmaps"} -// resolveSnapshotGVRForType fails closed when the API surface has not been observed yet — the -// per-type expression of the never-reconcile-a-partial-view invariant. -func TestResolveSnapshotGVRForType_FailsClosedWhenRegistryNotReady(t *testing.T) { - store := rulestore.NewStore() - addSecretsWatchRule(store) - empty := apiResourceDiscovery(staticCatalogDiscovery{}) - m := &Manager{ - Log: logr.Discard(), - RuleStore: store, - resourceCatalog: NewAPIResourceCatalog(), - discoveryClient: func() (apiResourceDiscovery, error) { return empty, nil }, - } - - _, _, err := m.resolveSnapshotGVRForType(context.Background(), myTargetRef(), secretsGVR) - require.Error(t, err, "an unobserved API surface must abort the per-type gather") - assert.Contains(t, err.Error(), "has not been observed yet") -} - -// tableWatchesGVR reports membership of a type in a GitTarget's resident table. func TestTableWatchesGVR(t *testing.T) { table := WatchedTypeTable{Types: []WatchedType{{GVR: secretsGVR}}} assert.True(t, tableWatchesGVR(table, secretsGVR), "a watched type is reported present") From 608ed1bf4be39fc8421db4ab4fb6756c1ce0923d Mon Sep 17 00:00:00 2001 From: Simon Koudijs Date: Mon, 20 Jul 2026 14:21:01 +0000 Subject: [PATCH 12/17] fix(attribution): let a CommitRequest attach when the two sides disagree about why nobody was named MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In the DEFAULT deployment no CommitRequest could attach to any commit window, so the user's message was silently dropped and the change landed under the generated one. Two independent causes, both introduced with the explicit outcome. 1. AttributionNotAttempted was "not_attempted", so it was NOT the zero value of AttributionOutcome. Most producers of an Event never assign the field — reconcile, resync, bootstrap, and configured-author mode's early return in attachAuthor (the only assignment site in the non-test tree) — and all of them mean "no actor was sought". The zero value was therefore a silent fourth state equal to none of the three named outcomes. Three comments asserted it already WAS AttributionNotAttempted and were false (types.go, target_watch.go, author_resolver.go). The constant is now "" so that it genuinely is, which makes all three true at once instead of patching one site. Nothing serializes it: it reaches no CRD field and no metric label, and authorKind branches on the typed value. 2. matchesWindow compared the two outcomes for exact equality. They are produced by DIFFERENT, INDEPENDENTLY CONFIGURED subsystems — the window's by mirrored-resource attribution (--author-attribution), the request's by command authorship (--admission-webhook), which cmd/main.go documents as independent — so equality couples the two flags: watch attr | webhook | window | CommitRequest | equality off | off | not_attempted | not_attempted | match off | on, miss | not_attempted | unresolved | NO MATCH on, unres. | off | unresolved | not_attempted | NO MATCH on, unres. | on, miss | unresolved | unresolved | match The two middle rows are real deployments that attach on main. Matching now goes through AttributionOutcome.NamesActor: an outcome either produced an actor to compare or it did not, and WHY it did not is a property of the producer's configuration, not of the commit. The author comparison stays UNCONDITIONAL, with the outcome class as an additional guard rather than a replacement. Skipping it when neither side names an actor looks equivalent — both authors are "" — but makes cross-author attachment depend on the outcome fields being right; TestAttach_ForeignWindowIsNotStolen caught exactly that, with bob's request claiming alice's window. Note this is the opposite call from openWindow.canAppend, which compares the enums directly and should: both of its sides come from the same watch pipeline. Intra-subsystem versus cross-subsystem. Also settles what the unresolved sentinel is scoped to. commitOptionsFor already DERIVES UnresolvedAuthor() at the write path from the carried outcome rather than storing it on the event, so the sentinel is the git author header and nothing else — not window grouping, not the grouped message body, not {{.Username}}. types.go documented the opposite and was the stated rationale for the three-string design. Stamping it onto event.UserInfo would change the commit text of every deployment that has attribution misses and force user templates to special-case a magic token; an unresolved event now renders {{.Username}} empty exactly as configured-author mode always has, while git log names the gap. Tests. The gap that let this ship is that every existing case in commit_request_attach_test.go left BOTH sides at the zero value, so "" == "" passed and the production combination was never exercised. - TestCommitRequestAndWindowOutcomesAgree drives each side from its REAL producer: the controller's gitOutcome() projection, and the watch package's actual resolver / an actual zero git.Event. It lives in internal/controller because that is the only package that imports both. - TestMatchesWindow_AcrossIndependentlyConfiguredSubsystems walks every pair either side can produce and names the deployment that produces it, including the different-authors-with-unset-outcomes case that pins the author check as unconditional. - TestAttributionZeroValueIsNotAttempted pins the coupling everything rests on. - The two tests that hand-injected a state production never produces (window.Author = sentinel; the sentinel in a {{.Username}} template) now assert production reality instead. Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/configuration.md | 6 + docs/design/attribution-outcome-and-author.md | 99 ++++++++++---- ...ommitrequest_attribution_agreement_test.go | 120 +++++++++++++++++ internal/git/attribution_outcome_test.go | 53 ++++++-- internal/git/commit_request_attach.go | 48 ++++--- internal/git/commit_request_attach_test.go | 127 ++++++++++++++++++ internal/git/open_window.go | 7 +- internal/git/types.go | 53 ++++++-- internal/watch/author_resolver.go | 9 +- internal/watch/target_watch.go | 7 +- 10 files changed, 453 insertions(+), 76 deletions(-) create mode 100644 internal/controller/commitrequest_attribution_agreement_test.go diff --git a/docs/configuration.md b/docs/configuration.md index 7578f561..86999c71 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -236,6 +236,12 @@ spec: - `Username` - `GitTarget` +`Username` is empty whenever no actor was named — both in configured-author mode and when +attribution ran and did not resolve. The `attribution-unresolved` sentinel is scoped to the Git +**author header** and deliberately does not reach templates or message bodies, so a template +rendering `{{.Username}}` never has to special-case it. Use `git log` (or +`author_kind="unresolved"`) to tell the two apart. + `groupTemplate` can use: - `Author` diff --git a/docs/design/attribution-outcome-and-author.md b/docs/design/attribution-outcome-and-author.md index a0a577e2..c04ab512 100644 --- a/docs/design/attribution-outcome-and-author.md +++ b/docs/design/attribution-outcome-and-author.md @@ -34,12 +34,20 @@ the first draft got wrong by trying to infer outcome from `UserInfo.Username`. type AttributionOutcome string const ( - AttributionNotAttempted AttributionOutcome = "not_attempted" // configured-author mode - AttributionResolved AttributionOutcome = "resolved" // a fact named the actor - AttributionUnresolved AttributionOutcome = "unresolved" // attempted, no usable fact + AttributionNotAttempted AttributionOutcome = "" // configured-author mode + AttributionResolved AttributionOutcome = "resolved" // a fact named the actor + AttributionUnresolved AttributionOutcome = "unresolved" // attempted, no usable fact ) ~~~ +`AttributionNotAttempted` is the **empty string on purpose**, so that it is also the type's zero +value. Most producers of an `Event` never assign the field — reconcile, resync, bootstrap, and +configured-author mode's early return in `attachAuthor` — and all of them mean "no actor was +sought". A name-shaped constant makes the zero value a silent fourth state that equals none of +the three outcomes; the first implementation did exactly that, and it stopped every +CommitRequest attaching in the default deployment (see the matching section below). Nothing +serializes the value — it reaches no CRD field and no metric label — so the empty string is free. + Carried on `git.Event` beside `UserInfo`, and on `PendingWrite`. Three consumers then read the outcome instead of sniffing a string: @@ -68,9 +76,13 @@ the right place for it. ## The sentinel identity +The sentinel is **derived at the write path** (`commitOptionsFor`) from the carried outcome. It +is never stamped onto an `Event`, so it is scoped to the git author header and reaches nothing +else: + | Field | Surfaces in | Value | |---|---|---| -| `Username` | window grouping, **grouped commit message body** ([commit.go:96](../../internal/git/commit.go#L96)), **per-event custom templates** ([commit.go:21-33](../../internal/git/commit.go#L21-L33)), author-name fallback | `attribution-unresolved` | +| `Username` | machine token inside the sentinel identity; greppable by tooling that parses the header | `attribution-unresolved` | | `DisplayName` | git author Name via `authorName` | `unknown (attribution unresolved)` | | `Email` | git author Email via `authorEmail` | `attribution-unresolved@gitops-reverser.invalid` | @@ -84,10 +96,16 @@ answers "on whose behalf", and the honest answer is "we do not know". `.invalid` (RFC 2606), so the address can never collide with a real one. Parentheses and spaces pass `isSafeSignatureField`, so the display form is valid in a commit object and a signed payload. -**`Username` reaches user-authored templates.** It is interpolated into `EventTemplate` as -`{{.Username}}`, not only into grouped messages. Anyone whose template renders the author will -see `attribution-unresolved` appear. That is the intent, but it is a visible change to -user-configured output and belongs in the release note. +**The sentinel deliberately does NOT reach message bodies or user templates.** An earlier draft +proposed stamping it onto `event.UserInfo`, which would have pushed it into window grouping, the +grouped commit-message body ([commit.go:96](../../internal/git/commit.go#L96)) and per-event +`{{.Username}}` templates ([commit.go:21-33](../../internal/git/commit.go#L21-L33)). Rejected: +the outcome is the fact and the sentinel is one *rendering* of it, so the rendering belongs at +the surface that needs it. Doing otherwise would change the commit text of every existing +deployment that has attribution misses, and force user-authored templates to special-case a +magic token they never had to handle. An unresolved event therefore renders `{{.Username}}` +exactly as configured-author mode always has — empty — while `git log` names the gap. No +release-note change to user-configured output. ## HIGH — CommitRequest attachment must not silently stop matching @@ -107,19 +125,39 @@ CommitRequest silently lands as a separate commit instead of naming the window i to. That is a real regression, and the first draft asserting "CommitRequest is unaffected" was simply wrong. -**Decision: match on outcome, not on the string.** Both sides carry an `AttributionOutcome`; -a request whose own attribution is `unresolved` matches a window whose attribution is -`unresolved`, for the same GitTarget. Concretely, `matchesWindow` compares -`(outcome, author, gitTarget, gitTargetNamespace)` where the author comparison is only -meaningful when both sides are `resolved`. +Because the sentinel is confined to the git author header (above), the window's `Author` stays +`""` and this naive-sentinel regression never arises. The outcome still participates in matching, +but **only via `NamesActor`, and never as enum equality**. + +**Decision: match on `(author, NamesActor(outcome), gitTarget, gitTargetNamespace)`.** The two +outcomes being compared are produced by *different, independently configured* subsystems — the +window's by mirrored-resource attribution (`--author-attribution`), the request's by command +authorship (`--admission-webhook`), which [cmd/main.go:311-316](../../cmd/main.go#L311-L316) +documents as independent. Requiring the enums to be equal silently couples the two flags: + +| watch attribution | admission webhook | window | CommitRequest | enum equality | `NamesActor` | +|---|---|---|---|---|---| +| off | off | `not_attempted` | `not_attempted` | match | match | +| off | on, miss | `not_attempted` | `unresolved` | **no match** | match | +| on, unresolved | off | `unresolved` | `not_attempted` | **no match** | match | +| on, unresolved | on, miss | `unresolved` | `unresolved` | match | match | + +The two middle rows are real deployments that attach correctly today; under enum equality the +user's commit message is dropped into a separate default-message commit with no error anywhere. + +The author comparison stays **unconditional** — an additional guard, never a replacement. +Skipping it when neither side names an actor looks equivalent (both authors are `""`, so they +compare equal anyway) but makes cross-author attachment depend on the outcome fields being +right: any path that left an outcome unset while the author was set would let one author's +request finalize another's window. Comparing both costs nothing. -This is strictly better than today's accidental match-on-empty-string: two *different* -unattributed actors currently coalesce into one window because both are `""`, and they still -will — but now that behaviour is stated rather than emergent, and it is visible in the -outcome rather than hidden in an empty field. +Note this is the opposite call from `openWindow.canAppend`, which *does* compare the enums +directly and should: it compares two events from the same watch pipeline, so they are directly +comparable. The distinction is intra-subsystem versus cross-subsystem. -**Requires an integration test**, not just a unit test: a fallback CommitRequest must attach -to an unresolved live window and produce one commit, before and after this change. +**Requires a test that sets each side from its real producer**, not two literals: the gap that +let the enum-equality bug ship was that every existing case left both sides at the zero value, +so `"" == ""` passed and the production combination was never exercised. ## HIGH — `author_kind` must not report a failure as a success @@ -191,7 +229,9 @@ docs lie in the way this design exists to prevent: 1. **Git history changes.** Tooling matching `GitOps Reverser` to mean "the operator wrote this" will no longer see unresolved commits. Arguably a fix — those commits were mislabelled — but a behaviour change on shipped data. -2. **User templates change.** `{{.Username}}` in a custom `EventTemplate` renders the sentinel. +2. **User templates and message bodies are unchanged.** The sentinel is confined to the git + author header, so `{{.Username}}` in a custom `EventTemplate` still renders empty for an + unresolved event, exactly as it does in configured-author mode. 3. **The defect becomes visible with no opt-out.** Adopters will see these commits appear and reasonably file bugs. Intended — but it makes the release note **mandatory**, pointing at `gitopsreverser_attribution_resolutions_total` and at @@ -213,8 +253,8 @@ everyone recognises it. 2. `ResolveAuthor` returns the outcome; `attachAuthor` stamps both. 3. `Event` / `PendingWrite` carry the outcome. 4. `authorKind()` reads the outcome; metrics docs updated together. -5. `matchesWindow` matches on outcome; **integration test** for fallback-CommitRequest - attachment. +5. `matchesWindow` matches on `NamesActor`, never on enum equality; the regression test must + drive each side from its real producer, and cover the `not_attempted`/`unresolved` boundary. 6. Commit author rendering reads the outcome. 7. Documentation sweep (list above) + release note. @@ -227,10 +267,17 @@ on its own. kinds; sentinel passes `isSafeSignatureField` and yields a valid commit object. - **Unit:** two unresolved events coalesce into one window; an unresolved and a resolved event do not. -- **Integration:** a fallback CommitRequest attaches to an unresolved live window and yields - **one** commit — the regression this design exists to avoid. -- **Compatibility:** a custom `EventTemplate` rendering `{{.Username}}` produces the sentinel - rather than an empty string. +- **Unit:** the zero value of `AttributionOutcome` **is** `AttributionNotAttempted`. Everything + below rests on it, and the paths that rely on it never assign the field, so nothing else + would catch a regression. +- **Regression:** the full cross-subsystem matrix for `matchesWindow`, with each side driven by + its real producer — the controller's `gitOutcome()` projection and the watch package's actual + resolver — including the default deployment (webhook off against attribution off) and the + `not_attempted`/`unresolved` boundary. Two literals set to the same value prove nothing here. +- **Regression:** a request whose author differs from the window's never attaches, *whatever* + the two outcomes are — so the author check cannot be made conditional on them. +- **Compatibility:** a custom `EventTemplate` rendering `{{.Username}}` still produces an empty + string for an unresolved event; the sentinel appears only in the git author header. - **e2e:** author assertions fail as `got unknown (attribution unresolved)`, naming the cause in the failure message. diff --git a/internal/controller/commitrequest_attribution_agreement_test.go b/internal/controller/commitrequest_attribution_agreement_test.go new file mode 100644 index 00000000..997297a8 --- /dev/null +++ b/internal/controller/commitrequest_attribution_agreement_test.go @@ -0,0 +1,120 @@ +// SPDX-License-Identifier: Apache-2.0 + +package controller + +import ( + "context" + "testing" + + "github.com/go-logr/logr" + "github.com/stretchr/testify/assert" + "k8s.io/apimachinery/pkg/runtime/schema" + k8stypes "k8s.io/apimachinery/pkg/types" + + "github.com/ConfigButler/gitops-reverser/internal/git" + "github.com/ConfigButler/gitops-reverser/internal/queue" + "github.com/ConfigButler/gitops-reverser/internal/watch" +) + +// absentLookup is an attribution index that never has a fact — the production shape of +// "attribution is enabled but nothing matched within the grace". +type absentLookup struct{} + +func (absentLookup) LookupAuthorResolution( + _ context.Context, + _ string, + _ schema.GroupVersionResource, + _ k8stypes.UID, + _ string, + _ bool, +) queue.AuthorResolution { + return queue.AuthorResolution{Result: queue.AttributionAbsent} +} + +// windowOutcomeAttributionOff is the outcome a commit window carries in configured-author mode. +// It is read off a zero git.Event on purpose rather than written as a constant: that IS the +// production value, because watch.Manager.attachAuthor returns early without assigning +// Attribution when the Manager's AuthorResolver is nil. +func windowOutcomeAttributionOff() git.AttributionOutcome { + return git.Event{}.Attribution +} + +// windowOutcomeAttributionMissed drives the real resolver over a lookup that has no fact, which +// is what a live watch event gets when attribution is on and nothing matched in the grace. +func windowOutcomeAttributionMissed(t *testing.T) git.AttributionOutcome { + t.Helper() + _, outcome := watch.NewAuthorResolver(absentLookup{}, 0, logr.Discard()).ResolveAuthor( + context.Background(), + "default", + schema.GroupVersionResource{Version: "v1", Resource: "configmaps"}, + k8stypes.UID("uid-1"), + "101", + true, + ) + return outcome +} + +// TestCommitRequestAndWindowOutcomesAgree is the cross-subsystem half of the P0 regression test. +// +// The gap that let the blocker through was that every existing test set both sides of the +// comparison to the same literal, so nothing ever checked that the two INDEPENDENT producers +// emit compatible values. This drives each side from its real source — the controller's +// gitOutcome projection, and the watch package's actual resolver / actual zero event — and +// asserts they agree on the only thing matching may depend on across that boundary. +// +// The default deployment is the first row: --admission-webhook defaults to false, so the +// controller emits attributionNotAttempted, while --author-attribution=false leaves the window's +// outcome at the zero value. When those two were different strings, no CommitRequest could +// attach to any window and the user's commit message was silently dropped. +func TestCommitRequestAndWindowOutcomesAgree(t *testing.T) { + tests := []struct { + name string + requestSide commitRequestAttribution + windowSide git.AttributionOutcome + wantNamesActor bool + }{ + { + name: "default deployment: webhook off, attribution off", + requestSide: attributionNotAttempted, + windowSide: windowOutcomeAttributionOff(), + }, + { + name: "webhook off, attribution on but missed", + requestSide: attributionNotAttempted, + windowSide: windowOutcomeAttributionMissed(t), + }, + { + name: "webhook on but no record, attribution off", + requestSide: attributionCommitter, + windowSide: windowOutcomeAttributionOff(), + }, + { + name: "webhook on but no record, attribution on but missed", + requestSide: attributionCommitter, + windowSide: windowOutcomeAttributionMissed(t), + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + requestOutcome := tc.requestSide.gitOutcome() + assert.Equal(t, tc.wantNamesActor, requestOutcome.NamesActor()) + assert.Equal(t, requestOutcome.NamesActor(), tc.windowSide.NamesActor(), + "the CommitRequest side produced %q and the window side produced %q; they must "+ + "agree on whether an actor was named, or the request cannot attach and the "+ + "user's commit message is silently dropped", + requestOutcome, tc.windowSide) + }) + } +} + +// TestAttributionOffLeavesZeroOutcome states the invariant the default deployment rests on, +// separately from the agreement table so a regression names itself precisely. +func TestAttributionOffLeavesZeroOutcome(t *testing.T) { + assert.Equal(t, git.AttributionNotAttempted, windowOutcomeAttributionOff(), + "configured-author mode must leave the event's outcome at AttributionNotAttempted") + assert.Equal(t, git.AttributionNotAttempted, attributionNotAttempted.gitOutcome(), + "a CommitRequest with command-author capture off must carry AttributionNotAttempted") + assert.Equal(t, git.AttributionUnresolved, windowOutcomeAttributionMissed(t), + "attribution that ran and found nothing must be AttributionUnresolved, not the zero value") +} diff --git a/internal/git/attribution_outcome_test.go b/internal/git/attribution_outcome_test.go index a0ab2aff..4b20e6b3 100644 --- a/internal/git/attribution_outcome_test.go +++ b/internal/git/attribution_outcome_test.go @@ -157,14 +157,17 @@ func TestOpenWindow_ResolvedWindowsStillSplitByAuthor(t *testing.T) { // THE REGRESSION THIS DESIGN EXISTS TO AVOID. // -// A CommitRequest that could not be attributed carries an empty author. Before the sentinel, -// that coincided with an unattributed window's empty author and attached by accident. Once the -// window carries the sentinel author the strings differ, so matching on the author alone would -// silently stop attaching and the request would land as its own commit. Matching on the -// outcome keeps them together. +// A CommitRequest that could not be attributed carries an empty author, and so does a window +// whose attribution ran and named nobody — the sentinel is a git-signature rendering only and +// never reaches the window's Author. The two therefore still match on the author, and the +// outcomes must not be allowed to push them apart: the request's outcome comes from command +// authorship and the window's from mirrored-resource attribution, which are configured +// independently, so they routinely differ. See the full cross-subsystem matrix in +// TestMatchesWindow_AcrossIndependentlyConfiguredSubsystems. func TestPendingCommitRequest_UnresolvedRequestAttachesToUnresolvedWindow(t *testing.T) { window := newOpenWindow(attributedEvent("", AttributionUnresolved), nil) - window.Author = UnresolvedAuthorUsername // as the writer will render it + require.Empty(t, window.Author, + "an unresolved window carries no author; the sentinel is applied at the write path") request := &pendingCommitRequest{ author: "", // a fallback CommitRequest has no author @@ -172,10 +175,20 @@ func TestPendingCommitRequest_UnresolvedRequestAttachesToUnresolvedWindow(t *tes gitTargetName: "target", gitTargetNamespace: "default", } - assert.True(t, request.matchesWindow(window), - "a fallback CommitRequest must still attach to the unresolved window it was meant for, "+ - "even though the two carry different author strings") + "a fallback CommitRequest must attach to the unresolved window it was meant for") + + // The default deployment: command-author capture is off, so the request says "not + // attempted" while the window says "unresolved". Neither names an actor, so they match. + notAttempted := &pendingCommitRequest{ + author: "", + attribution: AttributionNotAttempted, + gitTargetName: "target", + gitTargetNamespace: "default", + } + assert.True(t, notAttempted.matchesWindow(window), + "a request from a deployment with command-author capture off must still attach; "+ + "requiring the two subsystems' outcomes to be equal drops the user's commit message") } func TestPendingCommitRequest_DoesNotCrossAttributionOutcomes(t *testing.T) { @@ -220,15 +233,27 @@ func TestPendingCommitRequest_StillScopedToOneGitTarget(t *testing.T) { "the outcome match must not widen attachment across GitTargets") } -// Username reaches user-authored per-event templates via {{.Username}}, not only grouped -// commit messages. A custom template must render the sentinel rather than an empty string. -func TestRenderEventCommitMessage_UnresolvedUsernameReachesCustomTemplates(t *testing.T) { +// The sentinel is scoped to the git author header and deliberately does NOT reach message +// bodies or user-authored {{.Username}} templates — it is derived at the write path from the +// outcome, never stamped onto the event (see UnresolvedAuthor). An unresolved event therefore +// renders {{.Username}} exactly as configured-author mode always has: empty. Pushing a magic +// token in here would change the commit text of every deployment that has attribution misses +// and force user templates to special-case a value they never had to handle. +func TestRenderEventCommitMessage_UnresolvedUsernameStaysEmptyInTemplates(t *testing.T) { config := ResolveCommitConfig(nil) config.Message.EventTemplate = "{{.Operation}} by {{.Username}}" - event := attributedEvent(UnresolvedAuthorUsername, AttributionUnresolved) + // The production shape: attachAuthor sets UserInfo only when the outcome is resolved. + event := attributedEvent("", AttributionUnresolved) message, err := renderEventCommitMessage(event, config) require.NoError(t, err) - assert.Equal(t, "CREATE by "+UnresolvedAuthorUsername, message) + assert.Equal(t, "CREATE by ", message, + "the sentinel must not leak into user-authored templates") + + // The identity a human actually sees for this commit lives in the git author header, which + // TestCommitOptionsFor_UnresolvedAttributionUsesSentinelAuthor pins. + options := commitOptionsFor(commitWrite(event), ResolveCommitConfig(nil), nil, time.Now()) + require.NotNil(t, options.Author) + assert.Equal(t, UnresolvedAuthorDisplayName, options.Author.Name) } diff --git a/internal/git/commit_request_attach.go b/internal/git/commit_request_attach.go index a1f35fd1..c4f505c8 100644 --- a/internal/git/commit_request_attach.go +++ b/internal/git/commit_request_attach.go @@ -61,12 +61,11 @@ type AttachCommitRequest struct { // matches is attached; this binds "the open window" to "the requesting // author's open window". Author string - // Attribution is the outcome of attributing THIS CommitRequest. It must be matched - // alongside Author, not inferred from it: a request that could not be attributed carries - // an empty Author, which used to coincide with an unattributed window's empty author and - // attach by accident. Once unresolved windows carry a sentinel author that coincidence - // disappears, and matching on the outcome is what keeps the fallback request attaching to - // the window it was meant for instead of silently landing as its own commit. + // Attribution is the outcome of attributing THIS CommitRequest, from the command-authorship + // path. It is matched alongside Author rather than inferred from it, because an empty Author + // alone cannot say whether an actor was sought: it is both "attribution is off" and + // "attribution ran and named nobody". Only the NamesActor half is compared against the + // window's outcome — see matchesWindow for why the enums themselves must not be. Attribution AttributionOutcome // GitTargetName / GitTargetNamespace scope the finalize to one GitTarget. GitTargetName string @@ -111,28 +110,33 @@ type pendingCommitRequest struct { attached bool } -// matchesWindow reports whether the request identifies the given open window by -// attribution outcome, author, and GitTarget. +// matchesWindow reports whether the request identifies the given open window, by GitTarget and +// by author. // -// The outcome is matched FIRST and the author comparison only carries meaning when both sides -// resolved to a real actor. A CommitRequest that could not be attributed carries an empty -// author, and an unresolved window carries the sentinel author — different strings for the -// same situation, so comparing strings alone would stop them matching and the request would -// silently land as a separate commit. Two unresolved parties are treated as belonging together -// exactly as they did when both were the empty string; the difference is that it is now stated -// rather than an accident of two fields both happening to be empty. +// The two attribution outcomes compared here are produced by DIFFERENT, INDEPENDENTLY +// CONFIGURED subsystems: the window's comes from mirrored-resource attribution +// (--author-attribution, audit facts), the request's from command authorship +// (--admission-webhook, its own Redis corner) — see cmd/main.go:311-316. So they are matched on +// AttributionOutcome.NamesActor, not for enum equality. Requiring the enums to be equal silently +// couples the two flags: with attribution off and the webhook on the window says "not attempted" +// while the request says "unresolved", and with attribution on but missing and the webhook off +// it is the other way round. Both are real deployments, both attach correctly today, and exact +// equality would stop both — dropping the user's commit message into a separate default-message +// commit with no error anywhere. +// +// The author comparison stays UNCONDITIONAL, and the outcome class is an additional guard on +// top of it — never a replacement for it. Skipping the author check when neither side names an +// actor looks equivalent (an unnamed actor leaves the author empty on both sides, so the two +// empties compare equal anyway) but is not: it makes cross-author attachment depend on the +// outcome fields being right, so any path that leaves an outcome unset while the author IS set +// would let one author's request finalize another's window. Comparing both costs nothing and +// keeps "bob never claims alice's window" true regardless of what the outcomes say. func (p *pendingCommitRequest) matchesWindow(w *openWindow) bool { if p == nil || w == nil { return false } - if p.attribution != w.Attribution { - return false - } - if p.attribution == AttributionUnresolved { - // Neither side knows who acted; there is no author to agree on. - return p.gitTargetName == w.GitTarget && p.gitTargetNamespace == w.GitTargetNamespace - } return p.author == w.Author && + p.attribution.NamesActor() == w.Attribution.NamesActor() && p.gitTargetName == w.GitTarget && p.gitTargetNamespace == w.GitTargetNamespace } diff --git a/internal/git/commit_request_attach_test.go b/internal/git/commit_request_attach_test.go index b542995a..39ae795a 100644 --- a/internal/git/commit_request_attach_test.go +++ b/internal/git/commit_request_attach_test.go @@ -487,3 +487,130 @@ func TestAttach_ConflictReplayResolvesToPostReplaySHA(t *testing.T) { require.NoError(t, err) assert.Equal(t, message, commit.Message, "the replayed commit keeps the user's message") } + +// TestAttributionZeroValueIsNotAttempted pins the coupling the whole matching rule rests on: +// the zero value of AttributionOutcome must BE AttributionNotAttempted. +// +// Most producers of an Event never assign Attribution — reconcile, resync, bootstrap, and +// configured-author mode's early return in watch.Manager.attachAuthor all leave it zero, and all +// of them mean "no actor was sought". When this constant was "not_attempted" the zero value was a +// silent fourth state that compared equal to no named outcome, and no test caught it because +// every case set both sides to the zero value and compared "" to "". +func TestAttributionZeroValueIsNotAttempted(t *testing.T) { + var zero AttributionOutcome + assert.Equal(t, AttributionNotAttempted, zero, + "the zero value must be AttributionNotAttempted; three doc comments and every non-live "+ + "event path depend on it") + assert.False(t, zero.NamesActor(), "an unset outcome names no actor") + assert.False(t, AttributionUnresolved.NamesActor(), "attribution ran and named nobody") + assert.True(t, AttributionResolved.NamesActor(), "only a resolved outcome names an actor") +} + +// TestMatchesWindow_AcrossIndependentlyConfiguredSubsystems is the permanent regression test for +// the merge blocker: in the DEFAULT deployment no CommitRequest could attach to any window, so +// the user's commit message was silently dropped and the change landed under the generated +// message with no error anywhere. +// +// The window's outcome and the request's outcome come from two subsystems configured by two +// different flags (--author-attribution vs --admission-webhook, cmd/main.go:311-316), so this +// walks every pair either side can actually produce and names the deployment that produces it. +// TestCommitRequestAndWindowOutcomesAgree in internal/controller pins that these are the real +// values the two producers emit; here we pin what the matching rule does with them. +func TestMatchesWindow_AcrossIndependentlyConfiguredSubsystems(t *testing.T) { + const otherAuthor = "bob" + + tests := []struct { + name string + deployment string + window AttributionOutcome + windowAuth string + request AttributionOutcome + reqAuth string + want bool + }{ + { + name: "attribution off, webhook off", + deployment: "the DEFAULT deployment: --author-attribution=false, --admission-webhook=false", + window: AttributionNotAttempted, request: AttributionNotAttempted, want: true, + }, + { + name: "attribution off, webhook on but missed", + deployment: "webhook enabled without Redis, or no admission record for the request", + window: AttributionNotAttempted, request: AttributionUnresolved, want: true, + }, + { + name: "attribution on but unresolved, webhook off", + deployment: "attribution enabled, no audit fact within the grace; webhook off", + window: AttributionUnresolved, request: AttributionNotAttempted, want: true, + }, + { + name: "attribution on but unresolved, webhook on but missed", + deployment: "both enabled, neither named an actor", + window: AttributionUnresolved, request: AttributionUnresolved, want: true, + }, + { + name: "both resolved to the same actor", + deployment: "the fully configured deployment, request and window agree", + window: AttributionResolved, windowAuth: "alice", + request: AttributionResolved, reqAuth: "alice", want: true, + }, + { + name: "both resolved to different actors", + deployment: "two humans editing the same GitTarget; must never cross-attach", + window: AttributionResolved, windowAuth: "alice", + request: AttributionResolved, reqAuth: otherAuthor, want: false, + }, + { + name: "window named an actor, request did not", + deployment: "alice's window must not absorb an unattributable request", + window: AttributionResolved, windowAuth: "alice", + request: AttributionNotAttempted, want: false, + }, + { + name: "request named an actor, window did not", + deployment: "alice's request must not claim an unattributable window", + window: AttributionNotAttempted, + request: AttributionResolved, reqAuth: "alice", want: false, + }, + { + name: "different authors, neither outcome names an actor", + deployment: "not producible today (a set author implies a resolved outcome), but the " + + "author check must not become conditional on the outcome: if it did, any path that " + + "left an outcome unset while the author was set would let bob finalize alice's window", + window: AttributionNotAttempted, windowAuth: "alice", + request: AttributionNotAttempted, reqAuth: otherAuthor, want: false, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + p := &pendingCommitRequest{ + author: tc.reqAuth, + attribution: tc.request, + gitTargetName: crTarget, + gitTargetNamespace: "default", + } + w := &openWindow{ + Author: tc.windowAuth, + Attribution: tc.window, + GitTarget: crTarget, + GitTargetNamespace: "default", + } + assert.Equal(t, tc.want, p.matchesWindow(w), tc.deployment) + }) + } +} + +// TestMatchesWindow_GitTargetAlwaysScopes checks the GitTarget scope survives the outcome +// rework: no attribution pairing may let a request finalize another target's window. +func TestMatchesWindow_GitTargetAlwaysScopes(t *testing.T) { + for _, o := range []AttributionOutcome{AttributionNotAttempted, AttributionUnresolved, AttributionResolved} { + p := &pendingCommitRequest{attribution: o, gitTargetName: crTarget, gitTargetNamespace: "default"} + assert.False(t, p.matchesWindow(&openWindow{ + Attribution: o, GitTarget: "team-b", GitTargetNamespace: "default", + }), "outcome %q must not match across GitTarget names", o) + assert.False(t, p.matchesWindow(&openWindow{ + Attribution: o, GitTarget: crTarget, GitTargetNamespace: "other", + }), "outcome %q must not match across GitTarget namespaces", o) + } +} diff --git a/internal/git/open_window.go b/internal/git/open_window.go index 189ca8b8..876fc5db 100644 --- a/internal/git/open_window.go +++ b/internal/git/open_window.go @@ -10,9 +10,12 @@ type openWindow struct { // Author is event.UserInfo.Username verbatim. Author string // Attribution is the window's attribution outcome, taken from its first event. It pairs - // with Author for matching: two events share a window only when BOTH the outcome and the + // with Author for coalescing: two events share a window only when BOTH the outcome and the // author agree, so an unresolved event never coalesces with a resolved one that happens - // to carry an empty username. + // to carry an empty username. Exact enum equality is right HERE — canAppend compares two + // events from the same watch pipeline, so the outcomes are directly comparable. It is not + // right when comparing against a CommitRequest, which a different subsystem attributes; + // see pendingCommitRequest.matchesWindow. Attribution AttributionOutcome // GitTarget is the target this window is bound to. Each finalized commit // covers exactly one target so per-target encryption and bootstrap diff --git a/internal/git/types.go b/internal/git/types.go index c4f2f64f..ed0842be 100644 --- a/internal/git/types.go +++ b/internal/git/types.go @@ -28,7 +28,16 @@ type AttributionOutcome string const ( // AttributionNotAttempted is configured-author mode: attribution is switched off, so the // committer legitimately IS the author and no actor was ever sought. - AttributionNotAttempted AttributionOutcome = "not_attempted" + // + // It is deliberately the EMPTY string, so that it is also the ZERO VALUE of the type. Most + // paths that build an Event never assign Attribution at all — reconcile, resync, bootstrap, + // and configured-author mode's early return in the watch pipeline — and every one of them + // means exactly "no actor was sought". Any other string would make the zero value a silent + // fourth state equal to none of the three named outcomes, which is precisely the bug that + // stopped every CommitRequest attaching in the default deployment. Nothing serializes this + // value (it reaches no CRD field and no metric label; authorKind branches on the typed + // value), so the empty string costs nothing. TestAttributionZeroValueIsNotAttempted pins it. + AttributionNotAttempted AttributionOutcome = "" // AttributionResolved means an audit fact named the actor. AttributionResolved AttributionOutcome = "resolved" // AttributionUnresolved means attribution ran and did not arrive at an actor. @@ -41,13 +50,37 @@ const ( AttributionUnresolved AttributionOutcome = "unresolved" ) -// UnresolvedAuthor is the identity written to Git when attribution ran and did not resolve an -// actor. It exists so an unresolved attribution is visible in `git log` instead of being -// indistinguishable from a configured-author commit. +// NamesActor reports whether the outcome carries an actor to compare against. +// +// This is the ONLY distinction that survives across subsystem boundaries. Whether an outcome +// is "not attempted" or "unresolved" is a property of how the subsystem that produced it is +// configured — and the mirrored-resource attribution path (--author-attribution) and the +// command-authorship path (--admission-webhook) are configured independently of each other +// (cmd/main.go:311-316). Two independently configured producers can therefore disagree about +// the enum while agreeing perfectly about the thing that matters: whether there is an actor. +// Compare the enums across that boundary and you couple the two flags; compare NamesActor and +// you do not. Within a single subsystem the enum itself is meaningful and IS compared directly +// (openWindow.canAppend), because both sides come from the same producer. +func (o AttributionOutcome) NamesActor() bool { + return o == AttributionResolved +} + +// UnresolvedAuthor is the identity written to the Git AUTHOR HEADER when attribution ran and +// did not resolve an actor. It exists so an unresolved attribution is visible in `git log` +// instead of being indistinguishable from a configured-author commit. +// +// Scope: the git author header, and nothing else. It is DERIVED at the write path +// (commitOptionsFor) from the carried AttributionOutcome — it is never stamped onto an Event. +// The outcome is the fact; this identity is one rendering of it. So the sentinel deliberately +// does NOT reach window grouping, the grouped commit-message body, or user-authored +// {{.Username}} templates: those keep the empty author they have always had for an unnamed +// actor, on both this path and in configured-author mode. Pushing a magic token into message +// bodies would change the commit text of every existing deployment that has attribution misses, +// and force user templates to special-case a value they never had to handle. // -// Three fields, three different strings, because UserInfo feeds three surfaces: -// - Username reaches window grouping, the grouped commit message body, AND user-authored -// per-event templates via {{.Username}} — so it must stand alone as a machine token. +// Three fields, three different strings, because the header needs all three: +// - Username is the stable machine token, so tooling that parses the header has something +// greppable that will not drift with the human-facing wording. // - DisplayName is what a human reads in `git log`, so it leads with what they care about. // - Email uses the RFC 2606 reserved .invalid TLD, so it can never collide with a real // address and never routes mail. @@ -433,8 +466,10 @@ type Event struct { // It is the authority for author rendering, the author_kind metric, and CommitRequest // window matching — none of which may infer the outcome from UserInfo, because an empty // or sentinel username cannot distinguish "attribution is off" from "attribution ran and - // found nothing". The zero value is AttributionNotAttempted, which is correct for every - // non-live path (reconcile, resync, bootstrap) where no actor is ever sought. + // found nothing". The zero value is AttributionNotAttempted — the constant is defined as the + // empty string precisely so that it is — which is correct for every non-live path + // (reconcile, resync, bootstrap) where no actor is ever sought, and for configured-author + // mode. attachAuthor is the only assignment to this field outside tests. Attribution AttributionOutcome // Path is the POSIX-like relative path prefix for this event's files. diff --git a/internal/watch/author_resolver.go b/internal/watch/author_resolver.go index ca0eb0d8..25d8533b 100644 --- a/internal/watch/author_resolver.go +++ b/internal/watch/author_resolver.go @@ -77,6 +77,11 @@ type AuthorResolver interface { // author) versus AttributionUnresolved (attribution ran and found nothing — a gap worth // surfacing). An empty UserInfo cannot distinguish them, which is exactly how the loss // stayed invisible. A resolved outcome always carries a non-empty UserInfo. + // + // In production this method only ever returns the latter two: configured-author mode is + // expressed by leaving Manager.AuthorResolver nil (attachAuthor returns early, leaving the + // event's zero AttributionNotAttempted), never by constructing a resolver over a nil + // lookup. cmd/main.go:258 only builds one with a non-nil index. ResolveAuthor( ctx context.Context, providerName string, @@ -115,7 +120,9 @@ func (r *attributionResolver) ResolveAuthor( ) (git.UserInfo, git.AttributionOutcome) { start := time.Now() // A nil lookup is configured-author mode: attribution was never switched on, so nothing - // was attempted and the committer legitimately authors the commit. + // was attempted and the committer legitimately authors the commit. Defensive only — + // production expresses that mode with a nil Manager.AuthorResolver, so this branch is + // unreachable there (cmd/main.go:258 always passes a non-nil index). if r.lookup == nil { recordAttributionResolution(ctx, gvr, queue.AttributionAbsent, time.Since(start)) return git.UserInfo{}, git.AttributionNotAttempted diff --git a/internal/watch/target_watch.go b/internal/watch/target_watch.go index fa83f059..867863db 100644 --- a/internal/watch/target_watch.go +++ b/internal/watch/target_watch.go @@ -750,8 +750,11 @@ func (m *Manager) attachAuthor( gvr schema.GroupVersionResource, u *unstructured.Unstructured, ) { - // A nil resolver is configured-author mode: nothing is attempted, and the zero - // AttributionNotAttempted on the event is already correct. + // A nil resolver is configured-author mode: nothing is attempted, and the event's zero + // Attribution is already AttributionNotAttempted — the constant is the empty string so that + // this early return needs no stamp. Do not "fix" this by giving the constant a name-shaped + // value; every non-live path (reconcile, resync, bootstrap) relies on the same zero value, + // and a non-empty constant turns all of them into a fourth state that matches nothing. if m.AuthorResolver == nil { return } From 4a3780e5407cbf01d98774a2844e4ccf01c4a8a6 Mon Sep 17 00:00:00 2001 From: Simon Koudijs Date: Mon, 20 Jul 2026 14:21:28 +0000 Subject: [PATCH 13/17] fix(giteaclient): terminate pagination on an empty page, not a short one, and cap the loop MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The keys pagination added in 2f29c39 terminated on `len(keys) < keyPageSize`, which assumes the server's page size equals the one requested. It does not: MAX_RESPONSE_ITEMS is operator-configurable and ToCorrectPageSize clamps every request down to it, so against a server whose cap is below 50 EVERY page is short and the loop stops after page 1 with no error. Measured against a server clamping to 30 with a user holding 100 keys: `keys=30 (of 100)`, FindUserKeyByTitle found=false — byte-for-byte the original failure (`422 Key title has been used` for a key that demonstrably exists). It passed only because it happened to match an unpinned server default; the e2e lab sets no [api] block in gitea-values.yaml, so nothing held that default in place. Only an empty page terminates now. Correct for any server page size, at one extra round trip. (Gitea does emit X-Total-Count and Link, but Client.Do returns (int, []byte, error) and discards headers, so that route needs a signature change.) Separately the loop had no cap. Against a server that ignores `page` and returns a full page forever it issued 7193 requests in 1.5s before the context deadline, growing `all` by 50 structs each time. All callers pass deadlines, so it degraded to an undiagnosable "context deadline exceeded" rather than a hang. A maxKeyPages guard names the cause instead. The comment at the old termination was wrong on both halves and is rewritten with the fix: a short page is not necessarily the last page, and the server that loops forever is one that ignores `page`, not one that ignores the limit. The stranded "// ListUserKeys returns every public key…" doc comment, left above `const keyPageSize` when the new block was inserted, now documents the const it actually sits on. ListRepoHooks had the identical unpaginated bug — /repos/{owner}/{repo}/hooks paginates the same way. Latent only because e2e creates a fresh repo per spec and never accumulates a page of hooks on one repo. Both now go through one listPaginated helper, so the termination rule is stated once. Tests: a server that clamps below the requested limit (the regression above), a server that ignores `page` (the cap), and the exact-page-multiple case. The fakes now honour `page` rather than serving the same body forever — three of them would have tripped the new cap, which is the correct client behaviour against an unrealistic fake. Also fixes the handler-goroutine t.Fatal in this file's fakes: Fatal from an httptest handler runs runtime.Goexit on that goroutine, killing the connection mid-response, so the client reports a spurious EOF on top of the real message — two reported errors for one fault, misleading one first. Failures are now recorded and asserted on the test goroutine (or reported with t.Errorf, which does not Goexit). Co-Authored-By: Claude Opus 4.8 (1M context) --- internal/giteaclient/bootstrap.go | 24 ++- internal/giteaclient/bootstrap_test.go | 11 +- internal/giteaclient/keys.go | 35 ++-- internal/giteaclient/keys_test.go | 269 +++++++++++++++++++------ internal/giteaclient/pagination.go | 50 +++++ 5 files changed, 289 insertions(+), 100 deletions(-) create mode 100644 internal/giteaclient/pagination.go diff --git a/internal/giteaclient/bootstrap.go b/internal/giteaclient/bootstrap.go index 1a8182eb..9a577dec 100644 --- a/internal/giteaclient/bootstrap.go +++ b/internal/giteaclient/bootstrap.go @@ -122,18 +122,20 @@ func (c *Client) EnsureOrgRepo( } } -// ListRepoHooks returns every webhook configured on owner/repo. +// hookPageSize / maxHookPages bound the webhook listing. /repos/{owner}/{repo}/hooks paginates +// exactly like the keys endpoint — it is clamped to MAX_RESPONSE_ITEMS and has no "return +// everything" mode — so an unpaginated read goes blind past the first page. Latent rather than +// live today only because e2e creates a fresh repo per spec and never accumulates a page of +// hooks on one repo; the callers that delete hooks would silently miss the rest. +const ( + hookPageSize = 50 + maxHookPages = 200 +) + +// ListRepoHooks returns every webhook configured on owner/repo, following pagination. func (c *Client) ListRepoHooks(ctx context.Context, owner, repo string) ([]RepoHook, error) { - path := "/repos/" + PathEscape(owner) + "/" + PathEscape(repo) + "/hooks" - var hooks []RepoHook - code, raw, err := c.Do(ctx, http.MethodGet, path, nil, &hooks) - if err != nil { - return nil, err - } - if code != http.StatusOK { - return nil, unexpectedStatus(http.MethodGet, path, code, raw) - } - return hooks, nil + base := "/repos/" + PathEscape(owner) + "/" + PathEscape(repo) + "/hooks" + return listPaginated[RepoHook](ctx, c, base, hookPageSize, maxHookPages) } // DeleteRepoHook removes a single repository webhook. Missing hooks are treated as success. diff --git a/internal/giteaclient/bootstrap_test.go b/internal/giteaclient/bootstrap_test.go index 839a3ae9..d0a28360 100644 --- a/internal/giteaclient/bootstrap_test.go +++ b/internal/giteaclient/bootstrap_test.go @@ -150,12 +150,21 @@ func webhookTestHandler(t *testing.T) http.HandlerFunc { w.WriteHeader(http.StatusNoContent) case r.Method == http.MethodGet && r.URL.Path == "/repos/testorg/demo/hooks": w.Header().Set("Content-Type", "application/json") + // The hooks listing paginates; everything past page 1 is empty, and that empty page + // is what terminates ListRepoHooks. + if r.URL.Query().Get("page") != "1" { + _, _ = w.Write([]byte(`[]`)) + return + } _, _ = w.Write([]byte( `[{"id":42,"type":"gitea","active":true,` + `"config":{"url":"http://receiver.example/hook/demo","content_type":"json"}}]`, )) default: - t.Fatalf("unexpected request: %s %s", r.Method, r.URL.Path) + // t.Errorf, not t.Fatalf: Fatal from a handler goroutine Goexits it mid-response and + // the client reports a spurious EOF on top of the real failure. + t.Errorf("unexpected request: %s %s", r.Method, r.URL.Path) + w.WriteHeader(http.StatusBadRequest) } } } diff --git a/internal/giteaclient/keys.go b/internal/giteaclient/keys.go index ba3a41cb..e007acc3 100644 --- a/internal/giteaclient/keys.go +++ b/internal/giteaclient/keys.go @@ -24,12 +24,19 @@ func NormalizeAuthorizedKey(k string) string { return fields[0] + " " + fields[1] } -// ListUserKeys returns every public key registered for the named user. -// keyPageSize is the per-page item count requested when listing keys. Gitea caps it at -// MAX_RESPONSE_ITEMS (50 by default) and silently truncates beyond that, so requesting the cap -// keeps the number of round trips low without relying on the server's DEFAULT_PAGING_NUM. +// keyPageSize is the per-page item count REQUESTED when listing keys. It is a throughput hint +// only — never a termination signal. Gitea clamps every request down to MAX_RESPONSE_ITEMS +// (ToCorrectPageSize), which is operator-configurable, so the server's actual page size may be +// smaller than this and the loop must not assume otherwise. const keyPageSize = 50 +// maxKeyPages bounds the pagination loop. Termination depends on the server eventually +// returning an empty page, which a server that ignores the `page` parameter never does — it +// would serve a full first page forever, and the loop would spin until the caller's context +// deadline and report an undiagnosable "context deadline exceeded" instead of naming the cause. +// At keyPageSize this ceiling is 10 000 keys, far past any real user. +const maxKeyPages = 200 + // ListUserKeys returns EVERY public key on the named user, following pagination. // // The pagination is load-bearing, not tidiness. Gitea's default page size is 30, and this @@ -41,25 +48,7 @@ const keyPageSize = 50 // Gitea, so the second run onward exceeds one page and the stable-titled `playground` key // becomes unreachable. func (c *Client) ListUserKeys(ctx context.Context, login string) ([]PublicKey, error) { - var all []PublicKey - base := "/users/" + PathEscape(login) + "/keys" - for page := 1; ; page++ { - var keys []PublicKey - path := fmt.Sprintf("%s?page=%d&limit=%d", base, page, keyPageSize) - code, raw, err := c.Do(ctx, http.MethodGet, path, nil, &keys) - if err != nil { - return nil, err - } - if code != http.StatusOK { - return nil, unexpectedStatus(http.MethodGet, path, code, raw) - } - all = append(all, keys...) - // A short page is the last page. An empty page also terminates, which guards against a - // server that ignores the limit and would otherwise loop forever. - if len(keys) < keyPageSize { - return all, nil - } - } + return listPaginated[PublicKey](ctx, c, "/users/"+PathEscape(login)+"/keys", keyPageSize, maxKeyPages) } // FindUserKey looks up a public key on the named user by key material, diff --git a/internal/giteaclient/keys_test.go b/internal/giteaclient/keys_test.go index d2490f15..e482dc31 100644 --- a/internal/giteaclient/keys_test.go +++ b/internal/giteaclient/keys_test.go @@ -9,6 +9,7 @@ import ( "net/http/httptest" "strconv" "strings" + "sync" "testing" ) @@ -50,30 +51,41 @@ func newEnsureUserKeyAsAdminTestHandler( ) http.Handler { t.Helper() + // These handlers report with t.Errorf, never t.Fatalf: Fatal from a non-test goroutine runs + // runtime.Goexit on the handler, killing the connection mid-response, so the client reports a + // spurious EOF on top of the real failure. mux := http.NewServeMux() mux.HandleFunc("/users/giteaadmin/keys", func(w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodGet { - t.Fatalf("unexpected request: %s %s", r.Method, r.URL.Path) + t.Errorf("unexpected request: %s %s", r.Method, r.URL.Path) + w.WriteHeader(http.StatusBadRequest) + return } w.Header().Set("Content-Type", "application/json") - if !*deleteCalled { - _, _ = w.Write([]byte( - `[{"id":7,"title":"E2E Transport Key playground","key":"ssh-rsa old-key","fingerprint":"old"}]`, - )) + // Honour the page parameter: everything past the first page is empty, which is what + // terminates ListUserKeys. + if r.URL.Query().Get("page") != "1" || *deleteCalled { + _, _ = w.Write([]byte(`[]`)) return } - _, _ = w.Write([]byte(`[]`)) + _, _ = w.Write([]byte( + `[{"id":7,"title":"E2E Transport Key playground","key":"ssh-rsa old-key","fingerprint":"old"}]`, + )) }) mux.HandleFunc("/admin/users/giteaadmin/keys/7", func(w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodDelete { - t.Fatalf("unexpected request: %s %s", r.Method, r.URL.Path) + t.Errorf("unexpected request: %s %s", r.Method, r.URL.Path) + w.WriteHeader(http.StatusBadRequest) + return } *deleteCalled = true w.WriteHeader(http.StatusNoContent) }) mux.HandleFunc("/admin/users/giteaadmin/keys", func(w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodPost { - t.Fatalf("unexpected request: %s %s", r.Method, r.URL.Path) + t.Errorf("unexpected request: %s %s", r.Method, r.URL.Path) + w.WriteHeader(http.StatusBadRequest) + return } *createCalled = true w.Header().Set("Content-Type", "application/json") @@ -90,9 +102,15 @@ func TestClientFindUserKeyByTitle_ReturnsMatch(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodGet || r.URL.Path != "/users/giteaadmin/keys" { - t.Fatalf("unexpected request: %s %s", r.Method, r.URL.Path) + t.Errorf("unexpected request: %s %s", r.Method, r.URL.Path) // never Fatal off the test goroutine + w.WriteHeader(http.StatusBadRequest) + return } w.Header().Set("Content-Type", "application/json") + if r.URL.Query().Get("page") != "1" { + _, _ = w.Write([]byte(`[]`)) // the empty page that terminates the listing + return + } _, _ = w.Write([]byte( `[` + `{"id":1,"title":"first","key":"ssh-ed25519 AAAA first","fingerprint":"fp1"},` + @@ -135,6 +153,104 @@ func keysPageJSON(start, end, total int, lastTitle string) string { return b.String() } +// keysServer is a fake /users/{login}/keys endpoint that records what the client asked for and +// what it got wrong. +// +// Handler goroutine failures are RECORDED, not raised: calling t.Fatal from an httptest handler +// runs runtime.Goexit on the handler goroutine, which kills the connection mid-response and +// surfaces a spurious `EOF` from the client on top of the real message — two reported errors for +// one fault, with the misleading one first. Assertions run on the test goroutine via check(). +type keysServer struct { + mu sync.Mutex + // serverPageSize is what the server actually returns per page, regardless of the limit the + // client requests. Gitea's ToCorrectPageSize clamps to MAX_RESPONSE_ITEMS, which operators + // can set below our requested keyPageSize. + serverPageSize int + total int + lastTitle string + // ignorePageParam models a server that serves page 1 forever. + ignorePageParam bool + pagesServed []int + problems []string +} + +func (s *keysServer) failf(format string, args ...any) { + s.mu.Lock() + defer s.mu.Unlock() + s.problems = append(s.problems, fmt.Sprintf(format, args...)) +} + +// check surfaces recorded handler problems on the test goroutine, where t.Fatal is safe. +func (s *keysServer) check(t *testing.T) { + t.Helper() + s.mu.Lock() + defer s.mu.Unlock() + if len(s.problems) > 0 { + t.Fatalf("fake keys server rejected the client's requests:\n %s", strings.Join(s.problems, "\n ")) + } +} + +func (s *keysServer) requests() []int { + s.mu.Lock() + defer s.mu.Unlock() + return append([]int(nil), s.pagesServed...) +} + +// requireKeysPageRequest validates the request is an explicitly paginated keys read and returns +// the requested page. An unpaginated read is the bug this file exists to prevent. +func (s *keysServer) requireKeysPageRequest(r *http.Request) (int, bool) { + if r.Method != http.MethodGet || r.URL.Path != "/users/giteaadmin/keys" { + s.failf("unexpected request: %s %s", r.Method, r.URL.Path) + return 0, false + } + query := r.URL.Query() + page := query.Get("page") + if page == "" { + s.failf("ListUserKeys must request an explicit page; an unpaginated read silently truncates at 30") + return 0, false + } + if got := query.Get("limit"); got != strconv.Itoa(keyPageSize) { + s.failf("limit = %q, want %d", got, keyPageSize) + return 0, false + } + n, err := strconv.Atoi(page) + if err != nil { + s.failf("bad page parameter %q: %v", page, err) + return 0, false + } + return n, true +} + +func (s *keysServer) handler() http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + page, ok := s.requireKeysPageRequest(r) + if !ok { + w.WriteHeader(http.StatusBadRequest) + return + } + s.mu.Lock() + s.pagesServed = append(s.pagesServed, page) + tooMany := len(s.pagesServed) > maxKeyPages+1 + s.mu.Unlock() + if tooMany { + s.failf("client exceeded the maxKeyPages cap") + w.WriteHeader(http.StatusInternalServerError) + return + } + + if s.ignorePageParam { + page = 1 + } + start := (page - 1) * s.serverPageSize + w.Header().Set("Content-Type", "application/json") + if start >= s.total { + _, _ = w.Write([]byte(`[]`)) + return + } + _, _ = w.Write([]byte(keysPageJSON(start, min(start+s.serverPageSize, s.total), s.total, s.lastTitle))) + }) +} + // Gitea paginates key listings: DEFAULT_PAGING_NUM is 30 and MAX_RESPONSE_ITEMS is 50, and // `ToCorrectPageSize` clamps a limit of 0 or less straight back to the default — so there is // no server-side "return everything" mode for this endpoint. An unpaginated listing therefore @@ -149,42 +265,33 @@ func TestClientListUserKeys_FollowsPaginationBeyondTheFirstPage(t *testing.T) { t.Parallel() const ( - pageSize = 50 total = 124 // > 2 full pages, so a single-page read cannot pass by luck lastTitle = "E2E Transport Key playground" ) - var pagesServed []string - - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - page := requireKeysPageRequest(t, r) - pagesServed = append(pagesServed, strconv.Itoa(page)) - - start := (page - 1) * pageSize - w.Header().Set("Content-Type", "application/json") - if start >= total { - _, _ = w.Write([]byte(`[]`)) - return - } - _, _ = w.Write([]byte(keysPageJSON(start, min(start+pageSize, total), total, lastTitle))) - })) + fake := &keysServer{serverPageSize: keyPageSize, total: total, lastTitle: lastTitle} + server := httptest.NewServer(fake.handler()) defer server.Close() client := New(server.URL, "admin", "password") keys, err := client.ListUserKeys(context.Background(), "giteaadmin") + fake.check(t) if err != nil { t.Fatalf("ListUserKeys() error = %v", err) } if len(keys) != total { t.Fatalf("ListUserKeys() returned %d keys, want all %d — pagination was not followed", len(keys), total) } - if len(pagesServed) != 3 { - t.Fatalf("pages requested = %v, want 3 (two full pages plus a short final page)", pagesServed) + // Four requests, not three: termination is an EMPTY page, so the 24-key page 3 does not end + // the loop. That extra round trip is the price of being correct for any server page size. + if got := fake.requests(); len(got) != 4 { + t.Fatalf("pages requested = %v, want 4 (three pages of keys, then the empty page that ends it)", got) } // The point of the loop: a title on a later page must still be findable, because that is // what EnsureUserKeyAsAdmin needs in order to replace a stale key instead of colliding. found, ok, err := client.FindUserKeyByTitle(context.Background(), "giteaadmin", lastTitle) + fake.check(t) if err != nil { t.Fatalf("FindUserKeyByTitle() error = %v", err) } @@ -196,61 +303,93 @@ func TestClientListUserKeys_FollowsPaginationBeyondTheFirstPage(t *testing.T) { } } -// requireKeysPageRequest asserts the request is an explicitly paginated keys read and returns -// the requested page. An unpaginated read is the bug this file exists to prevent, so it fails -// here rather than silently truncating at the server default. -func requireKeysPageRequest(t *testing.T, r *http.Request) int { - t.Helper() - if r.Method != http.MethodGet || r.URL.Path != "/users/giteaadmin/keys" { - t.Fatalf("unexpected request: %s %s", r.Method, r.URL.Path) - } - query := r.URL.Query() - page := query.Get("page") - if page == "" { - t.Fatal("ListUserKeys must request an explicit page; an unpaginated read silently truncates at 30") +// A server whose page size is SMALLER than the limit we request must still be paginated +// correctly. MAX_RESPONSE_ITEMS is operator-configurable and ToCorrectPageSize clamps every +// request down to it, so against such a server EVERY page is short — and a loop that stops on a +// short page stops after page 1, byte-for-byte re-creating the original bug (FindUserKeyByTitle +// missing an existing key, then `422 Key title has been used`). The e2e lab pins no [api] block +// in gitea-values.yaml, so today's pass depends only on an unpinned server default. +func TestClientListUserKeys_ServerPageSizeSmallerThanRequested(t *testing.T) { + t.Parallel() + + const ( + serverPageSize = 30 // MAX_RESPONSE_ITEMS set below our requested keyPageSize + total = 100 + lastTitle = "E2E Transport Key playground" + ) + fake := &keysServer{serverPageSize: serverPageSize, total: total, lastTitle: lastTitle} + server := httptest.NewServer(fake.handler()) + defer server.Close() + + client := New(server.URL, "admin", "password") + + keys, err := client.ListUserKeys(context.Background(), "giteaadmin") + fake.check(t) + if err != nil { + t.Fatalf("ListUserKeys() error = %v", err) } - if got := query.Get("limit"); got != "50" { - t.Fatalf("limit = %q, want 50 (Gitea's MAX_RESPONSE_ITEMS; a larger value is silently clamped)", got) + if len(keys) != total { + t.Fatalf("ListUserKeys() returned %d keys, want all %d — the loop stopped on a short page, "+ + "which is not the last page when the server clamps below the requested limit", len(keys), total) } - n, err := strconv.Atoi(page) + + _, ok, err := client.FindUserKeyByTitle(context.Background(), "giteaadmin", lastTitle) + fake.check(t) if err != nil { - t.Fatalf("bad page parameter %q: %v", page, err) + t.Fatalf("FindUserKeyByTitle() error = %v", err) + } + if !ok { + t.Fatal("FindUserKeyByTitle() did not find an existing key: this is the `422 Key title has " + + "been used` failure the pagination exists to prevent") } - return n } // A listing that exactly fills a page must still terminate: the client asks for one more page, -// gets an empty body, and stops. Without that, an exact multiple would either loop forever or -// drop the tail. +// gets an empty body, and stops. func TestClientListUserKeys_ExactPageMultipleTerminates(t *testing.T) { t.Parallel() - const pageSize = 50 - requests := 0 - - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - requests++ - if requests > 5 { - t.Fatal("ListUserKeys did not terminate on an empty page") - } - page := requireKeysPageRequest(t, r) - w.Header().Set("Content-Type", "application/json") - if page != 1 { - _, _ = w.Write([]byte(`[]`)) - return - } - _, _ = w.Write([]byte(keysPageJSON(0, pageSize, pageSize+1, "unused"))) - })) + fake := &keysServer{serverPageSize: keyPageSize, total: keyPageSize, lastTitle: "unused"} + server := httptest.NewServer(fake.handler()) defer server.Close() keys, err := New(server.URL, "admin", "password").ListUserKeys(context.Background(), "giteaadmin") + fake.check(t) if err != nil { t.Fatalf("ListUserKeys() error = %v", err) } - if len(keys) != pageSize { - t.Fatalf("ListUserKeys() returned %d keys, want %d", len(keys), pageSize) + if len(keys) != keyPageSize { + t.Fatalf("ListUserKeys() returned %d keys, want %d", len(keys), keyPageSize) + } + if got := fake.requests(); len(got) != 2 { + t.Fatalf("requests = %v, want 2 (the full page, then the empty page that ends the loop)", got) + } +} + +// A server that ignores the `page` parameter serves a full first page forever. Without a cap the +// loop spins until the caller's context deadline and reports `context deadline exceeded`, which +// names neither the endpoint nor the cause. The cap turns it into a diagnosable error, bounded. +func TestClientListUserKeys_CapsPagesAgainstANonPaginatingServer(t *testing.T) { + t.Parallel() + + fake := &keysServer{ + serverPageSize: keyPageSize, + total: 1_000_000, + lastTitle: "unused", + ignorePageParam: true, + } + server := httptest.NewServer(fake.handler()) + defer server.Close() + + _, err := New(server.URL, "admin", "password").ListUserKeys(context.Background(), "giteaadmin") + fake.check(t) + if err == nil { + t.Fatal("ListUserKeys() returned no error against a server that never paginates") + } + if !strings.Contains(err.Error(), "ignore the page parameter") { + t.Fatalf("ListUserKeys() error = %v, want it to name the non-paginating server", err) } - if requests != 2 { - t.Fatalf("requests = %d, want 2 (the full page, then the empty page that ends the loop)", requests) + if got := len(fake.requests()); got != maxKeyPages { + t.Fatalf("requests = %d, want exactly the %d-page cap", got, maxKeyPages) } } diff --git a/internal/giteaclient/pagination.go b/internal/giteaclient/pagination.go new file mode 100644 index 00000000..9d99962e --- /dev/null +++ b/internal/giteaclient/pagination.go @@ -0,0 +1,50 @@ +// SPDX-License-Identifier: Apache-2.0 + +package giteaclient + +import ( + "context" + "fmt" + "net/http" +) + +// listPaginated walks every page of a Gitea list endpoint and returns the concatenated items. +// +// Two rules, both of which a hand-rolled loop tends to get wrong: +// +// - ONLY an empty page terminates. A short page is NOT necessarily the last page: Gitea's +// ToCorrectPageSize clamps the requested limit down to MAX_RESPONSE_ITEMS, which operators +// configure, so against a server whose cap is below pageSize EVERY page is short. Stopping +// on a short page would stop after page 1 and silently return a truncated list — the exact +// failure mode pagination is here to prevent. The cost is one extra round trip. +// - The loop is capped. A server that ignores the `page` parameter serves a full first page +// forever; without a cap this spins until the caller's context deadline and reports +// `context deadline exceeded`, naming neither the endpoint nor the cause. +// +// page is 1-indexed (Gitea routers/api/v1/utils/page.go, models/db/list.go). +func listPaginated[T any]( + ctx context.Context, + c *Client, + base string, + pageSize, maxPages int, +) ([]T, error) { + var all []T + for page := 1; page <= maxPages; page++ { + var items []T + path := fmt.Sprintf("%s?page=%d&limit=%d", base, page, pageSize) + code, raw, err := c.Do(ctx, http.MethodGet, path, nil, &items) + if err != nil { + return nil, err + } + if code != http.StatusOK { + return nil, unexpectedStatus(http.MethodGet, path, code, raw) + } + if len(items) == 0 { + return all, nil + } + all = append(all, items...) + } + return nil, fmt.Errorf( + "listing %s did not terminate within %d pages of %d: the server appears to ignore the "+ + "page parameter", base, maxPages, pageSize) +} From 9a9d76ae2c872041b95c0fb43a132343c3217dfe Mon Sep 17 00:00:00 2001 From: Simon Koudijs Date: Mon, 20 Jul 2026 14:21:50 +0000 Subject: [PATCH 14/17] fix(e2e): initialize the Prometheus client before the attribution report MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four e2e legs failed on run 29745528349 — quickstart-install, source-cluster, image-refresh and bi-directional — with every selected spec PASSING and the suite then panicking in SynchronizedAfterSuite: Test Panicked: invalid memory address or nil pointer dereference test/e2e.queryPrometheus helpers.go:70 test/e2e.reportAttributionStats e2e_suite_test.go:153 test/e2e.init.func18 e2e_suite_test.go:80 reportAttributionStats, added in c55c33b, queries Prometheus without calling ensurePrometheusClient() — unlike assertNoAnomalousAuditOutcomes on the very next line. promAPI is only ever set by that lazy initializer, so on any leg whose selected specs never touched a metric helper it was still nil. The four failing legs are exactly the small ones (1, 3, 6 and 11 specs); full-manager (43) and full-core (17) passed only because their specs had already initialized the client. A pure function of leg size and spec selection, not of what the legs test — which is why it read as unrelated flakiness. Worth noting the blast radius: the panic aborted the AfterSuite node, so the GATING assertNoAnomalousAuditOutcomes never ran on those four legs at all. queryPrometheus now reports an uninitialized client as a named error rather than nil-dereferencing. A panic there aborts whatever node is running, and from an AfterSuite that fails an entire leg with every spec green, which is unreadable from the console log. Also parses the author-attribution flag with strconv.ParseBool, as Go's flag package does. configuredAuthorModeFromArgs matched only the literal `--author-attribution=false`, so a `--set attribution.enabled=0` (or f, F, False, FALSE) would flip the probe to ATTRIBUTION mode while the controller ran configured-author — precisely the silent assertion swap a678dc7 exists to prevent. The `["--redis-addr="]` row is dropped: with attribution defaulting true that arg set is fatal at startup ("redis-addr is required when author-attribution is enabled"), so it documented a mode that cannot exist. Co-Authored-By: Claude Opus 4.8 (1M context) --- test/e2e/author_mode_test.go | 17 ++++++++++++++++- test/e2e/e2e_suite_test.go | 23 +++++++++++++++++++++-- test/e2e/helpers.go | 9 +++++++++ 3 files changed, 46 insertions(+), 3 deletions(-) diff --git a/test/e2e/author_mode_test.go b/test/e2e/author_mode_test.go index 23453dfa..2abab94d 100644 --- a/test/e2e/author_mode_test.go +++ b/test/e2e/author_mode_test.go @@ -8,6 +8,11 @@ import "testing" // ENABLED in cmd/main.go — so an absent flag means attribution mode, and only an explicit // opt-out selects configured-author mode. Getting this backwards silently flips the commit // author assertion instead of failing it. +// +// Note there is no `["--redis-addr="]` row: clearing Redis while attribution stays at its +// default of true is FATAL at startup ("redis-addr is required when author-attribution is +// enabled", cmd/main.go), so it is not a mode this probe can ever be asked to classify. Every +// configured-author row below therefore turns attribution off explicitly. func TestConfiguredAuthorModeFromArgs(t *testing.T) { // The real jsonpath output shape: a bracketed, quoted, comma-separated list. const live = `["--metrics-bind-address=:8443","--admission-webhook",` + @@ -21,9 +26,19 @@ func TestConfiguredAuthorModeFromArgs(t *testing.T) { {"live e2e deployment attributes authors", live, false}, {"no flags at all means attribution (both defaults enabled)", `[]`, false}, {"explicit attribution opt-out", `["--redis-addr=valkey:6379","--author-attribution=false"]`, true}, - {"redis explicitly cleared", `["--redis-addr="]`, true}, {"attribution off and redis cleared", `["--author-attribution=false","--redis-addr="]`, true}, {"bare --author-attribution is the true form, not an opt-out", `["--author-attribution"]`, false}, + + // Go's flag package parses booleans with strconv.ParseBool, so every one of these is a + // real opt-out a chart value could produce (e.g. `--set attribution.enabled=0`). Matching + // only the literal "=false" would classify them as attribution mode while the controller + // runs configured-author, flipping the commit-author assertion instead of failing it. + {"numeric false", `["--author-attribution=0","--redis-addr="]`, true}, + {"short false", `["--author-attribution=f","--redis-addr="]`, true}, + {"capitalised false", `["--author-attribution=False","--redis-addr="]`, true}, + {"upper false", `["--author-attribution=FALSE","--redis-addr="]`, true}, + {"numeric true is not an opt-out", `["--author-attribution=1"]`, false}, + {"capitalised true is not an opt-out", `["--author-attribution=True"]`, false}, } { t.Run(tc.name, func(t *testing.T) { if got := configuredAuthorModeFromArgs(tc.args); got != tc.want { diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index ef3c45a8..3e961701 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -150,6 +150,15 @@ var attributionWaitBuckets = []float64{0.5, 1, 2, 3, 5, 10} // Fact delivery is bounded below by the apiserver's --audit-webhook-batch-max-wait (1s in this // cluster), so a healthy run still shows most waits in the 0.5-2s range; the tail is what matters. func reportAttributionStats() { + // Mirrors assertNoAnomalousAuditOutcomes. Not optional: the AfterSuite runs on every leg, + // including ones whose selected specs never touched a metric helper, so promAPI may still be + // nil here. Without this the first queryPrometheus below nil-derefs and PANICS the + // SynchronizedAfterSuite — which fails the whole leg despite every spec passing, and takes + // the gating assertNoAnomalousAuditOutcomes call after it down with it. That is exactly what + // happened to the four small legs (1/3/6/11 specs) in run 29745528349, while the two large + // legs passed only because their specs had already initialized the client. + ensurePrometheusClient() + total, err := queryPrometheus(`sum(max_over_time(gitopsreverser_attribution_resolutions_total[2h])) or vector(0)`) if err != nil || total == 0 { _, _ = fmt.Fprintf(GinkgoWriter, @@ -257,8 +266,18 @@ func configuredAuthorModeFromArgs(args string) bool { redisAddr := "valkey:6379" for _, arg := range strings.Fields(strings.NewReplacer(`"`, " ", `[`, " ", `]`, " ", `,`, " ").Replace(args)) { switch { - case arg == "--author-attribution=false": - attribution = false + case arg == "--author-attribution": + // The bare form is the TRUE form for a boolean flag, not an opt-out. + attribution = true + case strings.HasPrefix(arg, "--author-attribution="): + // Parse with strconv.ParseBool, exactly as Go's flag package does. Matching only the + // literal "--author-attribution=false" would read `--set attribution.enabled=0` (or + // `f`, `F`, `False`, `FALSE`) as attribution ENABLED while the controller runs + // configured-author — silently swapping the commit-author assertion instead of + // failing it, which is the whole failure mode this probe exists to prevent. + if v, err := strconv.ParseBool(strings.TrimPrefix(arg, "--author-attribution=")); err == nil { + attribution = v + } case strings.HasPrefix(arg, "--redis-addr="): redisAddr = strings.TrimPrefix(arg, "--redis-addr=") } diff --git a/test/e2e/helpers.go b/test/e2e/helpers.go index 20f2df71..5f503237 100644 --- a/test/e2e/helpers.go +++ b/test/e2e/helpers.go @@ -64,6 +64,15 @@ func verifyPrometheusAvailable() { // queryPrometheus executes a PromQL query and returns the first scalar value // Returns 0 if no results found func queryPrometheus(query string) (float64, error) { + // A nil client is a wiring mistake in the caller (it skipped ensurePrometheusClient), not a + // query failure. Say so instead of nil-dereferencing: a panic here aborts whatever node is + // running — and from an AfterSuite that fails the entire leg with every spec passing, which + // is unreadable from the console log. + if promAPI == nil { + return 0, fmt.Errorf( + "prometheus client not initialized: call ensurePrometheusClient() before querying %q", query) + } + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) //nolint:mnd // reasonable query timeout defer cancel() From 87a8a8a671c9d768301803da705f4660a8c19b90 Mon Sep 17 00:00:00 2001 From: Simon Koudijs Date: Mon, 20 Jul 2026 14:22:02 +0000 Subject: [PATCH 15/17] chore(renovate): track the second devcontainer ENV block TILT_VERSION, DLV_VERSION, GOPLS_VERSION and STATICCHECK_VERSION (.devcontainer/Dockerfile) had no custom manager and are invisible to Dependabot too, so those four pins sat in exactly the "silently rotted" class the config's own description says it covers. The first ENV block is fully covered by 13 working regexes; this one was simply missed. Datasource per how each is actually installed, not by habit: - tilt downloads a GitHub release asset -> github-releases tilt-dev/tilt - the other three are `go install @$VERSION`, so the `go` datasource resolves the same module-proxy versions the build uses: github.com/go-delve/delve golang.org/x/tools/gopls (its own module, versioned apart from x/tools) honnef.co/go/tools (staticcheck's module path, not dominikh/go-tools releases, whose tags use a different scheme) NODE_MAJOR is left alone: it is a deliberate major-only pin tracking an LTS line, not a rotting version. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/renovate.json | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/.github/renovate.json b/.github/renovate.json index bd682b7d..07a70ed8 100644 --- a/.github/renovate.json +++ b/.github/renovate.json @@ -165,6 +165,38 @@ "matchStrings": ["COSIGN_VERSION=(?v[0-9.]+)"], "datasourceTemplate": "github-releases", "depNameTemplate": "sigstore/cosign" + }, + { + "customType": "regex", + "description": "devcontainer: tilt (second ENV block; release asset download)", + "managerFilePatterns": ["/^\\.devcontainer/Dockerfile$/"], + "matchStrings": ["TILT_VERSION=(?v[0-9.]+)"], + "datasourceTemplate": "github-releases", + "depNameTemplate": "tilt-dev/tilt" + }, + { + "customType": "regex", + "description": "devcontainer: delve (go install github.com/go-delve/delve/cmd/dlv)", + "managerFilePatterns": ["/^\\.devcontainer/Dockerfile$/"], + "matchStrings": ["DLV_VERSION=(?v[0-9.]+)"], + "datasourceTemplate": "go", + "depNameTemplate": "github.com/go-delve/delve" + }, + { + "customType": "regex", + "description": "devcontainer: gopls (go install golang.org/x/tools/gopls; its own module, versioned apart from x/tools)", + "managerFilePatterns": ["/^\\.devcontainer/Dockerfile$/"], + "matchStrings": ["GOPLS_VERSION=(?v[0-9.]+)"], + "datasourceTemplate": "go", + "depNameTemplate": "golang.org/x/tools/gopls" + }, + { + "customType": "regex", + "description": "devcontainer: staticcheck (go install honnef.co/go/tools/cmd/staticcheck)", + "managerFilePatterns": ["/^\\.devcontainer/Dockerfile$/"], + "matchStrings": ["STATICCHECK_VERSION=(?v[0-9.]+)"], + "datasourceTemplate": "go", + "depNameTemplate": "honnef.co/go/tools" } ] } From 074097144beb16699027b76219c70e4e714dd145 Mon Sep 17 00:00:00 2001 From: Simon Koudijs Date: Mon, 20 Jul 2026 15:23:44 +0000 Subject: [PATCH 16/17] docs: clarify attribution outcomes and prune stale notes --- .claude/hooks/block-unguarded-task-pipe.sh | 53 -- .claude/settings.json | 15 - .gitignore | 4 - README.md | 5 + api/v1alpha3/commitrequest_types.go | 6 +- .../validate-operator-types-webhook.yaml | 6 +- charts/gitops-reverser/values.yaml | 8 +- cmd/main.go | 12 +- config/deployment.yaml | 15 +- config/webhook/validating-webhook.yaml | 2 +- docs/INDEX.md | 12 +- docs/TODO.md | 4 +- docs/UPGRADING.md | 30 +- docs/architecture.md | 72 +- docs/attribution-setup-guide.md | 11 +- docs/configuration.md | 5 +- docs/debug/README.md | 19 - docs/debug/attribution-loss.md | 101 --- docs/debug/watch-construction.md | 205 ----- docs/design/attribution-outcome-and-author.md | 289 ------- docs/design/metrics-observability-plan.md | 4 +- docs/design/residual-e2e-flakes-2026-06-19.md | 233 ------ docs/finished/documentation-triage.md | 276 ------ docs/finished/redis-key-schema-v3.md | 3 +- .../watch-first-ingestion-architecture.md | 536 +----------- docs/future/design-commit-request-phase-2.md | 99 --- docs/future/idea-unify-pending-write-kinds.md | 13 +- docs/interpreting-metrics.md | 24 +- docs/spec/README.md | 2 +- .../commitrequest-admission-authorship.md | 636 +------------- docs/spec/commitrequest-design.md | 786 ++---------------- .../commitrequest-multi-finalize-design.md | 405 --------- .../deletecollection-attribution-expander.md | 28 +- docs/spec/status-conditions-guide.md | 31 +- docs/tasks/bounded-queue.md | 221 ----- hack/attribution-diagnostics.sh | 22 +- .../controller/commitrequest_controller.go | 53 +- .../commitrequest_controller_unit_test.go | 15 +- internal/controller/commitrequest_finalize.go | 8 +- .../controller/commitrequest_kstatus_test.go | 2 +- internal/controller/constants.go | 6 +- internal/git/attribution_outcome_test.go | 4 +- internal/git/commit_request_attach.go | 9 +- internal/git/commit_request_attach_loop.go | 7 +- internal/git/resync_flush.go | 3 +- internal/git/resync_push_test.go | 2 +- internal/git/types.go | 5 +- ...attribution_index_deletecollection_test.go | 3 +- internal/watch/target_watch.go | 2 +- internal/watch/target_watch_test.go | 2 +- .../validate_operator_types_handler.go | 16 +- .../e2e/commit_author_attribution_e2e_test.go | 2 +- test/e2e/commit_request_e2e_test.go | 12 +- test/e2e/e2e_suite_test.go | 35 +- 54 files changed, 392 insertions(+), 3987 deletions(-) delete mode 100755 .claude/hooks/block-unguarded-task-pipe.sh delete mode 100644 docs/debug/README.md delete mode 100644 docs/debug/attribution-loss.md delete mode 100644 docs/debug/watch-construction.md delete mode 100644 docs/design/attribution-outcome-and-author.md delete mode 100644 docs/design/residual-e2e-flakes-2026-06-19.md delete mode 100644 docs/finished/documentation-triage.md delete mode 100644 docs/future/design-commit-request-phase-2.md delete mode 100644 docs/spec/commitrequest-multi-finalize-design.md delete mode 100644 docs/tasks/bounded-queue.md diff --git a/.claude/hooks/block-unguarded-task-pipe.sh b/.claude/hooks/block-unguarded-task-pipe.sh deleted file mode 100755 index 3b1d8cdb..00000000 --- a/.claude/hooks/block-unguarded-task-pipe.sh +++ /dev/null @@ -1,53 +0,0 @@ -#!/usr/bin/env bash -# PreToolUse(Bash) hook: refuse `task … | something` when pipefail is not set. -# -# Why: a pipeline's exit status is the LAST command's status, so -# -# task prepare-e2e | tail -15 && task test-e2e -# -# reports `tail`'s success even when `prepare-e2e` failed — and the suite then -# runs against a cluster that was never created, failing much later in -# SynchronizedBeforeSuite with an unrelated-looking error. This has cost real -# debugging cycles more than once (see docs/design/support-boundary/ -# render-fidelity.md and kustomize-token-writeback-explained.md), which is why -# it is enforced mechanically here rather than written down a third time. -# -# The root Taskfile sets `pipefail` for commands *inside* tasks; that does -# nothing for a pipeline typed at the shell, which is what this guards. -# -# Reads the hook payload on stdin, emits a PreToolUse deny decision on stdout. -set -uo pipefail - -payload="$(cat)" -cmd="$(printf '%s' "${payload}" | jq -r '.tool_input.command // ""')" - -# Already guarded — the caller opted into pipefail semantics explicitly. -case "${cmd}" in -*pipefail*) exit 0 ;; -esac - -# Match a `task` invocation whose stdout is piped onward: -# (^|[;&(|]) start of command, or after a separator (covers && and ||) -# task[[:space:]] the `task` binary, not a word ending in "task" -# ([^|;&]|&[^&])* its arguments — a bare & is allowed so `2>&1` still counts, -# but && ends the command and stops the match -# \|[^|] a real pipe, not || -pipe_re='(^|[;&(|])[[:space:]]*task[[:space:]]([^|;&]|&[^&])*\|[^|]' - -if ! printf '%s' "${cmd}" | grep -Eq "${pipe_re}"; then - exit 0 -fi - -reason='Piping `task` into another command masks its exit status: the pipeline reports the LAST command'\''s status, so a failed task looks like success. This has caused a suite to run against a cluster that was never created. Either drop the pipe and let the output stream, or capture it and check the status yourself: - - task >/tmp/task.log 2>&1; rc=$?; tail -20 /tmp/task.log; exit $rc - -If you genuinely want the pipe, prefix the command with `set -o pipefail;`.' - -jq -nc --arg r "${reason}" '{ - hookSpecificOutput: { - hookEventName: "PreToolUse", - permissionDecision: "deny", - permissionDecisionReason: $r - } -}' diff --git a/.claude/settings.json b/.claude/settings.json index 608eddba..e619624c 100644 --- a/.claude/settings.json +++ b/.claude/settings.json @@ -10,20 +10,5 @@ "Bash(KUBEBUILDER_ASSETS=/workspaces/gitops-reverser/bin/k8s/1.35.0-linux-amd64 go test -count=1 -timeout 90s -ginkgo.focus=\"Should clear LastCommit to prevent information disclosure\" ./internal/controller/)", "Bash(gh pr *)" ] - }, - "hooks": { - "PreToolUse": [ - { - "matcher": "Bash", - "hooks": [ - { - "type": "command", - "command": "bash \"$CLAUDE_PROJECT_DIR/.claude/hooks/block-unguarded-task-pipe.sh\"", - "timeout": 10, - "statusMessage": "Checking for unguarded task pipelines" - } - ] - } - ] } } diff --git a/.gitignore b/.gitignore index 3e3d6cc3..37decd61 100644 --- a/.gitignore +++ b/.gitignore @@ -59,9 +59,5 @@ loadtest .env .claude/* -# Hook scripts are referenced by the tracked .claude/settings.json, so they must -# be tracked too — otherwise everyone else pulls a hook pointing at a file they -# do not have, and every Bash tool call fails with "No such file". -!.claude/hooks/ .agents/* /external-sources/ diff --git a/README.md b/README.md index b6f7c45b..4aae73eb 100644 --- a/README.md +++ b/README.md @@ -74,6 +74,11 @@ actor — user, service account, or CI identity — while the committer never mo kube-apiserver audit delivery (managed control planes like EKS/GKE/AKS generally do not expose it) plus Valkey/Redis: see the [attribution setup guide](docs/attribution-setup-guide.md). +If attribution is enabled but a live change has no usable audit fact, its author is explicitly +`unknown (attribution unresolved)`, not the configured committer. For a change that should have a named +actor, treat that identity as a signal to verify the audit policy, webhook route, source identity, and +Redis connectivity. + **Valkey/Redis is optional but advised.** Without it the default mode works fine; adding it unlocks warm-restart cursors, `CommitRequest` author capture, and attribution. diff --git a/api/v1alpha3/commitrequest_types.go b/api/v1alpha3/commitrequest_types.go index dc0e9c56..074b4337 100644 --- a/api/v1alpha3/commitrequest_types.go +++ b/api/v1alpha3/commitrequest_types.go @@ -55,9 +55,9 @@ type CommitRequestSpec struct { // while finalizing; Stalled=True when the finalize failed and needs attention. // - AuthorAttributed (domain): binary and settled immediately. True // (AttributedFromAdmission) when the submitter captured at admission named the -// commit author; False (CommitterFallback) when no admission record exists — the -// validate-operator-types webhook is not configured — and the commit is authored by the -// configured committer. False is not a failure and does not affect Ready. +// commit author; False (CommitterFallback) when no admission record exists and the +// request claims no actor. False is not a failure and does not affect Ready; the +// final Git author remains the matching watch window's author. // - Pushed (domain): True once the commit is in the remote repository. type CommitRequestStatus struct { // ObservedGeneration is the most recent generation observed by the controller. diff --git a/charts/gitops-reverser/templates/validate-operator-types-webhook.yaml b/charts/gitops-reverser/templates/validate-operator-types-webhook.yaml index 1ca2408a..0fd57a40 100644 --- a/charts/gitops-reverser/templates/validate-operator-types-webhook.yaml +++ b/charts/gitops-reverser/templates/validate-operator-types-webhook.yaml @@ -4,8 +4,8 @@ # at admission, into the command-author Redis corner the controller reads back with no # wait. Narrow by construction: one rule per command kind. failurePolicy is Ignore and # the handler always allows, so a user's CommitRequest never depends on this webhook — -# a miss degrades to a committer-authored commit (AuthorAttributed=False). -# See docs/spec/commitrequest-admission-authorship.md §4. +# a miss leaves the request without a claimed actor (AuthorAttributed=False). +# See docs/spec/commitrequest-admission-authorship.md. apiVersion: admissionregistration.k8s.io/v1 kind: ValidatingWebhookConfiguration metadata: @@ -26,7 +26,7 @@ webhooks: namespace: {{ .Release.Namespace }} path: /validate-operator-types port: {{ .Values.servers.admission.port }} - # Ignore, not Fail: a missed author capture must degrade to the committer, never + # Ignore, not Fail: a missed author capture must leave the request unnamed, never # reject a user's CommitRequest. failurePolicy: Ignore matchPolicy: Equivalent diff --git a/charts/gitops-reverser/values.yaml b/charts/gitops-reverser/values.yaml index 5c94ac64..e8bab7e6 100644 --- a/charts/gitops-reverser/values.yaml +++ b/charts/gitops-reverser/values.yaml @@ -102,16 +102,16 @@ servers: secretNameOverride: "" # Validating admission server hosting the validate-operator-types webhook, which captures - # the submitter of each CommitRequest at admission and names them as the commit author + # the submitter of each CommitRequest at admission and records it as the request's named actor # (read back by the controller with no wait). Independent of `attribution` below: # attribution governs *mirrored-resource* authorship (from audit), this governs - # *command* authorship (from admission). Off → CommitRequests commit as the configured - # committer and report AuthorAttributed=False. See + # *command* authorship (from admission). Off → CommitRequests claim no actor and report + # AuthorAttributed=False. See # docs/spec/commitrequest-admission-authorship.md. # # Command-author capture is Redis-backed: the captured author is written to Redis at admission and # read back at reconcile. Enabled by default so the webhook is always installed, but it is a no-op - # without Redis: the pod runs healthy and CommitRequests commit as the configured committer + # without Redis: the pod runs healthy and CommitRequests claim no actor # (AuthorAttributed=False) until queue.redis.addr is set. admission: enabled: true diff --git a/cmd/main.go b/cmd/main.go index cc2c4cc5..de2b5fb7 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -312,11 +312,11 @@ func main() { // lives in its own Redis corner (author:v1:command), independent of // --author-attribution (which governs mirrored-resource attribution). It is wired // whenever the admission server is on; the controller reads the captured submitter - // back with no wait (docs/spec/commitrequest-admission-authorship.md §2, §6). + // back with no wait (docs/spec/commitrequest-admission-authorship.md). // // AuthorLookup must be a nil interface — not a non-nil interface wrapping a nil // *CommandAuthorStore — when the webhook is off, so the controller's nil check - // selects finalize-as-committer immediately (AuthorAttributed=False) instead of + // selects the no-actor path immediately (AuthorAttributed=False) instead of // dereferencing a nil store. var commandAuthorStore *queue.CommandAuthorStore var commandAuthorLookup controller.CommandAuthorLookup @@ -325,10 +325,10 @@ func main() { commandAuthorStore = redisStore.CommandAuthorStore() commandAuthorLookup = commandAuthorStore setupLog.Info("validate-operator-types webhook enabled: command submitters are captured at admission " + - "and named as the commit author") + "and recorded as the request's named actor") } else { setupLog.Info("validate-operator-types webhook enabled without Redis: command author capture is a " + - "no-op; CommitRequests commit as the configured committer (AuthorAttributed=False). " + + "no-op; CommitRequests claim no actor (AuthorAttributed=False). " + "Set --redis-addr to capture command authors.") } } @@ -478,7 +478,7 @@ func parseFlagsWithArgs(fs *flag.FlagSet, args []string) (appConfig, error) { "and, when author attribution is enabled, the attribution facts. Leave empty to run without "+ "Redis: watches cold-replay on restart instead of resuming. Required by "+ "--author-attribution=true. --admission-webhook still runs without it, but command-author "+ - "capture is a no-op: CommitRequests commit as the configured committer.") + "capture is a no-op: CommitRequests claim no actor.") fs.BoolVar(&cfg.authorAttribution, "author-attribution", true, "Name the real actor (human or service account) who caused each change as the Git commit author, "+ "resolved from matching audit facts; this runs the audit webhook ingress (default true). When "+ @@ -661,7 +661,7 @@ func validateAdmissionWebhookConfig(cfg appConfig) error { return nil } // No redis-addr requirement here: the admission webhook stays enabled without Redis and - // simply no-ops command-author capture (commits fall back to the committer). Redis is + // simply no-ops command-author capture (CommitRequests claim no actor). Redis is // wired only when configured; see the commandAuthorStore setup. if _, _, err := splitBindAddress(cfg.admissionWebhookBindAddress); err != nil { return fmt.Errorf("invalid admission-webhook-bind-address %q: %w", cfg.admissionWebhookBindAddress, err) diff --git a/config/deployment.yaml b/config/deployment.yaml index c2ab3ca4..a03127ea 100644 --- a/config/deployment.yaml +++ b/config/deployment.yaml @@ -34,18 +34,15 @@ spec: - --redis-addr=valkey.valkey-e2e.svc.cluster.local:6379 # The e2e Valkey serves plain TCP, so opt out of the now-default Redis TLS. - --redis-insecure - # Attribution waits this long for a matching audit fact before committing as the - # configured committer, which silently loses the actor's name. Widened from the 3s - # default to buy headroom over the apiserver's --audit-webhook-batch-max-wait=1s + # Attribution waits this long for a matching audit fact before committing with the explicit + # `unknown (attribution unresolved)` author. Widened from the 3s default to buy headroom + # over the apiserver's --audit-webhook-batch-max-wait=1s # (test/e2e/cluster/start-cluster.sh): measured fact delivery is ~0.8-1.1s even on an # idle cluster, so 3s leaves only ~2s of slack for a 4-proc run. # - # Honest scope: this is NOT a fix for the ~7% of resolutions that find no fact. Raising - # 3s -> 10s left that rate unchanged, which means those facts never arrive rather than - # arriving late; only ~1 resolution per run lands in the 3-10s band. Kept because the - # headroom is cheap and makes the tail observable, not because it repairs anything. The - # after-suite report separates resolved-wait from absent-wait so the difference stays - # visible. + # This is not a substitute for correct audit attribution configuration. A resolution that + # still ends as `absent` exposes an audit policy, route, source-identity, Redis, or timing + # issue for the change being exercised; the after-suite report keeps that distinction visible. - --author-attribution-grace=10s # Dev/e2e only: the in-cluster Gitea SSH host key is not pinned, so permit SSH # when no known_hosts source exists. Never set this in production. diff --git a/config/webhook/validating-webhook.yaml b/config/webhook/validating-webhook.yaml index b4b96ea8..2d0b2c17 100644 --- a/config/webhook/validating-webhook.yaml +++ b/config/webhook/validating-webhook.yaml @@ -20,7 +20,7 @@ metadata: # kinds (a CommitRequest today) at admission. This one IS product behavior and is # packaged in the chart (charts/.../templates/validate-operator-types-webhook.yaml); the # entry here is the e2e SUT's copy. See -# docs/spec/commitrequest-admission-authorship.md §4. +# docs/spec/commitrequest-admission-authorship.md. # # The cert-manager.io/inject-ca-from annotation injects the admission server CA bundle # for every webhook in this configuration. diff --git a/docs/INDEX.md b/docs/INDEX.md index 90febd8d..ce92c857 100644 --- a/docs/INDEX.md +++ b/docs/INDEX.md @@ -1,10 +1,8 @@ # Index: what to read, and what is history -> Rebuilt 2026-07-11, when the tree went from 180 documents to 117. - -There are 136 tracked markdown files here. **About 35 of them bind.** This page names -those. If a document is not on this page, it is either a user guide (see -[`README.md`](README.md)) or history you can safely not read. +This page names the documents that bind the current implementation. If a document is not +listed here, it is either a user guide (see [`README.md`](README.md)) or historical context +you can safely skip. ## The four folders, and what each one means @@ -14,7 +12,6 @@ those. If a document is not on this page, it is either a user guide (see | [`design/`](design/) | **We are still deciding.** Open questions, proposals, unbuilt work. | yes — as intent, not as shipped behaviour | | [`facts/`](facts/) | Durable reference: how Kubernetes behaves, and what we discovered about it. | yes, as reference | | [`finished/`](finished/) | **This happened.** Shipped plans, closed investigations. Kept for context. | **no** | -| [`debug/`](debug/) | **This is wrong now.** Open investigations into live defects, with what has been ruled out. Retired to `finished/` once fixed. | **yes** — as known-broken behaviour | The rule that was missing before: `design/` used to hold shipped work and `finished/` used to hold live contracts. If you are adding a document, pick the @@ -54,7 +51,7 @@ misled. Full list in [`spec/README.md`](spec/README.md); the ones that carry a | [`gitpath-foreign-content-stringency.md`](spec/gitpath-foreign-content-stringency.md) | refusing a path that shadows foreign content | | [`unsupported-folder-refusal-plan.md`](spec/unsupported-folder-refusal-plan.md) | `GitPathAccepted`, and refusing what we cannot own | | [`commitrequest-design.md`](spec/commitrequest-design.md) | the CommitRequest window and its conditions | -| [`commitrequest-admission-authorship.md`](spec/commitrequest-admission-authorship.md) | how a real Kubernetes user becomes a commit author | +| [`commitrequest-admission-authorship.md`](spec/commitrequest-admission-authorship.md) | how command submitters are captured and matched to commit windows | | [`where-validation-lives.md`](spec/where-validation-lives.md) | schema → CEL → **the reconciler**; a webhook only for what exists solely at admission | | [`e2e-serial-registry.md`](spec/e2e-serial-registry.md) | which e2e specs must run Serial, and why | @@ -79,7 +76,6 @@ Eleven other open items: | [`release-image-reuse-plan.md`](design/release-image-reuse-plan.md) | PRs 2–5 unstarted | | [`e2e-coverage-gaps-and-improvements-plan.md`](design/e2e-coverage-gaps-and-improvements-plan.md) | tests A/B/C still proposals | | [`e2e-finish-plan.md`](design/e2e-finish-plan.md) | remaining e2e harness work | -| [`residual-e2e-flakes-2026-06-19.md`](design/residual-e2e-flakes-2026-06-19.md) | Flake B still open | | [`sensitive-resource-diagnostics-follow-up.md`](design/sensitive-resource-diagnostics-follow-up.md) | deferred diagnostics | | [`e2e-git-server-choice.md`](design/e2e-git-server-choice.md) | stay on Gitea or move to Forgejo — the `_csrf` pin is fixable in place on both, so the migration is now a preference call, not a fix; also why we adopt no SDK either way | | [`watchrule-source-namespace/`](design/watchrule-source-namespace/README.md) | letting a WatchRule address a differently-named namespace on its source cluster — a deny-by-default `allowedSourceNamespaces` on the **GitTarget** (so scope is per-tenant, not a provider-wide union), unlocked by a false-by-default delegation flag on the ClusterProvider. Split into five implementable PRs: three prerequisite scope fixes (the namespace-blind resync sweep that would delete other namespaces' manifests, the cluster-wide/named stream collapse, and ClusterWatchRule's unchecked GitTarget attachment), the field and its gate, and the ceiling that makes the allow-list bind ClusterWatchRule too — required because a multi-tenant deployment runs a ClusterWatchRule per tenant from day one to capture CRDs | diff --git a/docs/TODO.md b/docs/TODO.md index 43b6bdc0..5914d2ae 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -41,8 +41,8 @@ This file is meant to track the smaller current backlog, not historical notes. - [ ] Fix recurring full e2e flakiness around WatchRule/snapshot convergence. This has shown up more than once as timeout-based failures in manager SOPS bootstrap and - signing snapshot-message specs, then passed on rerun. Capture and mitigation notes live in - [docs/design/e2e-watchrule-cross-spec-interference.md](spec/e2e-serial-registry.md). + signing snapshot-message specs, then passed on rerun. Capture and mitigation notes live in the + [e2e serial registry](spec/e2e-serial-registry.md). This should be addressed before the next feature that expands commit-message, snapshot, or write-window behavior, otherwise new failures will be hard to separate from existing timing debt. diff --git a/docs/UPGRADING.md b/docs/UPGRADING.md index da2b4381..6ca538a2 100644 --- a/docs/UPGRADING.md +++ b/docs/UPGRADING.md @@ -7,6 +7,29 @@ guidance that the changelog's breaking-change entries link to. We are pre-1.0, so breaking changes bump the **minor** version (release-please is configured with `bump-minor-pre-major`) rather than the major. Read the relevant entry before upgrading across it. +## Unreleased — unresolved attribution is visible in Git (next minor; author identity changes) + +When `attribution.enabled=true` and the operator cannot match a live watch event to an audit fact within +`attribution.grace`, the Git author is now: + +```text +unknown (attribution unresolved) +``` + +Previously, this case used the configured committer identity, making an attribution miss +indistinguishable from configured-author mode. The Git **committer** remains the configured operator +identity in both cases. + +Configured-author mode is unchanged: when attribution is disabled, the configured committer is still both +the Git author and committer. A real Kubernetes actor is unchanged too. Only a live event for which +attribution ran but did not resolve now has the explicit author above. + +Update any automation that treats `GitOps Reverser` as meaning "attribution was not configured". Monitor +`gitopsreverser_commits_total{author_kind="unresolved"}` after the upgrade. For a live mutation that should +be attributable, this author normally means the audit policy, webhook route, source identity, Redis +connectivity, or grace window is not configured correctly. It can also represent a change with no usable +audit actor, so use the audit metrics before assigning a cause. + ## Unreleased — a `patches:` block no longer refuses the folder (next minor; more folders accepted) A kustomization declaring `patches:` used to refuse the whole `GitTarget`. Not the edit — the @@ -344,9 +367,10 @@ The Helm chart now defaults to the simple, Redis-free `configured-author` path, to `""` (was `valkey-auth`). Without a Redis endpoint the operator runs `configured-author` and watches cold-replay on restart. - `servers.admission.enabled` stays `true` by default, but the validate-operator-types admission - webhook no longer requires Redis. Without `queue.redis.addr` it runs as a no-op (CommitRequests - commit as the configured committer, `AuthorAttributed=False`); it captures authors once Redis is - configured. Previously enabling admission without Redis failed startup. + webhook no longer requires Redis. Without `queue.redis.addr` it runs as a no-op: CommitRequests + claim no actor (`AuthorAttributed=False`), while the Redis-free installation's matching windows are + configured-author. It captures submitters once Redis is configured. Previously enabling admission + without Redis failed startup. - The chart still rejects one invalid combination at render time: `attribution.enabled=true` without `queue.redis.addr` fails `helm install`/`upgrade` with an actionable message (attributed-author mode cannot run without Redis) instead of crash-looping the pod. diff --git a/docs/architecture.md b/docs/architecture.md index 30b7ba9e..85bb65e2 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -90,8 +90,10 @@ The solution, in the vocabulary used throughout this document: Older APIs that reject `sendInitialEvents` fall back to LIST plus buffered WATCH. The sweep fires on snapshot establishment, **never on a timer**. * **Audit, when enabled, only names the author.** It is an optional attribution lookup; a missing or - late fact costs author fidelity, never correctness, and with attribution disabled the product commits - as the configured committer. + late fact costs author fidelity, never correctness. With attribution disabled the product commits as the + configured committer. With attribution enabled, an unresolved live change is visibly authored as + `unknown (attribution unresolved)`, so an installation that expects attribution can investigate its audit + policy, ingress, Redis connection, or source identity instead of mistaking the result for configured-author mode. * **One `BranchWorker` per Git branch serializes all writes.** Every write to a branch funnels through a single worker and a single commit window, which keeps concurrent GitTargets and authors from racing each other into a corrupt tree. @@ -224,8 +226,9 @@ instead of waiting for the silence timer. The **entire spec is immutable**. Key * `status.conditions`: kstatus-compatible. **Ready** is the summary (True once the request reached a terminal outcome that is not an error — a pushed commit, or a benign no-commit); **Reconciling**/**Stalled** are the kstatus progress/blocked pair; **AuthorAttributed** reports - whether the `/validate-operator-types` validating admission webhook named the submitter or the request - fell back to the configured committer; **Pushed** reports whether the commit reached the remote. The `Ready` + whether the `/validate-operator-types` validating admission webhook named the submitter; an absent record + means the request claims no actor and can attach only to an unnamed window. **Pushed** reports whether the + commit reached the remote. The `Ready` condition's `reason` carries `Committed`, `NoWindowInGrace`, `WindowMismatch`, `AlreadyPresent`, or `FinalizeFailed`. A benign no-commit (e.g. `NoWindowInGrace`) is `Ready=True`, `Stalled=False` — a correct, non-error outcome — whereas a `FinalizeFailed` is `Ready=False`, `Stalled=True`. @@ -368,9 +371,10 @@ Following the ConfigMap edit: whose desired-state projection is unchanged) is dropped here. 3. **Resolve the author.** When attribution is enabled, the resolver waits a bounded grace window (`--author-attribution-grace`, default `3s`) for a matching audit fact in the attribution index, joining by - resourceVersion/UID. On a strong match the real user or named service account becomes the author; - otherwise (attribution off, no match, or expiry) the commit is authored by the configured committer. This - wait is per-event and never reorders a watch — see [Watch Event Ordering](#watch-event-ordering). + resourceVersion/UID. On a strong match the real user or named service account becomes the author. With + attribution disabled, the configured committer is the author; with attribution enabled but no usable fact, + the explicit `unknown (attribution unresolved)` author marks the unresolved change. This wait is per-event + and never reorders a watch — see [Watch Event Ordering](#watch-event-ordering). 4. **Route + window.** The event flows into the [GitTargetEventStream](../internal/reconcile/git_target_event_stream.go), and the [BranchWorker](../internal/git/branch_worker.go) appends it to the open commit window for @@ -403,15 +407,17 @@ flowchart TD FILTER --> AUTHOR{Attribution enabled?} AUTHOR -->|yes| AUDIT[Wait bounded grace for audit fact] AUTHOR -->|no| COMMITTER[Use configured committer as author] - AUDIT --> RESOLVED[Resolved author or committer fallback] - COMMITTER --> RESOLVED - RESOLVED --> WINDOW[BranchWorker open window
one author + one GitTarget] + AUDIT -->|fact matched| RESOLVED[Use resolved actor] + AUDIT -->|no usable fact| UNRESOLVED[Use explicit unresolved author] + COMMITTER --> WINDOW[BranchWorker open window
one author + one GitTarget] + RESOLVED --> WINDOW + UNRESOLVED --> WINDOW WINDOW --> TIMER{Close trigger} TIMER -->|silence timer| FLUSH[Plan YAML edits + commit] TIMER -->|buffer limit or resync boundary| FLUSH - CR[CommitRequest persisted] --> ADMISSION[Admission author cache lookup
present or committer fallback] + CR[CommitRequest persisted] --> ADMISSION[Admission submitter lookup
named or unnamed] ADMISSION --> ATTACH[Attach to matching open window] ATTACH -->|same author + GitTarget| FLUSH ATTACH -->|no matching window| BENIGN[Ready=True, Pushed=False] @@ -568,8 +574,10 @@ Attribution being optional does **not** make it casual or take-it-or-leave-it. W operator does everything it can to name the real actor — but it values **high certainty over a plausible guess**. Attaching a name to a change is a consequential, sometimes politically charged claim ("*this* person did *that*"), so it must not be made lightly. The design therefore fails toward honesty: a weak, -conflicting, late, or absent fact resolves to the committer rather than to a guessed author. **It is better -to clearly record that the operator made a change than to assert an actor we are not sure of.** +conflicting, late, or absent fact produces an explicit unresolved author rather than a guessed person or a +misleading committer identity. **It is better to say that attribution did not resolve than to assert an +actor we are not sure of.** In an installation that expects live mutations to be attributable, this outcome +is a concrete signal to check audit delivery and attribution configuration. Two engineering choices follow directly from that stance: @@ -580,7 +588,7 @@ Two engineering choices follow directly from that stance: * **Tests pin the behavior.** Because a misattribution is a real harm, the attribution and resolver paths carry unit and e2e tests that prove the concrete cases — strong match, weak/last-key match, deletes whose audit RV differs from the watch RV, missing/late/expired facts, service-account vs human actor, - impersonation, and the committer fallback — so these certainty guarantees cannot silently regress. + impersonation, and the explicit unresolved outcome — so these certainty guarantees cannot silently regress. ### Attribution fact shape @@ -608,8 +616,9 @@ A watch event waits a **bounded grace window** (`--author-attribution-grace`, de to arrive, then ships regardless. On a strong match the actor becomes the author — a human or a service account alike, always named by its own username (e.g. `system:serviceaccount:flux-system:kustomize-controller`). A weak, conflicting, missing, or expired fact -resolves to the committer. A late fact that arrives after a commit has shipped **never rewrites it**. -With attribution disabled the resolver is absent and every commit is committer-authored. +produces `unknown (attribution unresolved) ` instead of a +guessed actor. A late fact that arrives after a commit has shipped **never rewrites it**. With attribution +disabled the resolver is absent and every commit is committer-authored. The CommitRequest controller does **not** read this audit index. A CommitRequest's submitter is named by the `/validate-operator-types` validating admission webhook instead — captured synchronously at admission, @@ -633,8 +642,7 @@ and the *author* (who wrote the change) — and GitOps Reverser uses both on pur * **Attribution ran and did not resolve an actor**: the author is the explicit sentinel `unknown (attribution unresolved) `. It is deliberately NOT the operator, because authoring it as the operator made a lost actor byte-identical to a - configured-author commit — so the gap was invisible in Git history and only countable in a metric. See - [docs/design/attribution-outcome-and-author.md](design/attribution-outcome-and-author.md). + configured-author commit — so the gap was invisible in Git history and only countable in a metric. A commit whose author is a **person** is therefore a positive statement that the operator is confident who made the change; one authored by the sentinel is a positive statement that it tried and could not tell. @@ -810,8 +818,8 @@ namespaces. Desired state comes from one **raw watch per `(GitTarget, GVR, scope)`**. Each event is sanitized, diffed against current Git content, and applied — there is no separate per-type object store to reconstruct, -because Git already holds current state. The authoritative design is -[design/watch-first-ingestion-architecture.md](finished/watch-first-ingestion-architecture.md). +because Git already holds current state. This section is the authoritative contract; the +[watch-first design record](finished/watch-first-ingestion-architecture.md) is historical context. * **Manager**: [internal/watch/manager.go](../internal/watch/manager.go) * **Watch / replay / sweep**: [internal/watch/target_watch.go](../internal/watch/target_watch.go) @@ -1019,14 +1027,14 @@ counted in the resync summary (`placementSkipped`) — rather than written unsaf A `CommitRequest` finalizes the open commit window for its GitTarget. The request author is resolved from the `/validate-operator-types` validating admission webhook, which captures the authenticated submitter at admission (keyed by the object's UID) before the object is visible. The lookup is **present-or-never**: if -the record is missing, the request **finalizes as the configured committer** with `AuthorAttributed=False`, -and that fallback does not fail the request: +the record is missing, the request is marked `AuthorAttributed=False`, claims no actor, and still attaches +immediately. That is not a failure: 1. The controller stamps the in-progress conditions (`Reconciling=True`) and settles `AuthorAttributed` synchronously from the admission author cache. There is no audit wait on this path. 2. The controller eagerly **attaches** the request to the worker (`AttachCommitRequest`), anchoring the - finalize at `receipt + closeDelaySeconds`. The worker binds it to an open window only when author and - GitTarget match. It **never finalizes another author's window**; a window carries at most one request. + finalize at `receipt + closeDelaySeconds`. The worker binds it to an open window only when the author + state and GitTarget match. It **never finalizes another author's window**; a window carries at most one request. 3. The window finalizes on the deadline (or when it closes for any other reason). If a finalize closes an open window, the worker always schedules a push, so a window closed by an otherwise no-op resync is not stranded. @@ -1035,8 +1043,10 @@ and that fallback does not fail the request: `Pushed=False`; a failure sets `Ready=False` / `Stalled=True` with a message. The CommitRequest submitter is not recoverable from object state alone, so without an admission record the -finalize is committer-authored. There is no audit-fact join on this path: the submitter is captured at -admission by the validating webhook, not derived from the audit attribution index. +request cannot claim an actor. The final Git author remains the attached watch window's author: configured +committer when attribution was disabled, or the explicit unresolved author when live attribution ran but +could not resolve. There is no audit-fact join for the request itself: its submitter is captured at admission +by the validating webhook, not derived from the audit attribution index. *** @@ -1057,7 +1067,7 @@ Controllers watch their dependencies so dependents reconcile quickly after spec * `CommitRequestReconciler` runs with `MaxConcurrentReconciles=1` and attributes/attaches as above; its optional `AuthorLookup` is the command-author cache populated by the `/validate-operator-types` webhook (wired whenever the admission webhook is enabled — independent of `--author-attribution` — and nil - otherwise, so the request finalizes as the committer). + otherwise, so the request claims no actor). Dependency watches use narrow predicates to avoid status-only heartbeat churn: a `ClusterProvider` also admits a `Ready` transition, and a `Namespace` admits only label changes. `GitProvider`, `GitTarget`, and @@ -1119,8 +1129,8 @@ Metrics are exported over OTLP / the metrics server. The audit-attribution path (read by the restart-reconcile guarantee); * resync/background-apply failure counters so a silently-recovered fault stays visible. -Per-watch volume/restart/replay metrics and per-attribution result/wait histograms are designed -([watch-first metrics](finished/watch-first-ingestion-architecture.md#metrics)) but **not yet emitted**; see +Per-watch volume/restart/replay metrics and per-attribution result/wait histograms are designed in the +[metrics observability plan](design/metrics-observability-plan.md) but **not yet emitted**; see [Operational Boundaries](#operational-boundaries). *** @@ -1184,8 +1194,8 @@ Deeper dives live under [docs/design/](design/): **Ingestion, attribution, and reconcile:** -* [Watch-first ingestion architecture](finished/watch-first-ingestion-architecture.md) — the authoritative - model: watch is the only object-state source, audit is optional attribution. +* [Watch-first ingestion design record](finished/watch-first-ingestion-architecture.md) — historical context + for the current watch-only object-state model and optional audit attribution. * [Watch event ordering under the attribution grace window](facts/watch-event-ordering-and-attribution-grace.md) * [Reconcile via watchlist mark and sweep](spec/reconcile-via-watchlist-mark-and-sweep.md) * [CommitRequest design](spec/commitrequest-design.md) diff --git a/docs/attribution-setup-guide.md b/docs/attribution-setup-guide.md index 39e28493..404bddc3 100644 --- a/docs/attribution-setup-guide.md +++ b/docs/attribution-setup-guide.md @@ -33,6 +33,7 @@ With `attribution.enabled`, only the **author** column moves: | Human user (`simon@example.com`) | Simon | `ConfigButler Bot` | | Service account (`system:serviceaccount:team-a:deployer`) | the `team-a/deployer` service account | `ConfigButler Bot` | | CI / GitHub App identity | that CI / App identity | `ConfigButler Bot` | +| No usable audit fact for a live change | `unknown (attribution unresolved)` | `ConfigButler Bot` | The committer column never moves. That is the point. @@ -155,14 +156,16 @@ Audit facts and watch events arrive on independent paths, so the controller join wait rather than blocking: - **`attribution.grace`** (default `3s`) — how long a watch event waits for a matching audit fact - before it ships authored by the committer. Larger raises the attribution hit-rate at the cost of - commit latency. + before it ships with the explicit unresolved author. Larger raises the attribution hit-rate at the cost + of commit latency. - **`attribution.ttl`** (default `10m`) — how long an unmatched audit fact is retained waiting for its watch event. Attribution is opportunistic: on a strong match the named user or service account is the author; with -no match in the grace window, the commit still lands, authored by the committer. No change is dropped -for lack of a match. +no match in the grace window, the commit still lands, authored as +`unknown (attribution unresolved)`. No change is dropped for lack of a match. For a live mutation that +should be attributable, treat this author as an audit-attribution configuration or delivery problem until +the audit policy, webhook route, source identity, and Redis connectivity are verified. ## Verifying and reverting diff --git a/docs/configuration.md b/docs/configuration.md index 86999c71..314f36a1 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -807,8 +807,9 @@ Progress and outcome are reported through kstatus-compatible **conditions** (no and needs attention (kstatus reports the object Failed). - **AuthorAttributed**: `True` with reason `AttributedFromAdmission` when the internal commands admission webhook captured the request submitter. `False` with reason `CommitterFallback` when no - admission record exists; that is not a failure, and the command still commits as the configured - committer. + admission record exists; that is not a failure. The request then claims no actor and can attach only to + an unnamed watch window, whose Git author remains either the configured committer or the explicit + unresolved author according to the watch attribution outcome. - **Pushed**: `True` once the commit is in the remote repository. ## Audit ingestion settings diff --git a/docs/debug/README.md b/docs/debug/README.md deleted file mode 100644 index f0d949c4..00000000 --- a/docs/debug/README.md +++ /dev/null @@ -1,19 +0,0 @@ -# `debug/` — live investigations - -Working notes for defects that are **understood but not fixed**, or **not yet understood**. -Unlike [`finished/`](../finished/), these are open; unlike [`design/`](../design/), they -describe something that is wrong now rather than something being decided. - -Retire a page from here by fixing the defect and moving the write-up to `finished/`. - -| Page | What it is | Status | -|---|---|---| -| [attribution-loss.md](attribution-loss.md) | ~7–10% of live commits are authored by the configured committer instead of the real actor, silently | **open** | -| [watch-construction.md](watch-construction.md) | Reference: how watches are built, and at what cardinality | reference | - -## One-line state - -`WatchRule`/`ClusterWatchRule` stream scoping is correct as of -[PR 2](../design/watchrule-source-namespace/pr2-stream-scope-collapse.md). The open defect is -in **author attribution**, and it is older than that work — it was found while landing PR 2, -not caused by it. diff --git a/docs/debug/attribution-loss.md b/docs/debug/attribution-loss.md deleted file mode 100644 index 3aaeb90b..00000000 --- a/docs/debug/attribution-loss.md +++ /dev/null @@ -1,101 +0,0 @@ -# Author attribution silently loses the actor - -> **Status: OPEN.** Found 2026-07-20 while landing -> [PR 2](../design/watchrule-source-namespace/pr2-stream-scope-collapse.md); **not caused by -> it**. Predates that work. - -## The symptom - -A commit that should be authored by the human or service account that made the change is -authored by the configured committer instead. **Silently**, and **invisibly by inspection**: -`git.DefaultCommitterName` is `"GitOps Reverser"` ([internal/git/types.go:22](../../internal/git/types.go#L22)), -which is *also* the configured-author identity — so a lost-actor commit is byte-identical to a -correct configured-author commit. The only way to see it is to count it. - -Naming the actor is the product's core promise, so this is a correctness defect, not cosmetics. - -## How it happens - -`ResolveAuthor` ([internal/watch/author_resolver.go](../../internal/watch/author_resolver.go)) -polls the Redis attribution index for a fact matching the live event's `(uid, rv)`, waiting up -to `--author-attribution-grace` (default 3s). No match → commit as the committer. - -## The number, and how far it can be trusted - -`absent / total` from `gitopsreverser_attribution_resolutions_total`: **7.1%, 8.0%, 9.6%** -across three runs. Read it as *roughly 1 in 12–14*, not as a precise figure. - -**What supports it.** Attribution runs **only on live events** — `attachAuthor` has one call -site, in `routeLiveTargetWatchEvent` -([target_watch.go:728](../../internal/watch/target_watch.go#L728)). `foldTargetReplayEvent` -never calls the resolver, so initial-replay of pre-existing objects contributes nothing to -either side of the ratio. - -**What weakens it.** - -- The queries use `sum(max_over_time(…[2h]))`, reconstructing totals across the deliberate - controller restart in the `restart-reconcile` spec rather than counting exactly. The same - population read instantaneously gave 92 where `max_over_time` gave 791. -- `absent` means *no fact matched*, which is not identical to *a human's name was lost*. A - live change with no audited actor belongs there legitimately. - -**To make it precise**, the resolver's metric needs a label separating *no fact was ever -produced* (correct) from *a fact should exist and we failed to match it* (the bug). Today both -collapse into `absent`, which is exactly why the number cannot yet carry much weight. - -## Ruled out - -| Hypothesis | Why not | -|---|---| -| **PR 2 (stream-scope collapse fix)** | Loss rate 8.0% with it vs 9.6% without; a PR 2 run passed the spec outright; all six CI e2e legs green. Its only runtime effect is `targetWatchSpecs`, whose output is byte-identical unless one `WatchedType` holds both `""` and a named key. | -| **Late fact arrival / grace too short** | Raising grace 3s → 10s left the rate **unchanged** (7.1%). Only ~1 resolution per run falls in the 3–10s band. Facts missing at 3s are still missing at 10s — they never arrive. | -| **Watermark / high-water gating** | `older_than_high_water = 0` in every run. | -| **Extra load from PR 2's duplicate streams** | Audit volume does not correlate: the *failing* run had the lowest (5667), a *passing* run the highest (9040). | -| **Wrong fact key shape** | Every ConfigMap uid in Valkey has **both** an exact-`rv` key and a `:last` key (46/46). Facts are indexed correctly. | -| **Audit webhook misrouted** | Posts to `/audit-webhook/default`, the correct **named** route ([webhook-config.yaml](../../test/e2e/cluster/audit/webhook-config.yaml)). ~92% of resolutions succeed, so delivery fundamentally works. | -| **Audit policy excluding the events** | Policy captures create/update/patch/delete/deletecollection broadly ([policy.yaml](../../test/e2e/cluster/audit/policy.yaml)). | -| **Initial-replay events with no actor** | Replay never invokes the resolver (see above). | -| **The e2e suite being "just flaky"** | Argued three times without evidence and wrong each time — though the spec *is* ~10%-flaky, because it samples this defect once per run. | -| **Controller restart mid-run disrupting the spec** | `restart_reconcile_e2e_test.go` is `Serial`; it cannot overlap parallel specs. | -| **`playground` Gitea key collision** | Self-inflicted contamination from a prior partial run, not a defect. | - -## Fixed along the way - -| Fix | Why it mattered | -|---|---| -| **`ci.yml` validates every PR** | `branches: [main]` meant stacked PRs ran **no** CI — #255 merged with no lint/test/e2e. Retargeting doesn't re-trigger (`edited` isn't a default trigger type). | -| **Author-mode probe reads the Deployment** | Was `kubectl logs --since=30m` grepping for a banner, failing **open**. Could silently swap in a weaker assertion that passes. | -| **`test/e2e/Taskfile.yml` → `Taskfile-e2e.yml`** | Running `task` from `test/e2e/` stopped Task's upward search there, so root-relative paths resolved nowhere. Caused `clean-cluster` to delete the cluster but not its stamps → next `prepare-e2e` skipped cluster creation. | -| **Hook blocking unguarded `task … \| …`** | A pipeline reports the *last* command's status, so a failed `prepare-e2e` looked successful and the suite ran against a nonexistent cluster. | -| **Per-stream declare logging** | `targetWatchSpecs` now names every stream (`@=`) instead of a bare count. Found the real two-stream fan-out on `unsupported-folder-dest`. | -| **e2e after-suite attribution report** | Prints outcome + wait distribution every run, split resolved vs absent. | -| **Histogram `le` label bug** | Buckets render as `1.0`/`3.0`/`10.0`, not `1`/`3`/`10`; `%g` matched nothing above `0.5` and produced a *plausible but wrong* distribution. | - -## Tools - -- [`hack/attribution-diagnostics.sh`](../../hack/attribution-diagnostics.sh) — separates *fact - never delivered* from *fact arrived too late*. Fact keys carry a 10-minute TTL, so the - remaining TTL back-computes when each landed. Run **immediately** after an e2e run. -- After-suite report — outcome counts plus cumulative wait buckets, split by result. -- Valkey: `valkey-cli -a "$(kubectl -n valkey-e2e get secret valkey-auth -o jsonpath='{.data.*}' | base64 -d)" --scan --pattern '*author:v1:audit:*'` - -## Measured facts worth keeping - -- Fact delivery lag, **idle** cluster: `835–1101 ms`, bimodal — the signature of the - apiserver's `--audit-webhook-batch-max-wait=1s`. -- Resolver mean wait by result: `weak` 0.001s · `exact_user` 0.533s · `absent` 2.268s (at the - 3s grace). -- Wait distribution (92-sample snapshot): `≤0.5s` 66 · `≤1s` 84 · `≤2s` 88 · `≤3s` 89 · - `≤10s` 90 · `+Inf` 92. - -## Next steps - -1. **Label the metric** to split *no fact existed* from *fact existed, match failed*. Until - then the rate cannot be acted on. -2. If the second bucket is non-empty, compare the fact's recorded `rv` against the live event's - `rv` for a failing object — the exact-key join is the likeliest place to lose a match. -3. **Decide the shipped default.** e2e now runs `--author-attribution-grace=10s`, but the - default is still 3s and the evidence says the grace is *not* the constraint. Raising it - ships latency for no proven benefit. -4. **Make the fallback loud.** Losing the actor should be observable at the commit, not only in - a counter nobody reads. diff --git a/docs/debug/watch-construction.md b/docs/debug/watch-construction.md deleted file mode 100644 index 37e6a735..00000000 --- a/docs/debug/watch-construction.md +++ /dev/null @@ -1,205 +0,0 @@ -# How watches are built — an overview - -> Written 2026-07-20 while investigating [attribution-loss.md](attribution-loss.md). Describes the tree as it is **today**, -> verified against the code, not as designed. Every claim here was checked by reading the -> named function. -> -> Short answers to the three questions this page exists for: -> **(1)** Yes — watches follow from the GitTarget reconcile, but the *content* is projected -> from the rule set, not from the GitTarget. **(2)** No reuse across GitTargets: every -> GitTarget opens its own watches, even for an identical (GVR, namespace). **(3)** One watch -> per `(GitTarget, GVR, namespace-scope)` triple — so yes, combinations. - -## 1. The build pipeline - -Rules are compiled into a store; the store is projected into a per-GitTarget table; the table -is projected into a set of watch keys; each key becomes one goroutine holding one API watch. - -```mermaid -flowchart TD - subgraph CRs["Custom resources"] - WR["WatchRule
targetRef is namespace-local"] - CWR["ClusterWatchRule
targetRef names any namespace"] - GT["GitTarget"] - end - - subgraph Compile["Rule compilation"] - WRC["WatchRuleReconciler"] - CWRC["ClusterWatchRuleReconciler"] - BS["bootstrap.go
seeds the store on restart"] - RS[("RuleStore")] - end - - subgraph Project["Projection — watched_type_resolver.go"] - REG[("typeset.Registry
followable set, per source cluster")] - RWT["resolveWatchedTypeTables()"] - BWT["buildWatchedTypeTable()
fold selections by GVR"] - TBL[("WatchedTypeTable
per GitTarget")] - end - - subgraph Declare["Declaration — target_watch.go"] - DECL["DeclareForGitTarget()
from the GitTarget reconcile"] - EGW["EnsureGitTargetWatches()"] - RGW["replaceGitTargetWatches()"] - TWS["targetWatchSpecs()
the only consumer of WatchScopes"] - RUN["runTargetWatch() — one goroutine per key"] - API["dynamic client Watch()
one raw watch each"] - end - - WR --> WRC --> RS - CWR --> CWRC --> RS - BS --> RS - GT --> DECL - - RS --> RWT - REG --> RWT - RWT --> BWT --> TBL - TBL --> RGW - - DECL --> EGW --> RGW --> TWS --> RUN --> API -``` - -**What triggers a rebuild.** `refreshWatchedTypeTables()` is gated on three fingerprints, so -the common no-change reconcile is a cheap compare rather than a rescan: the rule set -(`rulesFingerprint`), every active cluster's discovery revision -(`activeRegistriesFingerprint`), and the GitTarget→source-cluster mapping -(`clusterMappingFingerprint`). - -**Your intuition is right, with one correction.** The watch set *is* driven by the GitTarget -reconcile — `DeclareForGitTarget` is what creates the `targetWatches` entry. But the GitTarget -contributes only its *identity and destination*; **what** to watch comes entirely from the -rules that point at it. A GitTarget with no rules gets an empty table and no streams. That -asymmetry matters: the rule controllers can change a resident table, but -`refreshRunningTargetWatches` only refreshes GitTargets that are **already running**, so a -rule alone cannot bootstrap a stream. (This is the incidental protection that -[PR 3](../design/watchrule-source-namespace/pr3-clusterwatchrule-target-admission.md) exists to make deliberate.) - -## 2. The namespace scope of a selection - -The one rule that determines everything downstream: - -| Rule kind | Namespace recorded in the selection | -|---|---| -| `WatchRule` | the rule's own source namespace (`rule.Source.Namespace`) | -| `ClusterWatchRule` | **always `""`** — regardless of the rule's `scope:` | - -`""` means *cluster-wide* — a watch with no namespace, i.e. all namespaces. A cluster-scoped -type (CRDs, Namespaces) can only ever be `""`, because `collectWatchRuleSelections` hardcodes -`ResourceScopeNamespaced` and so a WatchRule can never match a cluster-scoped record. - -`buildWatchedTypeTable` then folds selections into one `WatchedType` per GVR, carrying -`NamespaceOps: map[namespace]OperationSet` — the union of operation filters per namespace. - -## 3. Cardinality — what actually gets opened - -**One watch per `(GitTarget, GVR, namespace-scope)`.** Nothing is shared. - -```mermaid -flowchart LR - subgraph T1["GitTarget A"] - A1["configmaps @ team-a"] - A2["secrets @ team-a"] - end - subgraph T2["GitTarget B"] - B1["configmaps @ team-a
same GVR+ns as A1 —
still a separate watch"] - B2["crds @ *cluster-wide*"] - end - subgraph T3["GitTarget C — case C"] - C1["configmaps @ *cluster-wide*"] - C2["configmaps @ team-a
same object delivered twice"] - end - - A1 --> W1["watch #1"] - A2 --> W2["watch #2"] - B1 --> W3["watch #3"] - B2 --> W4["watch #4"] - C1 --> W5["watch #5"] - C2 --> W6["watch #6"] -``` - -So the total is a genuine product: - -```text -streams = Σ over GitTargets Σ over watched GVRs |namespace scopes for that GVR| -``` - -- **Across GitTargets: no reuse.** `m.targetWatches` is keyed by `gitDest.Key()`, and - `openTargetWatch` calls the dynamic client's `Watch()` directly — a raw watch, not a shared - informer cache. Two GitTargets mirroring the same ConfigMaps in the same namespace open two - independent watches against the API server. -- **Across reconciles: yes, reuse.** `replaceGitTargetWatches` compares the newly computed - spec map against the running one (`equalTargetWatchSpecs`) and returns early when unchanged, - so a no-op reconcile does not churn streams. The reuse that exists is *temporal*, not - *cross-target*. -- **Per namespace: one each.** A GVR selected in `team-a` and `team-b` is two watches, because - a namespaced watch takes exactly one namespace. - -## 4. What PR 2 changed - -Only the third bullet. Before PR 2, the presence of the `""` key made `SnapshotNamespaces()` -return `nil`, and `nil` meant *all namespaces* — so a co-resident named namespace was -**silently discarded**, along with its operation filter. - -```mermaid -flowchart TB - subgraph Input["One WatchedType, one GitTarget, one GVR"] - N["NamespaceOps
"" → UPDATE (ClusterWatchRule)
team-a → CREATE (WatchRule)"] - end - - N --> OLD["Before PR 2
SnapshotNamespaces() → nil
⇒ ONE stream
configmaps@*cluster-wide*=UPDATE

team-a's CREATE filter discarded;
rule silently widened to all namespaces"] - N --> NEW["After PR 2
WatchScopes() → ["", team-a]
⇒ TWO streams
configmaps@*cluster-wide*=UPDATE
configmaps@team-a=CREATE

each scope keeps its own filter"] - - NEW --> DUP["⚠ objects in team-a are now
delivered on BOTH streams"] -``` - -For every other shape the output is byte-identical: - -| `NamespaceOps` | Before | After | Same? | -|---|---|---|---| -| `{team-a}` | `configmaps@team-a` | `configmaps@team-a` | ✅ | -| `{""}` | `configmaps@*cluster-wide*` | `configmaps@*cluster-wide*` | ✅ | -| `{team-a, team-b}` | both | both | ✅ | -| `{"", team-a}` | cluster-wide **only** | **both** | ❌ the fix | -| `{}` (empty) | cluster-wide | **none** | ❌ unreachable in practice | - -## 5. Where the attribution suspicion sits - -A commit's author is not carried by the watch event. The event says *what* changed; a separate -audit-webhook fact says *who* did it, and the two are matched. - -```mermaid -sequenceDiagram - actor U as kubectl --as=jane@acme.com - participant API as kube-apiserver - participant AUD as audit webhook → Redis - participant W as target watch stream - participant M as match / write path - participant G as Git - - U->>API: create ConfigMap - API-->>AUD: audit event (user = jane@acme.com) - API-->>W: watch event (object + rv) - W->>M: live event - AUD->>M: audit fact - M->>M: match event ↔ fact - alt fact matched in window - M->>G: commit authored jane@acme.com - else no match - M->>G: commit authored "GitOps Reverser"
(DefaultCommitterName fallback) - end -``` - -The observed failure is the lower branch: the file and commit landed, only the author was -wrong. Note that `"GitOps Reverser"` is **also** the configured-author identity, which is why -the failure was initially misread — see -[attribution-loss.md](attribution-loss.md). - -**Why the diagram above makes case C interesting.** If one object is delivered on two streams, -the write path sees it twice, but there is only ever **one** audit fact. Whichever delivery -consumes or races the match, the other has none — and a second, unattributed write of the same -object is a plausible way to land a commit authored by the fallback. This is the hypothesis -the instrumented run is testing: the declare log now names every stream, so a GVR appearing -twice for one GitTarget is case C caught in the act. - -It is still a hypothesis. The controlled experiment says PR 2 causes the failure; it does not -yet say by which mechanism. diff --git a/docs/design/attribution-outcome-and-author.md b/docs/design/attribution-outcome-and-author.md deleted file mode 100644 index c04ab512..00000000 --- a/docs/design/attribution-outcome-and-author.md +++ /dev/null @@ -1,289 +0,0 @@ -# Make attribution outcome explicit, and name the gap in Git history - -> Design, 2026-07-20, **revised after review**. Not built. No CRD/API change. -> Motivated by [debug/attribution-loss.md](../debug/attribution-loss.md). -> -> **The revision is structural, not cosmetic.** The first draft stamped a sentinel author -> string and claimed nothing else was affected. Review showed that string is load-bearing in -> three places that would silently break. The fix is to stop encoding attribution outcome in -> the author string at all: carry it as an **explicit value**, and let the author string be a -> *rendering* of it. - -## The problem - -When attribution is enabled but no fact matches, the commit is authored by the configured -committer — the same `"GitOps Reverser"` identity used in configured-author mode -([commit.go:188-194](../../internal/git/commit.go#L188-L194), -[types.go:22](../../internal/git/types.go#L22)). - -| Mode | Fact matched? | Author today | -|---|---|---| -| configured-author | n/a | `GitOps Reverser` ← correct | -| attribution | yes | the real actor | -| attribution | **no** | `GitOps Reverser` ← **silent, indistinguishable from row 1** | - -Rows 1 and 3 are byte-identical in Git history. That ambiguity is why the ~7–10% loss took so -long to diagnose, and why the only way to see it today is a counter nobody reads. - -## The core change: an explicit outcome - -**Add an attribution outcome to the event, and derive everything from it.** This is the part -the first draft got wrong by trying to infer outcome from `UserInfo.Username`. - -~~~go -type AttributionOutcome string - -const ( - AttributionNotAttempted AttributionOutcome = "" // configured-author mode - AttributionResolved AttributionOutcome = "resolved" // a fact named the actor - AttributionUnresolved AttributionOutcome = "unresolved" // attempted, no usable fact -) -~~~ - -`AttributionNotAttempted` is the **empty string on purpose**, so that it is also the type's zero -value. Most producers of an `Event` never assign the field — reconcile, resync, bootstrap, and -configured-author mode's early return in `attachAuthor` — and all of them mean "no actor was -sought". A name-shaped constant makes the zero value a silent fourth state that equals none of -the three outcomes; the first implementation did exactly that, and it stopped every -CommitRequest attaching in the default deployment (see the matching section below). Nothing -serializes the value — it reaches no CRD field and no metric label — so the empty string is free. - -Carried on `git.Event` beside `UserInfo`, and on `PendingWrite`. Three consumers then read the -outcome instead of sniffing a string: - -1. **Git author rendering** — `unresolved` renders the sentinel identity (below). -2. **`author_kind` metric** — gets its own value, so a failure never counts as a success. -3. **CommitRequest window matching** — matches on outcome, not on an author string that now - varies. - -This also removes the need to guess "is attribution configured?" from a non-nil resolver, -which the first draft did and which is not a sound proxy. - -### Why "unresolved", not "failed" - -The first draft said `attribution failed`. That overclaims. `AttributionAbsent` collapses -several genuinely different situations: no fact was ever produced (correct — not every change -has an audited human actor), a cancelled wait, a Redis read error, and a malformed value. All -of them return the same "not found" from `matchFactKey` -([attribution_index.go:404-412](../../internal/queue/attribution_index.go#L404-L412)), which -discards the error deliberately. - -So the honest word is **unresolved**: we attempted attribution and did not arrive at an actor. -It still tells a reader something is worth investigating without asserting a fault we cannot -prove. If typed outcomes are added to the index later (distinguishing "no fact existed" from -"lookup errored"), a genuine `failed` state can be split out then — and the outcome enum is -the right place for it. - -## The sentinel identity - -The sentinel is **derived at the write path** (`commitOptionsFor`) from the carried outcome. It -is never stamped onto an `Event`, so it is scoped to the git author header and reaches nothing -else: - -| Field | Surfaces in | Value | -|---|---|---| -| `Username` | machine token inside the sentinel identity; greppable by tooling that parses the header | `attribution-unresolved` | -| `DisplayName` | git author Name via `authorName` | `unknown (attribution unresolved)` | -| `Email` | git author Email via `authorEmail` | `attribution-unresolved@gitops-reverser.invalid` | - -~~~text -Author: unknown (attribution unresolved) -Committer: GitOps Reverser -~~~ - -The operator really did make the commit, so it stays the **committer**; the **author** slot -answers "on whose behalf", and the honest answer is "we do not know". `.invalid` is reserved -(RFC 2606), so the address can never collide with a real one. Parentheses and spaces pass -`isSafeSignatureField`, so the display form is valid in a commit object and a signed payload. - -**The sentinel deliberately does NOT reach message bodies or user templates.** An earlier draft -proposed stamping it onto `event.UserInfo`, which would have pushed it into window grouping, the -grouped commit-message body ([commit.go:96](../../internal/git/commit.go#L96)) and per-event -`{{.Username}}` templates ([commit.go:21-33](../../internal/git/commit.go#L21-L33)). Rejected: -the outcome is the fact and the sentinel is one *rendering* of it, so the rendering belongs at -the surface that needs it. Doing otherwise would change the commit text of every existing -deployment that has attribution misses, and force user-authored templates to special-case a -magic token they never had to handle. An unresolved event therefore renders `{{.Username}}` -exactly as configured-author mode always has — empty — while `git log` names the gap. No -release-note change to user-configured output. - -## HIGH — CommitRequest attachment must not silently stop matching - -A CommitRequest that could not attribute its own author falls back to `Author == ""` -([commitrequest_controller.go:147-151](../../internal/controller/commitrequest_controller.go#L147-L151)), -and attachment matches the open window by **exact author string** -([commit_request_attach.go:106-114](../../internal/git/commit_request_attach.go#L106-L114)): - -~~~go -return p.author == w.Author && p.gitTargetName == w.GitTarget && ... -~~~ - -**Today** an unresolved live window also has `Author == ""`, so the two coincide and the -request attaches. **After a naive sentinel change** the window carries -`attribution-unresolved` while the request still carries `""` — they stop matching, and the -CommitRequest silently lands as a separate commit instead of naming the window it was meant -to. That is a real regression, and the first draft asserting "CommitRequest is unaffected" was -simply wrong. - -Because the sentinel is confined to the git author header (above), the window's `Author` stays -`""` and this naive-sentinel regression never arises. The outcome still participates in matching, -but **only via `NamesActor`, and never as enum equality**. - -**Decision: match on `(author, NamesActor(outcome), gitTarget, gitTargetNamespace)`.** The two -outcomes being compared are produced by *different, independently configured* subsystems — the -window's by mirrored-resource attribution (`--author-attribution`), the request's by command -authorship (`--admission-webhook`), which [cmd/main.go:311-316](../../cmd/main.go#L311-L316) -documents as independent. Requiring the enums to be equal silently couples the two flags: - -| watch attribution | admission webhook | window | CommitRequest | enum equality | `NamesActor` | -|---|---|---|---|---|---| -| off | off | `not_attempted` | `not_attempted` | match | match | -| off | on, miss | `not_attempted` | `unresolved` | **no match** | match | -| on, unresolved | off | `unresolved` | `not_attempted` | **no match** | match | -| on, unresolved | on, miss | `unresolved` | `unresolved` | match | match | - -The two middle rows are real deployments that attach correctly today; under enum equality the -user's commit message is dropped into a separate default-message commit with no error anywhere. - -The author comparison stays **unconditional** — an additional guard, never a replacement. -Skipping it when neither side names an actor looks equivalent (both authors are `""`, so they -compare equal anyway) but makes cross-author attachment depend on the outcome fields being -right: any path that left an outcome unset while the author was set would let one author's -request finalize another's window. Comparing both costs nothing. - -Note this is the opposite call from `openWindow.canAppend`, which *does* compare the enums -directly and should: it compares two events from the same watch pipeline, so they are directly -comparable. The distinction is intra-subsystem versus cross-subsystem. - -**Requires a test that sets each side from its real producer**, not two literals: the gap that -let the enum-equality bug ship was that every existing case left both sides at the zero value, -so `"" == ""` passed and the production combination was never exercised. - -## HIGH — `author_kind` must not report a failure as a success - -`authorKind()` classifies by string -([pending_writes.go:251-260](../../internal/git/pending_writes.go#L251-L260)): empty → -`committer`, `system:serviceaccount:` prefix → `serviceaccount`, otherwise → **`user`**. - -A sentinel username is none of those, so it would be classified **`user`** — and -[interpreting-metrics.md](../interpreting-metrics.md) defines `user` as a real named actor and -says "unattributed events use `committer`". **Failed attribution would appear in dashboards as -improved user attribution**, which is worse than the silence it replaces: it would actively -mask the defect, and could show attribution *improving* as it degrades. - -**Add `unresolved` as a fourth `author_kind`**, driven by the outcome rather than by string -sniffing. Requires, together, in one change: - -- `authorKind()` reads `AttributionOutcome`. -- `docs/interpreting-metrics.md` documents the new value and corrects the "unattributed events - use `committer`" sentence. -- Any dashboard or alert keying on `author_kind` — a `user`-vs-`committer` ratio panel would - otherwise misread the new value. -- Tests asserting the classification for all four kinds. - -## The resolver contract changes - -`ResolveAuthor` documents `ok=false` as **"commit as the configured committer"** -([author_resolver.go:67-82](../../internal/watch/author_resolver.go#L67-L82)). Returning a -placeholder instead silently overturns that contract while leaving the documentation intact — -the exact class of mismatch this whole investigation was caused by. - -Change the signature to return the outcome explicitly rather than a bool whose meaning is -documented one way and implemented another: - -~~~go -ResolveAuthor(...) (git.UserInfo, AttributionOutcome) -~~~ - -`attachAuthor` then stamps `UserInfo` and outcome together. A nil resolver yields -`not_attempted`; a resolver that ran and found nothing yields `unresolved`. This is also what -removes the unsound "non-nil resolver means attribution is configured" inference. - -## Documentation to update in the same change - -The committer-fallback promise is stated in several places; leaving any of them would make the -docs lie in the way this design exists to prevent: - -- [architecture.md](../architecture.md) — "The author identity is never invented". A sentinel - **is** an invented identity, so this must be rewritten to say the author is never - *fabricated as a person*: an unresolved attribution is labelled unresolved, not attributed - to someone. -- [configuration.md](../configuration.md) — the `--author-attribution` description. -- [attribution-setup-guide.md](../attribution-setup-guide.md) — the fallback behaviour users - are told to expect. -- [interpreting-metrics.md](../interpreting-metrics.md) — `author_kind`, as above. -- [UPGRADING.md](../UPGRADING.md) — mandatory release note (below). - -## What genuinely does not change - -- **Configured-author mode.** No resolver runs, outcome is `not_attempted`, author remains the - committer. -- **Resolved attribution.** Unchanged in every respect. -- **Window grouping shape.** Unresolved events share one identity and group with each other, - not with attributed ones — exactly as they do today via `""`. One equivalence class is - renamed, none split or merged. -- **No CRD or API change.** The outcome is internal; it never appears in a spec or status. - -## Cost - -1. **Git history changes.** Tooling matching `GitOps Reverser` to mean "the operator wrote - this" will no longer see unresolved commits. Arguably a fix — those commits were - mislabelled — but a behaviour change on shipped data. -2. **User templates and message bodies are unchanged.** The sentinel is confined to the git - author header, so `{{.Username}}` in a custom `EventTemplate` still renders empty for an - unresolved event, exactly as it does in configured-author mode. -3. **The defect becomes visible with no opt-out.** Adopters will see these commits appear and - reasonably file bugs. Intended — but it makes the release note **mandatory**, pointing at - `gitopsreverser_attribution_resolutions_total` and at - [debug/attribution-loss.md](../debug/attribution-loss.md). - -### No flag - -The only thing a flag could do is restore silently mislabelled authorship. That is not a -preference to preserve behind an opt-out — it is the bug. Attribution is already gated by -`--author-attribution`: an operator who does not want actor names turns that off and gets -configured-author mode, where the committer genuinely is the author. A second flag would add -an incoherent third state — attribution on, but lie when it fails. Making the placeholder -*string* configurable is likewise rejected: it destroys the one property that matters, that -everyone recognises it. - -## Implementation order - -1. `AttributionOutcome` type + the three sentinel constants in `internal/git`. -2. `ResolveAuthor` returns the outcome; `attachAuthor` stamps both. -3. `Event` / `PendingWrite` carry the outcome. -4. `authorKind()` reads the outcome; metrics docs updated together. -5. `matchesWindow` matches on `NamesActor`, never on enum equality; the regression test must - drive each side from its real producer, and cover the `not_attempted`/`unresolved` boundary. -6. Commit author rendering reads the outcome. -7. Documentation sweep (list above) + release note. - -Steps 4 and 5 are the ones that must not be deferred: each is a silent behavioural regression -on its own. - -## Tests - -- **Unit:** all three outcomes → correct author and committer; `author_kind` for all four - kinds; sentinel passes `isSafeSignatureField` and yields a valid commit object. -- **Unit:** two unresolved events coalesce into one window; an unresolved and a resolved event - do not. -- **Unit:** the zero value of `AttributionOutcome` **is** `AttributionNotAttempted`. Everything - below rests on it, and the paths that rely on it never assign the field, so nothing else - would catch a regression. -- **Regression:** the full cross-subsystem matrix for `matchesWindow`, with each side driven by - its real producer — the controller's `gitOutcome()` projection and the watch package's actual - resolver — including the default deployment (webhook off against attribution off) and the - `not_attempted`/`unresolved` boundary. Two literals set to the same value prove nothing here. -- **Regression:** a request whose author differs from the window's never attaches, *whatever* - the two outcomes are — so the author check cannot be made conditional on them. -- **Compatibility:** a custom `EventTemplate` rendering `{{.Username}}` still produces an empty - string for an unresolved event; the sentinel appears only in the git author header. -- **e2e:** author assertions fail as `got unknown (attribution unresolved)`, naming the cause - in the failure message. - -## Relationship to the open defect - -This does **not** fix [debug/attribution-loss.md](../debug/attribution-loss.md); it makes it -legible and gives the investigation a labelled instance tied to an object and time rather than -a counter increment. Worth doing first for that reason — and the explicit outcome is what the -eventual fix will be measured against. diff --git a/docs/design/metrics-observability-plan.md b/docs/design/metrics-observability-plan.md index 067a9100..01576243 100644 --- a/docs/design/metrics-observability-plan.md +++ b/docs/design/metrics-observability-plan.md @@ -51,8 +51,8 @@ serious, honest metrics to show, and the audit subsystem is glass-box. object's `name`/`namespace` in a label. Identity labels stay prefixed (`provider_*`, `gittarget_*`) to survive a `honor_labels=false` pod scrape — see the note in [exporter.go](../../internal/telemetry/exporter.go). -6. **Degradation is loud.** Committer fallback, absent Redis, `410` rebuilds, LIST fallback — each has - a metric, so running in a degraded shape is a visible state, not a silent one. +6. **Degradation is loud.** Unresolved attribution, absent Redis, `410` rebuilds, LIST fallback — each + has a metric, so running in a degraded shape is a visible state, not a silent one. ## 3. Current state (the map) diff --git a/docs/design/residual-e2e-flakes-2026-06-19.md b/docs/design/residual-e2e-flakes-2026-06-19.md deleted file mode 100644 index 3b127c53..00000000 --- a/docs/design/residual-e2e-flakes-2026-06-19.md +++ /dev/null @@ -1,233 +0,0 @@ -# Residual e2e flakes after the first-event-loss fix (2026-06-19) - -> **design** — open, not yet built. Index: [`../INDEX.md`](../INDEX.md) - -Status: ACTIVE FOR FLAKE B; FLAKE A CLOSED BY THE 2026-06-20 ATTRIBUTION FIX. After the -capture-before-baseline fix -(`first-event-loss-on-reclaim-plan.md`, committed in `5d85e7d`), -the IceCreamOrder / first-event-loss flake is gone (it passed every one of 4 warm validation runs). Two -**other**, pre-existing flaky specs remain and intermittently fail the `E2E (full)` job. This doc -explains both, leading with what was *observed*, then the *interpretation* with an explicit **certainty -number** on each conclusion. - -How to read the certainty numbers: they are my honest confidence that the stated conclusion is the true -explanation, given current evidence. **Facts** (observed) are stated separately and are ~100% — they are -measurements, not conclusions. - ---- - -## Flake A — CommitRequest "finalize on demand" times out - -### Facts (observed) - -- CI run `27830248680` (commit `5d85e7d`, the committed fix) failed **only** the `E2E (full)` job; all - other jobs (Build, Unit, Lint, Helm, Dev Container, **E2E quickstart**) passed. -- The failing spec: **"Commit Request — finalizes the open commit window on demand and reports the - resulting SHA"** ([commit_request_e2e_test.go:133](../../test/e2e/commit_request_e2e_test.go#L133)). -- The assertion that timed out: `status.phase == "Committed"` within **120 s** (`Eventually`). -- In the same run, a *different* CommitRequest (`commit-request-bundle-save-…`) reached `Committed` - fine (age 12.7 s) — so the commit/push path itself was working; one CR did not finalize. -- The run **before** this one on the branch (`27826911318`, 12:54) was **green**; an earlier era's - warm run also failed a commit_request spec (investigation §2/§5). So this spec fails *intermittently*. - -### What the spec does - -``` - 1. create Deployment ───────────────▶ opens a commit WINDOW (nothing committed; main not created yet) - 2. Consistently(10s): main is empty ─▶ proves the window is holding the edit - 3. create CommitRequest ────────────▶ asks the controller to FINALIZE the window NOW - 4. Eventually(120s): CR.status.phase == "Committed" ◀── THIS timed out -``` - -Finalizing a window must **attribute it to an author**. The controller does that by scanning the -`commitrequests` audit stream for the CommitRequest's *own* `create` event -(`LookupCommitRequestAuthor`); it **fails closed** (never finalizes someone else's window) if it cannot -find that event within its 60 s attribution window. - -``` - CommitRequest created - │ - ▼ - controller reconcile ──▶ scan commitrequests:audit:stream for THIS CR's create event - │ │ - │ found? ├── yes ─▶ attribute author ─▶ finalize window ─▶ phase=Committed ✅ - │ └── no ─▶ retry (2s) … up to 60s … ─▶ phase=Failed (fail-closed) ✗ - ▼ - spec polls phase==Committed for 120s ──▶ ✗ TIMEOUT (the observed failure) -``` - -### Interpretation - -- **This is a pre-existing flake, not caused by the first-event-loss fix.** **Certainty: 90%.** - Reasoning: the fix changed *when the demand gate opens* (claim-time); it did not touch the - CommitRequest finalize/attribution path, and `commitrequests` is force-mirrored by the static - `AlwaysAllow` set regardless. A commit_request spec also flaked in a pre-fix era (§2). *Residual 10%: - I could not pull this run's controller log (a re-run was in progress, logs locked), so I have not - *directly* confirmed the failure reason for this exact instance.* - -- **The likely root is the CommitRequest attribution-scan miss documented in - `e2e-flakes-2026-06-18-investigation.md §2`.** **Certainty: - 55%.** That investigation proved, for a sibling commit_request spec, that the CR's create event was - *provably present* in the scanned stream for the whole 60 s window — correct RV, within the 512-entry - scan bound, matching uid/name, not diverted, not trimmed — yet **missed by ~30 successive scans**. - Root cause was left **OPEN** (a suspected parse/skip edge or a live-scan race). That same mechanism - would make this `:133` finalize spec fail-closed and time out. *Why only 55%: (a) §2 itself never - closed the root cause; (b) the `:133` spec could instead time out for an adjacent reason — e.g. the - Deployment's window never opened, or the push lagged — and I have not pinned this instance's reason.* - -- **An ordering/sequencing cause is plausible and not yet ruled out.** **Certainty that ordering - contributes: 30%.** The spec checks `phase==Committed` **and** then `status.sha == HEAD`, the message - verbatim, and exactly one commit. A wrong/older/later HEAD (e.g. HEAD advancing past the - CommitRequest's commit, or the window finalizing a different commit) would fail the sha/message/count - checks even though *something* committed. The observed failure was the earlier `phase != Committed` - timeout — which points more at the attribution scan than at sha ordering — but with the commit/push - path proven working in the same run (a sibling CR committed), a sequencing mismatch cannot be excluded - without the diagnostic below. *Why only 30%: the recorded failure was the phase timeout, not a - sha/message mismatch; ordering is the secondary hypothesis.* - -- **It is a correctness fail-closed, not data corruption.** **Certainty: 95%.** By design the finalizer - refuses to finalize a window it cannot attribute; the cost is a failed/stuck CommitRequest, never a - mis-attributed commit. - -### What would raise certainty -The parallel agent has added a **"last 5 commits" diagnostic** (`recentCommitDiagnostics`, -[repo_assertions_test.go](../../test/e2e/repo_assertions_test.go)) to the phase/sha/message/count -assertions ([commit_request_e2e_test.go:123](../../test/e2e/commit_request_e2e_test.go#L123)): on a -failure it prints the latest 5 `origin/main` commits (sha, time, author, subject), path-scoped — so the -next reproduction makes "wrong / older / later commit, or a gap" obvious at a glance. Beyond that: the -one-line `scanForCommitRequestCreate` diagnostic from §2.3 (entries-scanned + whether a create-for-name -was seen on a miss) to separate attribution-miss from sequencing; or read the controller log on the next -reproduction. The durable fix candidate (deferred) is the early-ingestion author store -([internal-audit-type-demand.md](../architecture.md)), which removes the scan entirely. - ---- - -## Flake B — Commit Signing late-joining target: overlap ConfigMap missing under path B - -### Facts (observed) - -- Spec: **"Commit Signing — should not replay already-reconciled configmaps as per-event commits to a - late-joining target"** ([signing_e2e_test.go:507](../../test/e2e/signing_e2e_test.go#L507)). -- The convergence assertion: every `overlap-b-cm-NN` ConfigMap must be **present under path B**. It - runs under the **90 s** suite default (`SetDefaultEventuallyTimeout(90s)`, - [signing_e2e_test.go:82](../../test/e2e/signing_e2e_test.go#L82)); it does **not** override it. - *(Correction: an earlier doc draft said 30 s — that was the explicit timeout in the run-3/f3-era code - shown by `Timed out after 30.000s`; it has since been raised to the 90 s default.)* -- Observed failures: `overlap overlap-b-cm-17` (f3) and `overlap-b-cm-15` (run-3) not present under B. -- **The decisive new fact (a fresh reproduction by the parallel agent, at the 90 s timeout):** the - overlap band **split** — `overlap-b-cm-00 … 15` landed in B's **reconcile (backfill) commit**, - `overlap-b-cm-16` was **missing**, and `overlap-b-cm-17 … 19` arrived as **later per-event (live) - commits**. So an object was missing **even at 90 s**, in a contiguous gap between the backfilled - prefix and the live suffix. -- It did **not** fail the no-replay half (`"seed-cm-" must never appear as a per-event commit`); the - failure is the **convergence** half. -- It is intermittent: passed 3 of 4 warm validation runs and did **not** fail the CI run that Flake A - failed; a re-run after the missing-16 reproduction passed. - -### What the spec does — and what the code actually guarantees at the join - -B, joining late on a type A already tails, gets the object set under its own path from **two** sources: -its **initial backfill** (the splice folds the checkpoint `(R, head]` and writes a reconcile commit) and -the **coverage-watermarked tail** (forwards live entries with stream id `> Hc`). - -**Key code fact (checked this session):** the splice sets `coverageHead = the last entry it folded` -(redis_type_splice.go), and the watermark is published -as **exactly that value** — `publishTargetTypeWatermark(gitDest, gvr, snapshot.CoverageHead)` -([event_router.go:241](../../internal/watch/event_router.go#L241)) — and the tail forwards id `> Hc` -(audit_tail.go). So `Hc` is **contiguous with the fold by -construction**: everything `≤ coverageHead` is in the backfill, everything `> coverageHead` is forwarded -by the tail. There is no `(S, Hc]` window for an object to fall into — *unless its stream entry is -missing entirely*. **This demotes the "watermark set ahead of the backfill" boundary-gap I hypothesised -in the first two drafts.** - -The split that was actually observed (`00–15` backfilled, **`16` missing**, `17–19` live) fits a -**diverted cm-16** far better — the overlap band is one batch `kubectl apply`, so audit events can -arrive **out of RV order**; an event whose RV lands **below the stream high-water** is **diverted** -(rejected from the per-type stream, redis_bytype_queue.go). -A diverted cm-16 is in **neither** the fold (not in the stream) **nor** the tail (not in the stream), -and is healed only when the **divert nudge** re-anchors the type — which races the 90 s deadline: - -``` - overlap band applied in ONE batch → audit events arrive out of order - cm-00..15 ingested in order ─▶ in stream ─▶ folded by B's backfill (≤ coverageHead) ✅ - cm-16 arrives AFTER cm-17 (rv below high-water) ─▶ DIVERTED, never in stream - └─▶ not folded, not tailed ─▶ MISSING under B - └─▶ divert nudge → RequestResync → re-anchor re-LIST ─▶ heals cm-16 ⏱ races 90s - cm-17..19 ingested in order ─▶ in stream, id > coverageHead ─▶ forwarded by tail ✅ -``` - -### Interpretation - -- **Distinct, pre-existing race — not the first-event-loss bug, not caused by the fix.** - **Certainty: 90%.** `configmaps` is a common, already-Synced type; the per-event commits work (no-replay - half passes); the fix touched neither the splice/coverage code nor the divert path. *Residual 10%: the - fix mirrors a claimed type slightly earlier — could marginally shift join timing; not demonstrated.* - -> **Defer to the authoritative deep-dive:** -> [`signing-overlap-band-coverage-drop-investigation.md`](../finished/signing-snapshot-tail-replay-failure-investigation.md) -> is the rigorous (FACT/DERIVED/HYPOTHESIS-tagged) investigation of this exact flake, and it **measured -> the divert counter**. The bullets below are reconciled to it — and it walks back the "divert" lead I -> had given (the diagram above is now just *one — contradicted — candidate*). - -- **What is forced (DERIVED): cm-16 was not a normally-ordered main-stream entry within B's reconcile - cut.** **Certainty: 85%.** The splice's `Desired` (≤ `coverageHead`) and the tail (> `coverageHead`) - are one consistent cut, so a normally-ordered stream entry cannot be dropped — therefore the object - was off the main stream (never ingested, diverted, or an attach anomaly). This is §4.1 of the deep-dive. - -- **The specific path off the stream is UNCONFIRMED — and divert is *contradicted*, not confirmed.** - *(This corrects my earlier "divert leading, 55%".)* The deep-dive measured **`lateCount=0` for - ConfigMaps** at the failure (§4.4), so **H2 (divert) is argued *against*, not for** — the evidence is - the opposite of what I assumed. Current ranking (its §4.5): **H1 "never a normally-ordered stream - entry" is least-contradicted (~40%)**; H3 rv-missing-attach anomaly (`rvMissingCount=66`) open (~20%); - H4 `Hc`-seam **unlikely** because `Hc` is published as the fold's `coverageHead` (contiguous by - construction) *unless* a stale-high prior `Hc` is held (~15%); **H2 divert contradicted (~15%)**. No - mechanism is confirmed — a capture is required. - -- **On "just widen the timeout":** it depends on §5 of the deep-dive — *is "every object present under a - target within one reconcile" a guarantee, or only "by the next checkpoint/heal"?* If the latter, the - 90 s assertion is a **test over-assertion** and widening (or budgeting the heal cadence) is legitimate; - if the former, the join-time path has a real gap to fix. **Unresolved — certainty either way: ~50%.** - -- **Correctness: likely a bounded gap, but "self-healing" is NOT established.** *(Down from my earlier - confident "converges, 70%".)* The deep-dive's §8 says the object was **permanently absent** in that - run's repo (the heal did not visibly recover it within the window, and `DeferCleanup` removed B). So - whether it heals at the next re-anchor is **Q2/Q5 — open**, not a settled fact. The no-replay invariant - did hold. - -### What would raise certainty — and the instrumentation added this session -On the next reproduction, these now-added logs disambiguate divert vs boundary in one read: -- **`late audit event nudged a type resync`** (now **info**, [materialization.go](../../internal/watch/materialization.go)) — fires iff a divert happened; if it names `configmaps` around the join, the divert mechanism is confirmed. -- the scoped **`diag_all`** firehose already captures `configmaps`, so a diverted cm-16 shows as `outcome=older_than_high_water`. -- **`target-type watermark published / held`** (target_type_watermark.go) — shows B's actual `Hc` and whether a stale-high prior was held (the secondary hypothesis). -- **`audit-tail routed batch to target`** (V(1), audit_tail.go) — per-batch `routed / suppressedAtOrBelowHc / firstID / lastID`; if cm-16 appears here and is `suppressedAtOrBelowHc`, it is the boundary case; if it never appears, it was diverted upstream. - -If the repro shows divert: the durable answer is faster/again-reliable nudge-heal (or accept + widen the -test deadline). If it shows a held stale-high `Hc`: fix the publish to not hold a boundary ahead of the -fold. **No fix is applied yet — the mechanism is not confirmed (trust gate not met).** - ---- - -## Summary - -| | Flake A — CommitRequest finalize | Flake B — Signing late-join | -| --- | --- | --- | -| Failing spec | `commit_request_e2e_test.go:133` | `signing_e2e_test.go:507` | -| Symptom | `phase==Committed` not reached in 120 s | one `overlap-b-cm-NN` missing under B (at **90 s**) | -| Observed in | CI `27830248680`; era run-2 | local f3 + run-3 + a fresh split repro (00–15/✗16/17–19) | -| Pre-existing, not the fix | **90%** | **90%** | -| Likely mechanism | attribution-scan miss (§2, root OPEN) — **55%**; ordering a secondary — **30%** | object was **off the main stream** (DERIVED, 85%); *which* path **unconfirmed** — divert **contradicted** by `lateCount=0`; H1 never-ingested least-contradicted (see deep-dive) | -| Correctness | fail-closed, not mis-attributed — **95%** | bounded gap *likely* but self-heal **NOT established** (object was permanently absent in the repro) | - -**Bottom line (certainty 80%):** the first-event-loss fix is sound and landed; `E2E (full)` stays -intermittently red on these two *independent, pre-existing* flakes (the same commit `5d85e7d` failed -E2E(full) on attempt 1 and **passed on re-run** — direct proof they are flaky, not deterministic). -**Neither is evidence against the fix.** Two self-corrections this session: (1) the watermark is -published *as* the splice's `coverageHead`, so the boundary is contiguous by construction (the -`Hc := S` boundary-fix I first proposed is largely moot); (2) the divert lead I then gave is itself -**contradicted** by the deep-dive's `lateCount=0` measurement — so the honest state for Flake B is -"the object was off the main stream, **but by which path is unconfirmed**," tracked rigorously in -[`signing-overlap-band-coverage-drop-investigation.md`](../finished/signing-snapshot-tail-replay-failure-investigation.md). -I deliberately **did not apply a fix** (trust gate not met); instead I landed the instrumentation -(nudge→info, watermark publish/held, per-batch tail-route summary) that, with that doc's §6.1 -ingestion-side capture, collapses H1–H4 on the next reproduction. Flake A's leading hypothesis stays -the attribution-scan miss (55%), ordering a tracked secondary (30%). diff --git a/docs/finished/documentation-triage.md b/docs/finished/documentation-triage.md deleted file mode 100644 index 99f174fd..00000000 --- a/docs/finished/documentation-triage.md +++ /dev/null @@ -1,276 +0,0 @@ -# Documentation triage: what binds, what is history, what to do - -> **finished** — this plan was executed on 2026-07-11. Kept as the record of *why* the -> tree looks the way it does. For what to read now, see [`../INDEX.md`](../INDEX.md). -> -> Outcome: 180 documents -> 117. `docs/spec/` created (25 docs the code depends on), -> `docs/design/manifest/` deleted (19 -> 1 summary + 4 promoted specs), ~50 dead documents -> removed, and every dangling citation repaired — including the 17 that were already broken -> before this started. -> -> Scope as surveyed: all 180 markdown files under `docs/` (~55,000 lines). - -## The headline - -**About 35 documents actually bind. The other ~145 are history.** You cannot -currently tell which is which without reading them, and that — not the volume — is -the problem. - -| Group | Files | Of which still binding | -|---|---:|---:| -| `docs/*.md` (user/operator guides) | ~20 | all — they are the product surface | -| `docs/facts/` | 4 | all — durable reference | -| `docs/design/*.md` + `design/stream/` | 57 | **9 active**, 9 reference | -| `docs/design/support-boundary/` | 13 | **all — the live workstream** | -| `docs/design/manifest/` + `version2/` | 19 | ~3 — the rest shipped | -| `docs/finished/` | 41 | **10 load-bearing** (see below — this is the bug) | -| `docs/future/` | 13 | 8 still wanted | -| `docs/ci/`, `docs/audit-setup/`, misc | 13 | reference | - -## The diagnosis: the folder names lie, in both directions - -This is the whole finding, and it explains why the tree feels unmanageable. - -**`docs/design/` contains 23 documents whose work has already shipped**, sitting -beside the 9 that are still open, with nothing to distinguish them. Several still -say `Status: PROPOSAL` while the code that implements them has been in `main` for -weeks (`unsupported-folder-refusal-plan.md` is the clearest case — it ships as -`GitPathAccepted` with an e2e test, and its header still says "PROPOSAL"). - -**`docs/finished/` contains 10 documents that are the *sole written record* of a -live invariant.** They are not finished at all. The worst case: - -> **`docs/finished/current-manifest-support-review.md`** is the live specification -> of `internal/manifestanalyzer`. It is cited by section name from eight Go files, -> and it is a cited input to active downstream work. It holds the -> non-negotiable rules — *a GitTarget makes an all-or-nothing claim on a folder*; -> *never partially materialize a multi-doc file*; *refuse the folder rather than -> prune unwatched KRM*. It is filed under "finished." - -Nine more like it are listed below. **Archive `finished/` wholesale and you delete -the contract.** - -Only 74 of 180 files carry a `> Status:` line at all, and several of those are -stale. There is no mechanical way to ask "what is still true?" - -## Hard constraint: 17 doc citations in the Go code are already broken - -A previous reorganisation moved documents and never fixed the pointers. Today, -`grep`ping the Go sources for `docs/**.md` yields **17 paths that do not exist**: - -``` -docs/guide.md docs/keep.md -docs/design/manifest/implementation-plan.md docs/design/commit-window-refactor.md -docs/design/manifest/current-manifest-support-review.md docs/design/audit-readiness-probe-plan.md -docs/design/stream/api-source-of-truth-reconcile.md docs/design/support-boundary/f1-...md -docs/future/secret-value-retention-plan.md docs/design/support-boundary/f7-...md -docs/design/stream/audit-diagnostic-streams-plan.md …and 6 more -``` - -Six of those point at documents that no longer exist anywhere. The rest point at -documents that *moved into* `docs/finished/`. - -Two consequences, and they shape the entire plan below: - -1. **The code already thinks the load-bearing docs live in `docs/design/`.** That - is an argument for moving them back rather than for leaving them where they are. -2. **Any move must fix citations in the same commit,** or it makes this worse. This - is exactly how the current mess was made. - -## What actually binds — the read-this list - -If you read nothing else, read these. This is the set that describes how the system -works *today* or what is being decided *now*. - -### The live workstream (13) — `docs/design/support-boundary/` - -All of it. This is the branch you are on. My separate proposal -([expansion-boundary-and-corpus-organisation.md](../design/support-boundary/expansion-boundary-and-corpus-organisation.md)) -recommends adding a `support-contract.md` front door and folding four restatements -of the fan-in rule into one. - -### Load-bearing, but misfiled under `finished/` (10) — **promote these** - -Each is the only place a still-current invariant is written down, and most are cited -from the code: - -| Doc | The invariant that lives only here | -|---|---| -| `current-manifest-support-review.md` | all-or-nothing folder claim; never partially materialize a multi-doc file; refuse rather than prune | -| `manifestedit-field-ownership-spike.md` | "the API wins" — full-object ownership, no field-subset ownership, plus the explicit do-not-build list | -| `sops-single-file-no-multidoc.md` | SOPS encrypts a *file* as one cryptographic unit → one document per encrypted file | -| `scale-subresource-audit-rehydration.md` | the `/scale` → `FieldPatch` translation rules and denied-subresource list | -| `commit-window-refactor.md` | one grouped commit = exactly one (author, GitTarget) tuple | -| `commitrequest-multi-finalize-design.md` | why `MaxConcurrentReconciles=1` on the CommitRequest controller | -| `type-lifecycle-events-and-wobble-settling.md` | the settle-window / flap-coalescing spec `internal/typeset` implements | -| `gittarget-isolation-on-rule-change.md` | a rule change on target A must never re-snapshot target B | -| `audit-readiness-probe-plan.md` | why liveness must never depend on Redis (the restart-loop trap) | -| `gvk-gvr-mapping-layer.md` | GVK↔GVR contract; still cited by four active design docs | - -### Still-open design (9) — `docs/design/` - -`e2e-coverage-gaps-and-improvements-plan.md`, `e2e-finish-plan.md`, -`metrics-observability-plan.md`, `multi-source-audit-ingress-hardening.md`, -`reconcile-triggering.md`, `release-image-reuse-plan.md`, -`sensitive-resource-diagnostics-follow-up.md`, `watch-and-catalog-architecture.md`, -`stream/residual-e2e-flakes-2026-06-19.md`. - -### Durable reference (9 + `docs/facts/`) - -`status-conditions-guide.md`, `e2e-serial-registry.md`, `e2e-test-design.md`, -`kubernetes-api-discovery.md`, `kubernetes-api-resource-catalog.md`, -`kubernetes-apf-and-inflight-tuning.md`, `audit-webhook-api-server-connectivity.md`, -`watch-event-ordering-and-attribution-grace.md`, `sops-repo-bootstrap-out-of-scope.md`. - -### Still wanted (8) — `docs/future/` - -`idea-application-editing.md` (the product seed of a future application-editing workstream — -still holds the branch/session grouping strategies nothing else covers), -`ha-gittarget-distribution-plan.md` (cited three times by `architecture.md`), -`least-privilege-remaining-work.md`, `design-commit-request-phase-2.md` (verified -live gap: nothing GCs `CommitRequest` objects), `idea-unify-pending-write-kinds.md`, -`idea-cross-kind-dependency-watches.md`, `publish-age-recipients-as-metadata.md`, -`idea-commit-signing-key-validation.md`. - -## What is history - -### Shipped, but still sitting in `docs/design/` (23) — move to `finished/` - -`audit-webhook-tls-design`, `commitrequest-admission-authorship`, -`deletecollection-attribution-expander`, `dynamic-analysis-fuzzing-plan`, -`e2e-aggregated-apiserver-test-design`, `e2e-bi-directional-corner`, -`e2e-ci-runner-sharding-plan`, `e2e-signing-followups`, `e2e-speedup-plan`, -`git-credentials-interop`, `gitpath-foreign-content-stringency`, -`image-refresh-test-design`, `mutation-capture-lab-design`, `redis-key-schema-v3`, -`sensitive-resource-classification-plan`, -`sops-repo-bootstrap-and-key-management-architecture`, -`startup-robustness-cert-and-crd-wobble`, `tilt-playground-plan`, -`typeset-owns-discovery-grace`, `unsupported-folder-refusal-plan`, -`watch-first-ingestion-architecture`, `stream/commitrequest-design`, -`stream/streaming-readiness-status-machine-design`. - -Note `watch-first-ingestion-architecture.md` is the *most* important of these — it -is the rewrite that deleted half the audit world — but it has landed, so it is now -an explainer, not a plan. It should arguably graduate into `docs/architecture.md` -rather than into `finished/`. - -### Superseded (6 + 13) — **delete** - -Git is the archive. These have named successors and nothing cites them: - -- In `design/`: `e2e-aggregated-apiserver-proxy-hookup-plan`, - `gittarget-lifecycle-and-repo-architecture`, - `gittarget-state-machine-bootstrap-assessment`, `status-design-git-target`, - `watchrule-wildcard-and-resolution-semantics`, - `stream/per-type-streaming-readiness-plan`. -- In `finished/`: `api-source-of-truth-reconcile`, - `commitrequest-attribution-divert-reliability`, `internal-audit-type-demand`, - `design-commit-request-api`, `deletecollection-resync-nudge-plan`, - `audit-metrics-overhaul-plan`, `materialization-tail-and-live-readiness-review`, - `watch-list-checkpoint-plan`, `per-resource-type-rv-keyed-streams-experiment`, - `idea-audit-enrichment-side-channel`, `rule-set-snapshot-discovery-lag-fix`, - `watchrule-gvr-resolution-plan`, `manifestedit-new-file-placement-spike`. - -One catch before deleting `materialization-tail-and-live-readiness-review.md`: -`internal/typeset/materializer.go:34` still cites its "Rec 6" as an open -optimisation. Move that note into the code comment first. - -### Closed investigations (9) — **delete** - -Forensics whose bugs are fixed: `e2e-full-suite-shared-state-investigation`, -`e2e-watchrule-cross-spec-interference`, `upgrade-finding`, -`watch-audit-rule-matching-improvement`, `watch-first-merge-readiness`, and four -under `stream/`. These are the genre with the worst read-value-to-length ratio — -long, dramatic, and entirely about bugs that no longer exist. - -Keep one exception: `stream/signing-snapshot-tail-replay-failure-investigation.md` -is cited from `internal/git/branch_worker.go:330` as the reason a watermark gate -exists. Move that reasoning into the code comment, then delete. - -### Archaeology (18 in `finished/`) — cold-store or delete - -Mostly the pre-watch-first audit world: `demand-gated-audit-ingestion`, -`canonical-stream-retirement`, `shallow-audit-event-misclassification`, -`partial-object-audit-event-handling`, `design-audit-ingestion-hardening`, and 13 -more. `internal/gate` and the audit-as-state pipeline they describe are **deleted -code**. Nothing forward-looking depends on them. - -### Overtaken / stale (5) — delete - -`future/design-snapshot-engine-evolution` (every part landed elsewhere in a -different shape), `future/idea-end-user-commit-messages` + -`future/addendum-end-user-commit-messages-audit-transports` (both rest on "the audit -stream is the source of truth", which watch-first made **false**), -`future/watchrule-wildcard-support-plan`, -`future/further-away/accesspolicy-design` (targets `GitRepoConfig`, a CRD two -renames dead), `design/voter-gitops-demo-instances`. - -## The plan - -Phased so that the cheap, zero-risk win lands first and nothing breaks. - -### Phase 0 — the index (zero risk, most of the benefit) - -**Write `docs/INDEX.md`: the ~35 documents that bind, grouped by topic.** Move -nothing. Delete nothing. You immediately get "read these 35, the rest is history", -which is the actual thing you asked for. Everything below is optional cleanup. - -### Phase 1 — status headers (mechanical) - -Put a `> Status:` line on every design doc, from a closed vocabulary: -`active` | `shipped` | `superseded-by: ` | `closed-investigation` | -`reference`. Then `grep '^> Status: active'` answers the question forever, and the -next person does not have to redo this triage. - -### Phase 2 — fix the 17 broken citations, and promote the 10 load-bearing docs - -Do these **in one commit**, because the fix and the move touch the same lines. The -code already points at `docs/design/` for most of them, so promoting them back -*is* the fix. - -Suggested destination: a `docs/spec/` folder meaning **"this is true now and the -code depends on it"** — distinct from `design/` ("we are still deciding") and -`finished/` ("this happened"). The three-way split is what the current two-way one -cannot express, and it is precisely the distinction that broke. - -### Phase 3 — move the 23 shipped docs out of `design/` - -Now `docs/design/` means what it says: undecided work. It shrinks from 57 files to -~9. - -### Phase 4 — delete the dead (~50 files) - -Superseded, closed investigations, archaeology, stale. Extract the three code-cited -facts first (noted above). Git keeps everything; nothing is lost that a -`git log --follow` cannot recover. - -### Phase 5 — documentation consolidation - -Separately proposed in -[expansion-boundary-and-corpus-organisation.md](../design/support-boundary/expansion-boundary-and-corpus-organisation.md): -add a `support-contract.md` front door, fold the four restatements of the fan-in -invariant into one, and de-duplicate the `--dry-run=server` evidence table. - -## What I would *not* do - -- **Do not bulk-archive `docs/finished/`.** Ten of those documents are the contract. - That is the single highest-risk action available here, and it is the one that - looks most obviously safe. -- **Do not move anything without fixing the Go citations in the same commit.** That - is how the current 17 broken pointers were created. -- **Do not delete the investigations to save space.** Delete them because their bugs - are gone. Two of them still hold the *reason* a guard exists in live code; those - reasons must land in the code comment first, or you will lose them and someone - will remove the guard. - -## Open question - -**Should `docs/design/manifest/` survive at all?** Its three subsystems — -`internal/typeset`, `internal/git/manifestedit`, `internal/manifestreport` — have -all shipped, so most of its 19 files are archaeology. But it is also where the -*reasoning* behind the manifest model lives, and downstream work is built -directly on top of it. My instinct is that two or three docs graduate to `spec/` -(`contextual-namespace-and-kustomize-folder-editing.md`, -`version2/gittarget-new-file-placement-rules.md`, -`version2/type-followability.md`) and the other sixteen go, but that is the call I -am least sure of and it is worth your eye rather than mine. diff --git a/docs/finished/redis-key-schema-v3.md b/docs/finished/redis-key-schema-v3.md index 3cccc312..41dd5f32 100644 --- a/docs/finished/redis-key-schema-v3.md +++ b/docs/finished/redis-key-schema-v3.md @@ -216,7 +216,8 @@ unique across resource types, and aggregated/CRD API servers can mint their own. an `objectRef` without a UID. Under the `RequestResponse` audit level this project relies on, `IdentityFromAuditEvent` ([`identity.go`](../../internal/auditutil/identity.go)) backfills the UID from the body even for `generateName` creates, so a no-UID fact is uncommon — but when it happens, `rv:` -is the difference between a precise author and a committer fallback. It is **cheap and shipped now**: one +is the difference between a precise author and an explicit unresolved author. It is **cheap and shipped +now**: one extra key, written *only* when the fact has no UID (a UID-bearing fact's `rv:` key would be dead — the watch side always carries a UID, so it would resolve via `object::…` first and never reach it). This is a deliberate improvement over v2, which wrote a dead rv-only key for *every* event. diff --git a/docs/finished/watch-first-ingestion-architecture.md b/docs/finished/watch-first-ingestion-architecture.md index 42ffa3a8..5d377006 100644 --- a/docs/finished/watch-first-ingestion-architecture.md +++ b/docs/finished/watch-first-ingestion-architecture.md @@ -1,506 +1,36 @@ -# Watch-first ingestion architecture - -> **finished** — shipped or closed. Kept for context only; **nothing here binds**. For current behaviour see [`../spec/`](../spec/). Index: [`../INDEX.md`](../INDEX.md) - -> Status: **accepted direction — big-bang rewrite** -> Date: 2026-06-25 -> -> Related: -> [Current architecture](../architecture.md), -> [Mutation Capture Lab](../spec/mutation-capture-lab-design.md), -> [API-source-of-truth reconcile](../architecture.md), -> [Mutation lab corpus](../../test/mutationlab/corpus/), -> [Mutation lab README](../../test/mutationlab/README.md) - -## Summary - -GitOps Reverser today treats the **audit webhook** as the authoritative live mutation stream, and -spends a large amount of machinery making that stream ordered, gap-free, demand-gated, and -attributable. The mutation-capture corpus proved that Kubernetes **WATCH** is a strictly better source -for persisted object state on exactly the cases that hurt today (aggregated APIs, shallow bodies, CRD -conversion, deletecollection fan-out). This document is the decided target: **make WATCH the only -source of object state, run one watch set per GitTarget, and demote audit to an optional attribution -lookup table.** - -Three decisions frame this rewrite: - -1. **Per-GitTarget watches.** Each GitTarget opens and owns the watches for the resource types it - claims. A GitTarget becomes a self-contained unit — its watches, its event stream, its branch - worker, its commit window — which makes horizontal scaling and HA a matter of *assigning GitTargets - to replicas*, not coordinating a shared per-type pipeline. It also keeps each watch's replay and - mark-and-sweep bound to a single Git tree, which is what makes them simple (see the dedup note under - [HA](#per-gittarget-ownership-and-ha)). Sharing watches across GitTargets is an explicit future - enhancement, not the first cut. -2. **Redis is required; audit attribution is the optional layer on top.** Redis/Valkey holds each - GitTarget's watch **resume cursors** (state continuity / work re-pickup) and is the substrate for HA - and the planned durable branch-worker queue, so it is a hard dependency in every mode. What is - optional is **audit attribution**, toggled by `--author-attribution` (chart - `attribution.enabled`). With attribution off the product runs **configured-author**: it mirrors cluster - state to Git, authored by the configured committer identity, and Redis still backs the resume cursors. - With attribution on (and the audit webhook configured to post to it), the product additionally - resolves the **author** from audit facts when the evidence is strong. Attribution off → committer. - Attribution on → committer **and** author. -3. **Big-bang.** The audit-as-correctness pipeline is removed, not kept behind a flag. The current - design is not perfect and is not worth preserving in parallel; the simplification is large enough - that a clean replacement is cheaper to reason about than a dual-mode coexistence. - -The conceptual core: **the Git tree is a projection of persisted state observed by watch.** Audit -explains *who* caused a change when it can; it never defines *what* changed. - -## The realization that deletes a subsystem - -Once a live watch is the source of truth, two facts follow that erase most of the current engine: - -- **Git is the materialized state; the watch is the only feed into it.** Each watch event is sanitized - and diffed against current Git content — there is no per-type Redis object log to reconstruct, and no - separate in-memory cache to keep authoritative (Git already holds current state). -- **The apiserver delivers events already ordered by `resourceVersion` per type.** There is nothing to - re-order. The entire reason the current system exists — making interleaved, out-of-order audit - batches into a per-type ordered stream — goes away. - -So the following all disappear, not as an optimization but as dead concepts: - -- the per-type Redis audit streams and their RV-anchored stream positions; -- the **late-lane / divert / nudge** machinery (out-of-order numeric cross-writes were an audit-batch - artifact; watch has no such thing) — the recurring flaky invariant retires with it; -- the **demand gate** (it existed to stop a per-type *mirror* exploding across cluster types; there is - no mirror to bound); -- the **audit log splice fold**, its RV-anchored stream position, and the **periodic checkpoint LIST + - timer-driven drift sweep** — a mark-and-sweep survives, but only on watch re-establishment, never on a - timer (see [Resume and recovery](#resume-and-recovery)); -- the **materialization phase machine** (Dormant→Requested→Syncing→Synced…) and its durable - `:objects:state` — the live watch plus the Redis RV cursor replace it, and the "re-claimed type stuck - in phase=removed" class of bug retires with the phase machine. - -The whole engine is one raw watch per `(GVR, scope)` with `sendInitialEvents`, a Redis resume cursor, -and a mark-and-sweep on replay — **no periodic LIST, no informer resync, no `SharedInformer`** (a -`SharedInformer`'s reflector re-LISTs on every process start and exposes no cross-restart resume cursor, -which is exactly what we rely on). See [State ingestion](#state-ingestion-raw-watch-per-type) and -[Resume and recovery](#resume-and-recovery). - -## What each question is answered by - -| Question | Source | -|---|---| -| **What changed?** | WATCH (with `sendInitialEvents` replay) over claimed, followable types. The only correctness path. | -| **Who caused it?** | Optionally, audit facts in Redis — only on strong evidence, only when audit attribution is enabled. | - -A missing, late, shallow, conflicting, failed, dry-run, or simply absent audit fact never blocks or -alters state capture. It can only change the commit *author* from the bot to a named user/service -account. - -## Target architecture - -```mermaid -flowchart TD - subgraph K8S["Kubernetes API"] - DISC["Discovery / CRDs / APIServices"] - WATCH["WATCH + sendInitialEvents replay (per claimed GVR + scope)"] - AUDIT["Audit webhook (optional)"] - end - - subgraph PERGT["Per GitTarget"] - OWN["GitTarget watch set\n(claimed ∩ followable (GVR, scope))"] - RELEVANCE["Relevance filter\n(sanitize · followability · no-op diff)"] - ESTREAM[("GitTarget event stream")] - WORKER["BranchWorker → writer → push"] - end - - subgraph ATTR["Attribution (only when enabled)"] - AFACTS["Audit fact extractor"] - AINDEX[("Redis attribution index\n(TTL, keyed for join)")] - RESOLVE["Conservative resolver\n(bounded grace window)"] - end - - DISC --> OWN - WATCH --> OWN - OWN --> RELEVANCE - RELEVANCE --> RESOLVE - RESOLVE --> ESTREAM - ESTREAM --> WORKER - - AUDIT -. configured .-> AFACTS - AFACTS --> AINDEX - AINDEX -. lookup .-> RESOLVE -``` - -In this shape: - -- A GitTarget reconcile resolves its claimed `(GVR, scope)` set, then opens one watch (replay + live) per - `(GVR, scope)`. **Scope is part of the watch identity:** a namespaced WatchRule opens a namespaced - watch (or a field-selector-bounded one), never a cluster-wide watch filtered in-process — that would - be both an RBAC/privilege escalation and a load multiplier. The existing namespace tracking - (`NamespaceOps` / `SnapshotNamespaces`) supplies the scope. There is no global per-type pipeline; - ownership is per GitTarget. -- The **relevance filter** — owned by product code — discards the controller noise the cluster's audit - policy used to discard for free (status churn, runtime-owned subresources, no-op diffs). -- Every Git write derives from persisted state observed by watch (live, or via the `sendInitialEvents` replay). -- The **resolver** attaches an author only when audit attribution is enabled *and* an audit fact matches - strongly; otherwise the commit is authored by the configured committer. -- Audit events never create object changes and never repair object bodies. - -This is still "watch + replay," not "watch from nothing": a reliable watch needs an initial replay of -current state (`sendInitialEvents`) and a fresh replay after `410`. "Watch-first" means the watch is the -*only* object-state ingestion mechanism — there is no separate LIST/checkpoint path. Audit is advisory -and optional. - -## State ingestion: raw watch per type - -A per-GitTarget set of raw watches replaces the per-type audit stream as the live object log. - -| Current audit-first concept | Watch-first replacement | -|---|---| -| `...:audit:stream` (per type, in Redis) | raw watch event loop → diff against Git (no separate object store) | -| Audit event payload | sanitized watch object | -| Audit tail reader (blocking XREAD per type) | raw watch per `(GVR, scope)` | -| Checkpoint + audit log splice | `sendInitialEvents` replay-as-ADD on every (re)connect; `(GVR,scope)→RV` resume cursor in Redis | -| Audit body joiner | removed (watch carries the body) | -| Shallow-audit drop / wait-for-body | not relevant to state content | -| Demand gate (bound the mirror) | open watches only for claimed ∩ followable `(GVR, scope)` | -| **Audit policy noise filter** | **product-owned relevance filter** (new responsibility) | - -Each watch event carries everything Git needs: GVR, scope, event type (`ADDED` / `MODIFIED` / `DELETED`, -plus the transport-level `BOOKMARK` and `ERROR` a raw watch delivers), -namespace/name/UID/resourceVersion/generation/deletionTimestamp, and the sanitized object for -object-bearing events. We consume `BOOKMARK` directly: the `initial-events-end` bookmark marks the end -of the replay, and bookmark RVs advance the resume cursor we persist to Redis; an `ERROR` (e.g. `410 -Gone`) triggers a fresh `sendInitialEvents` reconnect. Each event flows into the existing per-GitTarget -[`GitTargetEventStream`](../../internal/reconcile/git_target_event_stream.go) as a -[`git.Event`](../../internal/git/types.go), then to the BranchWorker — the same seam audit events use -today, so the entire downstream writer is untouched. - -### Resume and recovery - -Losing a watch — pod eviction, rollout, crash, `410 Gone` — is normal, and the design's job is to come -back *correctly*, never silently dropping a delete. There are exactly two resume modes, and the choice -between them is the choice between "we still know precisely where we were" and "we don't." - -**Mode A — resume from the exact point (efficient, preferred).** We persist -`(GVR, scope) → last resourceVersion` to Redis as events arrive, and keep it fresh with watch bookmarks -even when the type is quiet. On restart we re-open the watch *from that RV* — a plain delta watch, no -replay. If the apiserver still holds that RV in its watch cache, it streams every event since, -**including the `DELETED`s we missed**, and we simply continue. No replay, no sweep. This is the common -case for short gaps, and it is the efficient path: we pick up exactly where we left off. - -**Mode B — replay with mark-and-sweep (the correctness fallback).** If the exact RV is gone (compacted -→ `410 Gone`, no cursor yet, or a cold start), a delta is untrustworthy — objects may have been -deleted while we were away and a delta from an unknown point would never reveal it. So we open with -`sendInitialEvents=true` and run a **mark-and-sweep** over the replay: - -1. **Mark** every object the replay delivers as `ADDED`, up to the `initial-events-end` bookmark. -2. **Sweep** at the bookmark: for this `(GVR, scope)`, any Git file whose object was *not* marked no - longer exists — emit a `DELETED` for it (committer-authored; we never witnessed the actual delete). -3. Then stream live. - -**This mark-and-sweep is load-bearing and must not be dropped.** It is the *only* thing that reconciles -a delete that happened while no watch was running — replay-as-ADD alone tells us what *exists*, never -what is *gone*, so without the sweep an orphaned Git file would linger indefinitely. Mode B is what -makes watch-first safe to lose and restart. It reuses the existing reconcile/sweep writer machinery; -what changes from the old design is the **trigger** — it fires on watch re-establishment (a replay), -**not** on a periodic timer. We deliberately do **not** run an hourly sweep: it runs exactly when -continuity was lost, and never while the watch has been continuously live. - -Net: keep the watch live and every delete is seen directly; resume from the exact RV and the delta -carries the deletes; otherwise replay and the mark-and-sweep reconciles them. **No path silently loses a -delete.** - -### Relevance filtering becomes product code (the cost that does not shrink) - -This is the one place the rewrite *adds* work, and it must be counted honestly. The committed e2e -audit policy ([test/e2e/cluster/audit/policy.yaml](../../test/e2e/cluster/audit/policy.yaml)) is not a -passthrough — it is a three-tier relevance filter that drops `*/status`, HPA `*/scale`, leases, -events, node heartbeats, and keeps human-meaningful create/update/patch/delete. **Watch has no such -policy.** It delivers every persisted `MODIFIED`, including all the churn the policy dropped at the -source before it ever reached the webhook. - -Watch-first mode must reproduce that filter in product code, on the hot path: - -1. **No-op suppression.** A `*/status` write bumps `resourceVersion` and produces a `MODIFIED` whose - sanitized desired-state projection equals the prior commit. The writer already diffs to no-op, but - now we pay per-event CPU on every status churn before discarding it. -2. **Followability encodes "controller-owned."** [`internal/typeset`](../../internal/typeset/) is the - home for "we do not mirror this type's churn" — the analogue of audit policy Rule 1. -3. **Sanitization is mandatory and on the hot path.** [`internal/sanitize`](../../internal/sanitize/) - strips status, managedFields, and volatile metadata before diffing so runtime churn never - masquerades as a desired-state change. - -We are not removing a filter; we are *moving* it from the cluster's audit policy into the product. That -is more honest and version-portable, but the event-volume cost is real and continuous for status-heavy -clusters. The filter must be observable (see metrics), so a mis-tuned filter is visible rather than -silently dropping intent. - -### History granularity changes (accept and document) - -Watch carries only the versions it observes. While connected it sees each `MODIFIED`; across a replay -after `410`, a compaction, or process downtime, it **collapses every intermediate version into current -state**. Those become one replay/committer commit (or none, if the net diff is empty), not the N user -commits audit would have produced. - -Honest guarantee: **watch-first delivers every persisted mutation observed while watching, and -collapses to current state across gaps.** It is a *state mirror with opportunistic per-mutation -history*, not a guaranteed per-mutation change log. This must be explicit in user docs. - -## Per-GitTarget ownership and HA - -A GitTarget owns the watches for its claimed types. This is the deliberate design choice and it buys -the HA story almost for free: - -- **Sharding is per GitTarget.** Assign whole GitTargets to replicas (leader-elected ownership or a - lease per GitTarget). A replica runs the watch sets for the GitTargets it owns. There is no shared - per-type pipeline to coordinate, no per-type handoff to lose. -- **Failover resumes from the cursor.** A replica taking over a GitTarget re-opens each watch with - `sendInitialEvents=true`, floored at the `(GVR, scope) → RV` cursor in Redis when one exists, or from - current state when the type has no cursor yet. Either way the replay-as-ADD reconstructs state - idempotently against Git; no durable object checkpoint is needed. -- **Correctness rests on replay, not on Redis durability.** Redis is a required dependency, but it - stores only two small things — the resume cursors and the attribution facts — and neither is a - *correctness* input. A missing or stale cursor never produces wrong state: the failover just degrades - to a full Mode-B replay (replay-as-ADD + mark-and-sweep) instead of an exact-RV delta resume. This is - what lets a GitTarget move between replicas freely. - -**The dedup tradeoff — accepted, and why.** Per-GitTarget watches mean that if two GitTargets both -claim `ConfigMaps` in the same scope, two ConfigMap watches run — duplicate API watch load and a -duplicate connection, ×N for a widely-claimed or wildcard type. We accept that inefficiency, and not -only for simplicity: **the replay makes per-GitTarget the natural unit.** Resume state is inherently -per-GitTarget — each watch's Mode-A cursor and its Mode-B mark-and-sweep are tied to *one* GitTarget's -Git tree, and two GitTargets sharing a `(GVR, scope)` can be in different resume modes at once (one -resuming from an exact RV, the other cold and needing a full replay). A single shared watch cannot be -both, so it would have to decouple the transport from the per-GitTarget replay/sweep and run N -independent mark-and-sweeps off one stream — exactly the fragility we are avoiding. So the watch -identity is `(GitTarget, GVR, scope)`, and each replay stays clean. - -**Future enhancement (explicit).** When overlap proves costly, share one watch per `(GVR, scope)` within -a replica and fan its events out to every owned GitTarget event stream that claims it (scope-filtered) — -the event-router fan-out seam already exists. The hard constraint is that it must preserve per-GitTarget -resume cursors and per-tree mark-and-sweep; it may deduplicate only the transport, never the replay -bookkeeping. Deferred until measured. - -## Attribution: an optional lookup table with slack - -Audit ingestion becomes an optional fact extractor that runs **only when audit attribution is enabled**. -It stores the smallest facts needed to name an author, not an authoritative object log. - -Fact shape (minimized): - -| Field | Purpose | -|---|---| -| `auditID` | diagnostics / dedupe | -| `user` / `impersonatedUser` | author candidate (human *or* service account) | -| `verb`, `subresource` | explain the write | -| `responseStatus.code`, `dryRun` | reject failures and non-persistent requests | -| GVR, namespace, name, UID (when available) | exact join keys | -| response object RV (when available) | exact watch-event match | -| request/stage timestamps | bounded time matching | - -The index is keyed for the join, strongest first: - -- `(group, resource, namespace, name, uid, responseRV)` — exact; -- `(group, resource, namespace, name, uid)` — for `deletecollection`, expand the response List into one - candidate per item and join the watch `DELETED` by UID, **without** RV equality; -- `(group, resource, namespace, name, responseRV)` — exact when UID absent; -- `(group, resource, namespace, name, time-bucket)` — weak, last resort. - -Retention is bounded by max audit delay + the attribution grace period — minutes, not hours. Old facts -are never needed for correctness because watch owns state. - -**The "slack."** A watch event waits a **bounded grace window** for a matching fact to arrive in the -index, then ships regardless. This is the one place watch-first still waits on audit, and it is what -makes "a late audit arrival must not rewrite a shipped commit" enforceable: we wait briefly *before* -shipping rather than rewrite afterwards. It is per-event, bounded, never a barrier, and it expires to -committer-authored rather than blocking state. - -### Confidence policy (strict — a wrong author is worse than no author) - -Because audit captures service-account activity too, a matched non-human actor is a *named* -attribution, not "unknown." Naming `system:serviceaccount:flux-system:kustomize-controller` is useful. - -| Confidence | When | Author | -|---|---|---| -| Exact (human) | watch UID/name/GVR/RV matches audit response; success; non-dry-run; real user | real user | -| Exact (service account) | same match strength; actor is an SA/controller | named SA (its own username) | -| Strong causal | scale subresource with matching parent objectRef + response RV (the parent watch event lands at exactly that RV) | real user / SA | -| Deletecollection | a `deletecollection` audit fact whose `responseObject` is a List is expanded into per-object candidates; each watch `DELETED` joins by **GVR + namespace + name + UID** — **not** RV (audit lists the pre-delete RV, watch the later delete RV; UID is the stable key) | real user / SA (reason `exact-deletecollection-item`) | -| Terminal delete (finalizer / cascade) | finalizer-removal or owner-ref cascade: the watch `DELETED` lands at a later RV and the actor that removed the object is a controller (corpus Rows 8/10) | committer, unless an *exact* match to the removing actor's audit fact exists (then that named SA) | -| Weak | same GVR/name/time only, missing RV/body, or multiple candidates | committer | -| Conflict | failure, dry-run, mismatched UID/RV, multiple users, stale | committer, with metric | -| Absent | attribution disabled, no audit fact, policy dropped the write, or no match | committer | - -`deletecollection` is the one delete case strong enough to attribute: implement it as a fact *expander* -— one collection fact becomes N per-object candidates keyed by GVR/namespace/name/UID — and join each -watch `DELETED` by UID. This holds only when audit captures the `responseObject` List; if the policy -gives request bodies only, or an aggregated API omits the response, it falls back to weak selector/time -matching or committer. Do **not** apply the expander to finalizer-delayed terminal deletes. - -The finalizer/cascade downgrade is deliberate, and it is exactly the case the strict rule exists for: -attributing a terminal `DELETED` to the human who *asked* for the delete looks right but commits the -file removal under the wrong actor. The human's `delete` only stamped `deletionTimestamp` (an earlier -event at an earlier RV); a controller removed the finalizer and caused the actual removal. Per "a wrong -author is worse than no author," that commits as committer unless we match the finalizer-removing patch -exactly — which usually names a service account, not the human. - -A matched actor — human **or** service account — is always named by its own username; there is no knob -to collapse a service account to the committer. The one thing the product exposes is a machine-readable -**reason code** on every outcome (`exact-user`, `exact-sa`, `exact-deletecollection-item`, `weak-no-rv`, -`conflict-multi-user`, `absent-attribution-disabled`, `absent-policy-dropped`, `expired`) so unknown -rates are explainable, not mysterious. - -## Commit windows and authors - -The current `BranchWorker` window accepts one `(author, GitTarget)` pair at a time -([open_window.go](../../internal/git/open_window.go), keyed on `event.UserInfo.Username`). Watch-first -needs only small adjustments: - -- exact-attributed events (human or named SA) use that actor as the author bucket — unchanged; -- everything else uses the configured committer identity; -- the bounded grace window may delay routing a watch event while a fact arrives; on expiry it routes as - committer and is **not** rewritten later; -- replay/reconcile changes use the committer identity; -- committer and attributed events must not be grouped into one real user's commit (the existing - no-blended-authors safety property is preserved). - -This means watch-first can produce *more* commits than audit-first when attribution is mixed, and -sometimes *fewer* when bursts/downtime collapse to current state. Both are acceptable and documented. - -## CommitRequest implications - -`CommitRequest` currently **fails closed** if it cannot attribute the requester -([commitrequest_controller.go](../../internal/controller/commitrequest_controller.go), -`attributionFailedMessage`). That does not fit a configured-author install. - -New behavior: - -- the controller-runtime watch on the `CommitRequest` object still triggers finalization; -- attribution is optional: with attribution enabled + a matching fact, the requester is named; otherwise - the request **finalizes as committer** with a status note that finalization happened without end-user - attribution; -- the request must never fail solely because attribution is disabled or no audit fact matched. - -If preserving the CommitRequest submitter matters without audit, that needs an explicit user field or a -signed request — Kubernetes object state alone does not identify the human who created it. - -## Operating modes - -| Mode | State source | Redis / audit | Author fidelity | Use | -|---|---|---|---|---| -| **Configured-author** | watch | Redis optional (no audit webhook); cold-replay without it | committer identity | simplest install, state mirror | -| **attributed-author** | watch | Redis + audit webhook | named user/SA on strong match; committer otherwise | recommended target | - -The product must not imply configured-author equals attributed-author mode, nor that attributed-author mode recovers -every author (the audit policy still bounds which writes carry a fact, and the CommitRequest submitter -is not recoverable from state alone). Both modes deliver a continuously updated Git mirror of desired -cluster state. - -## Keep / cut / reshape inventory - -What the big-bang touches, by package. Roughly **~5–6k LOC deleted**, replaced by **~1–1.5k** of -raw-watch ingestion plus a small attribution index and RV-resume cursor. - -| Package | Verdict | Notes | -|---|---|---| -| [internal/git](../../internal/git/) | **KEEP** | The whole downstream writer is source-agnostic. Only `open_window.go` (committer/SA/unknown buckets + grace) and `commit_request_attach*.go` (attribution optional) change. | -| [internal/manifestanalyzer](../../internal/manifestanalyzer/), [internal/manifestreport](../../internal/manifestreport/) | **KEEP** | Runtime manifest planning/diff/report. Source-agnostic. | -| [internal/typeset](../../internal/typeset/) | **KEEP, promote** | Followability/registry becomes the home of the relevance filter ("controller-owned → don't mirror"). | -| [internal/sanitize](../../internal/sanitize/) | **KEEP, promote** | Now mandatory on the hot path before every diff. | -| [internal/controller](../../internal/controller/) | **KEEP, reshape** | GitTarget reconcile opens watches instead of consuming a splice; CommitRequest flips fail-closed → finalize-as-committer. | -| [internal/reconcile](../../internal/reconcile/) | **KEEP** | `GitTargetEventStream` is the per-GitTarget seam the design leans on. | -| [internal/watch](../../internal/watch/) | **KEEP core, gut LIST/checkpoint half** | Keep discovery/catalog, `event_router`, `type_lifecycle`, `scope_resolve`, `watched_type_table`. Promote `watch_state.go`/`watch_compare.go` to the real path. Cut the `materialization.go` phase machine, `audit_tail.go`, `splice_snapshot.go`, `target_type_watermark.go`, and the heavy Redis objects-snapshot write in `type_objects_mirror.go` — but **keep that file's `sendInitialEvents` watch as the replay/resume path**, swapping the snapshot write for a tiny `(GVR,scope)→RV` cursor. | -| [internal/queue](../../internal/queue/) | **CUT ~90%** | Delete `redis_bytype_queue`, `redis_type_splice`, `redis_objects_snapshot`, `redis_audit_queue`, `redis_watch_splice`, `redis_watch_stream`, `subresource_translate`. Keep only the attribution-fact pieces (`commitrequest_author` folds into the new index). | -| internal/gate | **CUT** | Demand-gating existed to bound the per-type mirror. No mirror, no gate. | -| [internal/webhook](../../internal/webhook/) | **SHRINK hard** | `audit_joiner.go` (body-joining) → **gone**. `audit_handler.go` → shrinks to fact extraction → index. `admission_allow_handler.go` stays as the future-policy seam. | -| [internal/auditutil](../../internal/auditutil/) | **SHRINK** | Keep only what feeds attribution facts. | -| giteaclient, ssh, sshsig, telemetry, types, rulestore, mutationlab | **KEEP** | Provider/credential/signing/metrics/test infra, all source-agnostic. | - -## Change plan (big-bang, deletion-forward) - -Done as a single replacement on a branch; mostly deletion, so it should be quick relative to its -footprint. Validation gate is the full e2e suite green (`task lint` → `task test` → `task test-e2e`), -with the watch-vs-audit object-set diff used as the parity check during bring-up. - -**Stage 1 — Raw-watch ingestion as the GitTarget event source.** -Build a per-GitTarget watch manager: on GitTarget reconcile, resolve claimed ∩ followable -`(GVR, scope)` and open one raw watch per `(GVR, scope)` with `sendInitialEvents=true`. Persist -`(GVR, scope) → last RV` to Redis (when present) and resume from it; on `410`/disconnect, re-open with a -fresh replay. Wire each event through the relevance filter (sanitize → followability → no-op diff) into -the existing `GitTargetEventStream` → BranchWorker. **No timer sweep**; deletes are reconciled by the -Mode-A delta resume or the Mode-B mark-and-sweep on replay (see -[Resume and recovery](#resume-and-recovery)). This replaces the materialization phase machine + audit -tail + splice as the desired-set source. *Net: new code is small; it plugs into existing seams, and -`type_objects_mirror.go` already does the `sendInitialEvents` watch.* - -**Stage 2 — Optional attribution.** -When attribution is enabled: extract minimal audit facts in the (shrunk) audit handler → Redis index -with TTL; add the conservative resolver with the bounded grace window. When attribution is disabled: -skip the audit handler and the attribution index entirely (Redis still backs the resume cursors); -commits are committer-authored. Flip `CommitRequest` to finalize-as-committer without attribution. - -**Stage 3 — Demolition.** -Delete the audit-as-correctness machinery now that nothing imports it: `internal/queue` (minus the -moved attribution bits), `internal/gate`, `webhook/audit_joiner.go`, the materialization phase machine, -`audit_tail.go`, `splice_snapshot.go`, `target_type_watermark.go`, the objects snapshot/checkpoint, and -the late-lane/divert/nudge code. Shrink `audit_handler.go` to fact extraction. - -**Stage 4 — Config, wiring, docs.** -Note: the shipped behavior later relaxed this. Redis is now **optional** in `cmd/main.go` (an empty -`--redis-addr` is valid and runs configured-author with cold-replay; the readiness gate only applies -when Redis is configured). The audit webhook / attribution remains optional via -`--author-attribution=false`; when Redis *is* configured it still backs the resume cursors. Remove the dead audit-stream flags -(`--audit-redis-max-len`, `--audit-bytype-*`, `--watch-state-stream`, body TTL/wait, etc.). Update the -Helm chart and `config/` so the audit webhook is opt-out while Redis stays a hard dependency. Rewrite -`docs/architecture.md` to the watch-first model and document the two operating modes and the -history-granularity guarantee. - -## Reliability rules (non-negotiable) - -1. Audit attribution must never be required for object correctness, and Redis durability must never be - a correctness input — a missing or stale resume cursor degrades to a full Mode-B replay, never to - wrong state. (Redis is still a required dependency; it just isn't what guarantees correctness.) -2. A watch event with no confident audit match must still write state (as committer). -3. A failed, rejected, or dry-run audit fact must never create state. -4. Conflicting attribution facts produce committer, not "first wins." -5. A later audit arrival must not rewrite a commit already shipped as committer. -6. A resume replay's mark-and-sweep is authoritative only after it reaches `initial-events-end`; a - partial replay must not be treated as complete current state. Deletes missed while not watching are - reconciled by the Mode-A delta resume or the Mode-B mark-and-sweep — never silently dropped. -7. Every attribution outcome carries a machine-readable reason code. -8. Unknown-author rate is visible by GitTarget, GVR, verb/event type, and reason. -9. The relevance filter is observable: how many watch events were dropped as no-op/noise, by GVR. +# Watch-first ingestion architecture — retired design record + +> **Historical context only.** The current implementation contract is +> [`../architecture.md`](../architecture.md). This short record replaces the superseded rollout plan that +> previously occupied this path, so older links remain useful without presenting abandoned design work as live +> behaviour. + +The shipped model is watch-first: one raw watch per `(GitTarget, GVR, scope)` supplies object state. A +watch’s initial replay and any recovery replay establish the desired state; Git is the persisted mirror. +Audit events never create or repair object state. They are optional evidence used only to name a live +mutation’s Git author. + +## Current invariants + +- A branch worker serializes writes. A commit window belongs to one GitTarget and either one named actor or + an unnamed author state. +- With attribution disabled, and for replay/resync writes, Git uses the configured committer as author. +- With attribution enabled, a strong audit match names the authenticated actor. A weak, conflicting, late, + or missing fact produces `unknown (attribution unresolved) `. + A late fact never rewrites an already-created commit. +- Redis is optional in configured-author mode. It is required when audit-backed attribution is enabled and + can also retain watch-resume cursors. +- A `CommitRequest` captures its command submitter at validating admission when that optional Redis-backed + path is enabled. A request without a captured actor claims no actor; the matching window, not the request, + determines the final Git author. + +For the live details, use the [watch ingestion and reconcile](../architecture.md#watch-ingestion-and-reconcile), +[attribution](../architecture.md#attribution-audit-names-the-author-never-the-state), and +[CommitRequest finalize](../architecture.md#commitrequest-finalize) sections. ## Metrics -| Metric | Why | -|---|---| -| `gitopsreverser_watch_events_total{gvr,type,outcome}` | watch volume and drops | -| `gitopsreverser_watch_events_filtered_total{gvr,reason}` | relevance-filter behavior (no-op/status/runtime) | -| `gitopsreverser_watch_restarts_total{gvr,scope,reason}` | watch stability / `410 Gone` pressure (replay reconnects) | -| `gitopsreverser_watch_replay_seconds{gvr,scope}` | time to reach `initial-events-end` on (re)connect — resume cost | -| `gitopsreverser_attribution_total{result,reason,gvr}` | exact-user / exact-sa / weak / conflict / absent / expired | -| `gitopsreverser_attribution_wait_seconds{result}` | grace-window latency cost | - -## Decision table - -| Question | Answer | -|---|---| -| Source of object state? | WATCH with `sendInitialEvents` replay, per GitTarget. No separate LIST/checkpoint. | -| Is audit ever required for committing state? | No. It only changes the author. | -| Is Redis required? | Yes — a hard dependency in every mode. It holds each GitTarget's RV resume cursors (and, with attribution on, the audit facts) and underpins HA / the durable branch-worker queue. It is not a *correctness* input: a missing cursor just forces a full Mode-B replay. | -| Watch ownership granularity? | Per GitTarget. HA = assign GitTargets to replicas; failover resumes from the Redis RV cursor (or replays). | -| Periodic LIST / drift sweep? | No *timer* sweep. Deletes are reconciled by the Mode-A delta resume (exact RV) or a **mark-and-sweep on replay** (Mode B); the sweep fires on watch re-establishment, never hourly. | -| Dedup watches across GitTargets? | No — one watch per `(GitTarget, GVR, scope)`, because resume cursors and mark-and-sweep are per-GitTarget. Sharing the transport is an explicit future enhancement, deferred. | -| Guess an author from timing alone? | No. Weak matches commit as committer. | -| Does watch-first preserve per-mutation history? | No across gaps — it collapses to current state. State mirror with opportunistic history. | -| Migration shape? | Big-bang: remove the audit-as-correctness pipeline, don't keep it in parallel. | -| What gets harder? | The relevance filter (now product code on the hot path) and honest, deterministic attribution. | - -## Bottom line - -Watch is the better source for persisted object state, and the audit-as-correctness pipeline carries -the most incidental complexity (ordering, late-lane, demand-gating, the materialization phase machine). -Making watch the only state source per GitTarget deletes that complexity, makes the audit webhook -optional (Redis stays required for resume cursors), and turns shallow/aggregated audit events from -correctness problems into attribution limitations. - -Two prices are accepted in exchange: the audit *policy's* relevance filtering moves into product code -on the hot path, and authorship becomes probabilistic with per-observation (not per-mutation) history -granularity. The product stays honest about both — name a user or service account when the evidence is -strong, otherwise commit as the committer and say why, and filter controller churn deliberately rather -than relying on a cluster's audit policy to do it. +Metric names and operational interpretation are maintained in +[`../interpreting-metrics.md`](../interpreting-metrics.md). The telemetry design backlog is +[`../design/metrics-observability-plan.md`](../design/metrics-observability-plan.md); a planned metric is not +evidence that the runtime currently emits it. diff --git a/docs/future/design-commit-request-phase-2.md b/docs/future/design-commit-request-phase-2.md deleted file mode 100644 index cb69c0f5..00000000 --- a/docs/future/design-commit-request-phase-2.md +++ /dev/null @@ -1,99 +0,0 @@ -# Phase 2: `CommitRequest` — deferred work - -> Status: proposed. Follow-up to [design-commit-request-api.md](../spec/commitrequest-design.md). -> Date: 2026-05-18 - -The first cut of `CommitRequest` is intentionally minimal. This doc tracks what was deliberately -left out, with enough analysis that phase 2 does not start cold. The object-lifecycle question is -the largest gap and is treated first. - -## 1. Object lifecycle — when are `CommitRequest`s deleted? - -This is the biggest open question. Today **nothing deletes `CommitRequest` objects.** Every save -leaves a permanent object in the namespace. With `generateName` and a frontend that saves often, -a namespace accumulates them without bound. - -There are two distinct populations, and they need different answers. - -### a. Terminal objects (`Committed` / `NoOpenWindow` / `Failed`) - -These have done their job. They are useful only as a short-lived receipt the frontend reads once. -Options: - -- **TTL-after-finished controller**, modeled on Job's `ttlSecondsAfterFinished`. A controller - deletes terminal objects a configurable interval after `status.observedTime`. Owns the clock - server-side and survives a frontend that never comes back. Recommended. -- **Frontend deletes after reading status.** Pushes the clock to the client. Clean when the - client is well-behaved; leaks on every crashed or forgetful client. Not sufficient on its own. -- **Owner references.** GC would cascade if a save were owned by some parent object — but there - is no natural owner here, so this does not really apply. - -### b. Objects stuck in `WaitingForAuditEvent` - -This is the subtle one. If the audit pipeline is degraded or the audit event is dropped, the -object **never reaches a terminal phase** — so a TTL keyed on "terminal + `observedTime`" will -never collect it. These objects are both a leak *and* a silent failure: the user's save did -nothing and nothing says so. - -A TTL-after-finished controller must therefore be paired with a **max-age sweep**: any -`CommitRequest` older than some bound that is still `WaitingForAuditEvent` should be transitioned -to `Failed` ("audit event never observed") and then become eligible for normal terminal cleanup. -The bound must be safely longer than worst-case audit-pipeline lag. - -### Decisions phase 2 must make - -- Who owns the delete (a TTL controller — recommended) and the default TTL. -- The separate max-age for stuck `WaitingForAuditEvent`, and whether it flips the object to - `Failed` or just deletes it. Flipping to `Failed` is preferable — it stays observable. -- Whether the TTL is global (a Helm value) or per-object (a `spec.ttlSecondsAfterFinished`-style - field), or both. -- Whether the controller emits a metric/event when it reaps a stuck `WaitingForAuditEvent` object, - so the underlying audit-pipeline problem is visible (see §4). - -## 2. Retry for transient finalize failures - -Today any finalize error — including the transient `ErrFinalizeQueueFull` — becomes a terminal -`Failed`, and the caller's custom message is lost (the silence timer still commits the edits later -with a generated message). For a saturated-queue case specifically, a bounded retry before giving -up would preserve the custom message. - -This needs care: the audit message is ACKed exactly once with no redelivery, so a retry must -happen within the consumer's processing of that one event, or via an explicit requeue mechanism. -Worth doing only if queue-full turns out to be common in practice — gate it on the metrics from -§4. - -## 3. Multi-target "save everything I edited" - -`spec.gitTargetRef` is required precisely so `status` stays flat (one SHA, no list). A bare -"finalize every open window I have in this namespace" form is a plausible UX, but it makes -`status` a list and the matching logic non-trivial. Deferred until there is a concrete need. - -## 4. Observability - -There are no metrics specific to `CommitRequest` yet. Phase 2 should add at least: - -- a counter of outcomes by phase (`committed` / `no_open_window` / `failed`); -- a counter (or gauge) of objects reaped while still `WaitingForAuditEvent` — the - audit-pipeline-degraded signal from §1b; -- finalize latency (object creation → terminal phase). - -## 5. Helm: ship an audit-policy fragment - -The consumer only needs `Metadata`-level audit on `commitrequests` `create`. The repo's e2e -policy captures this via a catch-all `create` rule, but operators applying their own restrictive -audit policy have to derive the rule themselves. The chart should ship a recommended fragment, or -document it prominently, so `CommitRequest` works on a cluster with a narrow audit policy. - -## 6. Smaller items - -- **Cluster-scoped use.** `CommitRequest` is namespaced only. No current need for a cluster-scoped - variant; noted for completeness. - -## Suggested order - -1. **Lifecycle** — the TTL-after-finished controller **and** the stuck-`WaitingForAuditEvent` - max-age sweep (§1). This is the only item that is a correctness / operational problem rather - than a nicety. -2. **Outcome metrics** (§4) — needed to even know whether the retry work in §2 is worth doing. -3. **Audit-policy fragment** in the Helm chart (§5). -4. Revisit **retry** (§2) and **multi-target** (§3) only if real usage asks for them. diff --git a/docs/future/idea-unify-pending-write-kinds.md b/docs/future/idea-unify-pending-write-kinds.md index e362bbd1..1d20374a 100644 --- a/docs/future/idea-unify-pending-write-kinds.md +++ b/docs/future/idea-unify-pending-write-kinds.md @@ -23,12 +23,11 @@ The four use cases in [commit-window-refactor.md](../spec/commit-window-refactor 1. **Burst collapse.** Per-event-only behavior (atomic is one commit by definition). Unification neither helps nor breaks this. -2. **Honest authorship.** Reconcile snapshots are operator-authored; audit-driven - commits carry the audit user. Today this is enforced by *type*: the atomic path - never reads `event.UserInfo`. Under unification it becomes a *convention*: the - git layer falls back to committer-as-author when `event.UserInfo.Username == ""`, - so producers must not populate `UserInfo` for snapshot batches. Same observable - result, weaker compile-time guarantee. +2. **Honest authorship.** Reconcile snapshots are configured-author; live commits carry + either a resolved audit actor or the explicit unresolved identity. Today this is enforced by + *type*: the atomic path is `AttributionNotAttempted` and never reads `event.UserInfo`. Under + unification it becomes a *convention*: producers must retain the correct + `AttributionOutcome` for snapshot batches. Same observable result, weaker compile-time guarantee. 3. **Target isolation.** Single-target on both paths today. Unaffected. 4. **Safe replay.** Both paths carry resolved metadata. Unaffected. @@ -40,7 +39,7 @@ that one is comfortable being a convention rather than a type-system invariant. | Concern | Per-event / grouped | Atomic | Survives unification? | |---|---|---|---| -| Authorship source | `event.UserInfo.Username` | Operator (request-level, ignores `UserInfo`) | Yes, via "empty `UserInfo` → committer-as-author" fall-through at the git layer | +| Authorship source | resolved actor or explicit outcome | Configured author (`AttributionNotAttempted`) | Yes, by preserving `AttributionOutcome` rather than relying on an empty username | | Commit message | Template render | Caller-provided string | Yes, via `CommitMessage` non-empty as the dispatch signal | | Finalize timing | Author/target change, silence, byte cap, shutdown | Request boundary | Yes, by adding "request has `CommitMessage` set" as a finalize trigger | | Byte-cap behavior | Mid-stream finalize is fine | Must not split one snapshot across commits | Yes, via the rule "finalize the *next* organic window early; don't actively split a request" | diff --git a/docs/interpreting-metrics.md b/docs/interpreting-metrics.md index bc9d9907..a0eb34b8 100644 --- a/docs/interpreting-metrics.md +++ b/docs/interpreting-metrics.md @@ -94,10 +94,9 @@ reconcile/resync commits and configured-author mode use `committer`. the commit carries the `unknown (attribution unresolved)` author instead of a person. It is deliberately not folded into `user` (which would make a lost actor look like a named one, so a degrading attribution path would read as an improving one) nor into `committer` (which would -hide it among legitimately unattributed reconcile writes). A non-zero and growing `unresolved` -share is a real defect, not noise — see -[`gitopsreverser_attribution_resolutions_total`](#audit-attribution-optional) and -[docs/debug/attribution-loss.md](debug/attribution-loss.md). The namespace/name keys are +hide it among legitimately unattributed reconcile writes). For a live mutation that should be +attributable, a non-zero and growing `unresolved` share means the audit-attribution configuration +or delivery path needs investigation. The namespace/name keys are **prefixed on purpose** — a Prometheus pod scrape with `honor_labels=false` overwrites a bare `namespace` attribute with the scraping pod's namespace, so a per-provider `namespace` selector would silently match nothing. The same reasoning applies to @@ -251,10 +250,10 @@ histogram_quantile(0.95, ``` **Does audit actually attribute watch events?** Match coverage is the share of resolutions that named an -actor (human or service account) rather than falling back to the committer: +actor (human or service account) rather than producing an explicit unresolved author: ```promql -sum(rate(gitopsreverser_attribution_resolutions_total{result=~"exact_.*"}[5m])) +sum(rate(gitopsreverser_attribution_resolutions_total{result=~"exact_.*|weak"}[5m])) / sum(rate(gitopsreverser_attribution_resolutions_total[5m])) ``` @@ -266,21 +265,20 @@ sum(rate(gitopsreverser_attribution_resolutions_total[5m])) | `exact_user` | Exact UID+resourceVersion match for a human user. | | `exact_serviceaccount` | Exact UID+resourceVersion match for a named service account. | | `weak` | Non-exact match, such as UID-only or RV-only. | -| `conflict` | Multiple authors wrote facts for the selected join key. | -| `expired` | A fact existed but aged out before the watch event joined it. | -| `absent` | No fact or expiry evidence matched. | +| `absent` | No usable fact matched before the grace window elapsed. The resulting live commit is authored as `unknown (attribution unresolved)`. | **Is the grace window paying for itself?** Misses waiting near the configured grace window mean the operator is delaying commits without finding facts: ```promql histogram_quantile(0.95, - sum by (le, result) ( - rate(gitopsreverser_attribution_resolution_wait_seconds_bucket{result=~"absent|expired"}[5m]))) + sum by (le) ( + rate(gitopsreverser_attribution_resolution_wait_seconds_bucket{result="absent"}[5m]))) ``` -**Is the Redis fact index healthy?** Facts should be written and later matched; a high `late` or -`expired_unmatched` rate points at timing mismatch between audit and watch: +**Is the Redis fact index healthy?** Facts should be written and later matched; a high rate of +`op="written"` alongside `result="absent"` points at a timing, key, or source-identity mismatch +between audit and watch: ```promql sum by (op) (rate(gitopsreverser_attribution_fact_events_total[5m])) diff --git a/docs/spec/README.md b/docs/spec/README.md index 58ec3b38..cef1ba89 100644 --- a/docs/spec/README.md +++ b/docs/spec/README.md @@ -37,6 +37,6 @@ If you change one of these behaviours, change the document in the same commit. | [`sops-single-file-no-multidoc.md`](sops-single-file-no-multidoc.md) | one encrypted file is one document | | [`scale-subresource-audit-rehydration.md`](scale-subresource-audit-rehydration.md) | `/scale` → bounded field patch; every other subresource ignored | | [`commit-window-refactor.md`](commit-window-refactor.md) | one grouped commit = one (author, GitTarget) | -| [`commitrequest-multi-finalize-design.md`](commitrequest-multi-finalize-design.md) | why `MaxConcurrentReconciles=1` on the CommitRequest controller | +| [`commitrequest-design.md`](commitrequest-design.md) | how a request binds to a same-actor commit window and reports its outcome | | [`gittarget-isolation-on-rule-change.md`](gittarget-isolation-on-rule-change.md) | a rule change on target A never touches target B | | [`audit-readiness-probe-plan.md`](audit-readiness-probe-plan.md) | why liveness must never depend on Redis | diff --git a/docs/spec/commitrequest-admission-authorship.md b/docs/spec/commitrequest-admission-authorship.md index 838a8f25..fc869a3b 100644 --- a/docs/spec/commitrequest-admission-authorship.md +++ b/docs/spec/commitrequest-admission-authorship.md @@ -1,613 +1,65 @@ ---- -title: CommitRequest authorship from admission — a command, captured at the source -status: implemented (investigate branch) -date: 2026-06-29 -related: - - ../finished/design-commit-request-api.md - - ../finished/commitrequest-attribution-divert-reliability.md - - watch-first-ingestion-architecture.md - - watch-event-ordering-and-attribution-grace.md - - reconcile-triggering.md - - redis-key-schema-v3.md ---- - # CommitRequest authorship from admission -> **spec** — current behaviour. The code depends on this document; change one, change the other. Index: [`../INDEX.md`](../INDEX.md) - -> **Thesis.** A `CommitRequest` is a **command** ("save now, as me"), not a piece of -> mirrored state. Its author should be captured where the command is issued — a -> **validating admission webhook** dedicated to our own command kinds (the -> *validate-operator-types* webhook) — into its **own Redis corner**, and read back by the -> controller with **no wait**. This is the opposite of how we attribute mirrored -> resources (audit facts joined to watch events), and deliberately so: the two have -> different provenance and must not share a store. This design **replaces** the -> audit-sourced CommitRequest attribution rather than layering on top of it. - -This is the forward-facing companion to -[commitrequest-attribution-divert-reliability.md](commitrequest-admission-authorship.md), -which adopted "Option C" (capture the author as a keyed fact at audit ingestion). -Option C was the right *shape* — a divert-immune keyed fact, not an ordered-stream -scan — but it kept the **wrong source** (the audit event) and therefore kept a -**wait**. Watch-first has since severed the only other thing the audit event did for -a CommitRequest (timing — now the close-delay collect window), so the audit event's -last remaining job here is to carry one string. We can get that string earlier, more -reliably, and without the audit webhook at all. +> **spec** — current behaviour. The code depends on this document; change one, change the other. +> Index: [`../INDEX.md`](../INDEX.md) ---- +A `CommitRequest` is a command to close an existing GitTarget commit window. Its submitter is captured +by the `/validate-operator-types` validating admission webhook, while mirrored-resource attribution is +separately derived from kube-apiserver audit facts. The two paths answer different questions and must not +be conflated. -## 1. Two kinds of authorship, two provenances +## Contract -We answer two different questions with two different mechanisms. Keeping them -separate is the whole point. - -| | **Mirrored-resource attribution** | **Command authorship** | +| Concern | CommitRequest submitter | Mirrored-resource author | |---|---|---| -| Question | "Who performed mutation X that *persisted* at RV N?" | "Who *issued* this command?" | -| Source | Audit event (`ResponseComplete`, 2xx, RV changed) | Admission request `userInfo` | -| Timing vs persist | **After** persist (proof of persistence) | **Before** persist (an assertion) | -| What makes it trustworthy | The audit accept gate ([`classifyAuditIngress`](../../internal/webhook/audit_handler.go)) | The controller only finalizes **persisted** objects (the informer never sees an un-persisted command) | -| Join | identity **+ resourceVersion**, matched within a grace window | identity 1:1 (`uid`), no join, no window | -| Store | `AttributionIndex` (`AuthorFact`, conflict markers, RV/auditID) | dedicated `CommandAuthorStore` under `author:v1:command` (this doc) | -| Lifetime | the watch-event join grace | admission → finalize (effectively immediate, §2) | - -The reliability argument the project built the audit path on — *admission sees -attempts, not persistence* ([mutation-capture-lab-design.md](mutation-capture-lab-design.md), -rows 11–13) — **still holds for mirrored state and is unchanged.** It does not bite -a command, because we never write the command object into git. We extract one thing -from it (the author), and whether that extraction is ever *used* is gated downstream -by persistence: the controller reconciles off the informer, which only ever delivers -objects that actually landed in etcd. The persistence guarantee is not lost — it is -**relocated** from the audit event to the controller's watch. - -### Why not reuse the attribution index (the "don't pretend it came from audit" rule) - -Folding the admission assertion into the audit `AuthorFact` -([attribution_index.go:88-102](../../internal/queue/attribution_index.go#L88-L102)) -would silently inherit semantics that are wrong for a command: - -- **Conflict detection.** `storeFactKey` writes a `Conflict` marker when two users - hit the same key ([attribution_index.go:457-473](../../internal/queue/attribution_index.go#L457-L473)). - A command has exactly one creator; that machinery is dead weight at best and a - corruption vector at worst. -- **RV / auditID / stageTimestamp.** All empty for an admission capture — the fields - exist to disambiguate a *post-persist join*, which we do not do. -- **TTL semantics.** The fact TTL is tuned for the watch-join grace; the command - record's TTL is purely for cleanup (§3). -- **Provenance blur.** A reader could no longer tell "proven persisted, by audit" - from "asserted at admission, persistence enforced by the watch." Those are - different trust models and should be legible as different. - -So: a **separate store, separate author subnamespace, separate record type.** It is -not cosmetic separation — it keeps each path's invariant clean. The Redis naming plan -is `author:v1:audit` for persisted-resource facts and `author:v1:command` for -admission-captured commands ([redis-key-schema-v3.md](../finished/redis-key-schema-v3.md)). - ---- - -## 2. The authorship invariant (why there is no wait) - -> **By the time a command object is visible to anything — `kubectl get`, the informer, -> the controller — its author has already been recorded.** - -This falls out of admission ordering: - -1. A client `CREATE`s a `CommitRequest`. -2. The apiserver resolves `generateName → name`, assigns `metadata.uid`, runs - mutating admission, then runs **validating** admission — our validate-operator-types handler. -3. Our handler **synchronously writes** `{uid → author}` to Redis and returns - `Allowed`. The write completes *before* `Handle` returns. -4. *Only then* does the apiserver persist the object to etcd and return to the client. -5. *Only after persist* does the informer deliver it and the controller reconcile. - -Step 3 strictly precedes steps 4–5. So when the controller first sees the object, the -record is **present-or-never**: present if the webhook ran, never if it did not (the -validate-operator-types webhook is not configured, or a Redis write failed under -`failurePolicy: Ignore`). **Waiting cannot help** — there is no asynchronous arrival -to wait for, unlike an audit event that trickles in seconds after persist. This is the -precise reason the 60 s `commitRequestAttributionTimeout` wait -([commitrequest_controller.go:77-82](../../internal/controller/commitrequest_controller.go#L77-L82)) -disappears: it was an artifact of the audit event's *post-persist* timing, and that -timing is gone. - -Contrast with audit, where the order is inverted (persist → audit event → maybe -seconds later → fact in Redis), which is exactly why the audit-sourced controller -*had* to poll and fall back on a timeout. - -**This does not weaken the "prior edits already landed" guarantee.** That was an -audit-stream-ordering property in the audit-first era; in watch-first it is the -**close-delay collect window** (`finalizeAt = receipt + CloseDelaySeconds`, -[commit_request_attach.go:87-90](../../internal/git/commit_request_attach.go#L87-L90)), -which is independent of where the author string comes from. Moving authorship to -admission touches the *who*, never the *when*. - ---- - -## 3. The dedicated Redis corner - -A small store hung off the always-present `RedisStore` -([redis_store.go](../../internal/queue/redis_store.go)), mirroring how -`AttributionIndex` and the watch-cursor store share the one connection but own -disjoint key namespaces. It lives under the same top-level `author` Redis domain as -audit-sourced authorship, but in the separate `command` subfamily because its -provenance and lookup semantics are different. It is generic over command kinds -(keyed by `uid` alone, which is globally unique) so a future command CRD reuses it -unchanged. - -```go -// internal/queue/command_author_store.go - -// commandAuthorKeySuffix namespaces the keys this store owns. It shares the top-level -// author domain with audit-sourced resource facts, but the command subfamily has a -// different provenance and no grace-window join. -const commandAuthorKeySuffix = ":author:v1:command:" - -// commandAuthorRecordTTL bounds a captured authorship record. It is NOT tuned to cover -// any wait: by the authorship invariant (§2) the record is written before the object -// exists, and the controller reads it on the first reconcile after persist — typically -// sub-second. The TTL exists ONLY so an orphan record (a command object deleted before -// its reconcile) self-cleans. It is a fixed internal constant, deliberately not a flag: -// there is nothing to tune. An hour is generous headroom for a slow reconcile backlog -// or a leader failover. -const commandAuthorRecordTTL = time.Hour - -// CommandAuthor is the minimal authorship captured at admission for one command object. -// It carries only what a git commit author needs — no RV, no auditID, no conflict bit: -// this is a 1:1 command capture, not a post-persist join. -type CommandAuthor struct { - Author string `json:"author"` - DisplayName string `json:"displayName,omitempty"` - Email string `json:"email,omitempty"` - RequestedAt string `json:"requestedAt,omitempty"` // RFC3339Nano, for lag metrics/debug -} - -// CommandAuthorStore records and reads command authorship. It shares RedisStore's -// connection but is wired whenever the validate-operator-types webhook is enabled — -// independent of --author-attribution, which only governs mirrored-resource attribution. -type CommandAuthorStore struct { - client *redis.Client - ttl time.Duration -} - -// RecordCommandAuthor is the admission-side write: capture the authenticated submitter -// the instant a command CREATE is admitted, before it persists. Last-write-wins (a -// CREATE fires admission once; a retried admission re-asserts the same user). -func (s *CommandAuthorStore) RecordCommandAuthor( - ctx context.Context, uid types.UID, author CommandAuthor, -) error { - raw, err := json.Marshal(author) - if err != nil { - return fmt.Errorf("marshal command author: %w", err) - } - return s.client.Set(ctx, s.key(uid), raw, s.ttl).Err() -} - -// LookupCommandAuthor is the controller-side read, keyed by the persisted object's UID. -// ok=false means no record was captured — the validate-operator-types webhook is not -// configured (or a best-effort write missed) — and the controller finalizes as the -// committer, immediately, with AuthorAttributed=False (§5). -func (s *CommandAuthorStore) LookupCommandAuthor( - ctx context.Context, uid types.UID, -) (CommandAuthor, bool) { - raw, err := s.client.Get(ctx, s.key(uid)).Bytes() - if err != nil { - return CommandAuthor{}, false - } - var a CommandAuthor - if json.Unmarshal(raw, &a) != nil || a.Author == "" { - return CommandAuthor{}, false - } - return a, true -} - -// key identifies the command by UID alone — globally unique, like the watch cursor key, -// so namespace/name/kind would be redundant (kept out for a tight key). -func (s *CommandAuthorStore) key(uid types.UID) string { - return keyPrefix + commandAuthorKeySuffix + escapeKeyField(string(uid)) -} -``` - -And the builder on `RedisStore`, alongside `AttributionIndex(...)`: - -```go -// CommandAuthorStore builds the command-authorship store on this connection. Wire it -// when the validate-operator-types webhook is enabled; it does not depend on attribution. -func (s *RedisStore) CommandAuthorStore() *CommandAuthorStore { - return &CommandAuthorStore{client: s.client, ttl: commandAuthorRecordTTL} -} -``` - -> **Keying by UID only.** The controller reads by the persisted object's `uid`, which -> admission already assigned. `generateName` is a non-issue: the server-assigned name -> *and* uid are both present in `request.Object` by the time validating admission runs, -> so we never have to recover a name from a response body the way the audit path did. - ---- - -## 4. The validate-operator-types admission handler - -One handler on its own path, `/validate-operator-types`, that recognizes **our own command -kinds** and captures the submitter of each. It is not CommitRequest-specific by -construction — it dispatches on a small registry of command resources, so a future -command CRD is one table entry plus one webhook rule, not a new handler. It replaces -the no-op [`AdmissionAllowHandler`](../../internal/webhook/admission_allow_handler.go) -*for this purpose*; the catch-all `*` observer in -[validating-webhook.yaml](../../config/webhook/validating-webhook.yaml) is a separate, -future policy extension point and stays as-is. - -```go -// internal/webhook/internal_commands_handler.go - -const ValidateOperatorTypesPath = "/validate-operator-types" - -// commandKinds is the registry of our own command CRDs whose submitter we capture at -// admission. Adding a command kind is one entry here plus one rule in the webhook -// config — the handler body does not change. -var commandKinds = map[metav1.GroupResource]struct{}{ - {Group: "configbutler.ai", Resource: "commitrequests"}: {}, - // future: {Group: "configbutler.ai", Resource: ""}: {}, -} - -// ValidateOperatorTypesHandler captures the authenticated submitter of one of our command -// objects into the CommandAuthorStore and always allows. It is pure observation with a -// single side effect (a Redis upsert); it never rejects. -type ValidateOperatorTypesHandler struct { - Store *queue.CommandAuthorStore - Decoder admission.Decoder -} - -func (h *ValidateOperatorTypesHandler) Handle(ctx context.Context, req admission.Request) admission.Response { - gr := metav1.GroupResource{Group: req.Resource.Group, Resource: req.Resource.Resource} - if _, ok := commandKinds[gr]; !ok { - return admission.Allowed("not a command kind") // belt-and-suspenders; rules already scope us - } - // Dry-run never persists, so the controller will never read this record — and we - // declare sideEffects: NoneOnDryRun, so we must honor it. - if req.DryRun != nil && *req.DryRun { - return admission.Allowed("dry-run: not recorded") - } - - // metadata.uid identifies the object (not req.UID, which identifies the review). - obj := &metav1.PartialObjectMetadata{} - if err := h.Decoder.Decode(req, obj); err != nil { - // Cannot key the record without a uid; allow the command, fall back to committer. - log.FromContext(ctx).Error(err, "decode command object for authorship", "name", req.Name) - return admission.Allowed("undecodable: not recorded") - } - - author := queue.CommandAuthor{ - Author: req.UserInfo.Username, // effective (impersonated) user — apiserver resolved it - DisplayName: firstExtraValue(req.UserInfo.Extra, displayNameExtraKey), - Email: firstExtraValue(req.UserInfo.Extra, emailExtraKey), - RequestedAt: time.Now().UTC().Format(time.RFC3339Nano), - } - if author.Author == "" { - return admission.Allowed("no user to attribute") // anonymous never reaches a persisted CREATE - } - - // Synchronous, best-effort: the write completes before we return, so it is present - // before the object is visible (the authorship invariant, §2). A failure degrades to - // the committer — never block the command. - if err := h.Store.RecordCommandAuthor(ctx, obj.UID, author); err != nil { - log.FromContext(ctx).Error(err, "record command author; finalizing as committer", - "namespace", req.Namespace, "name", req.Name) - } - return admission.Allowed("author recorded") -} -``` +| Source | validating admission request `userInfo` | post-persist audit fact joined to watch event | +| Key | CommitRequest UID | source provider, GVR, UID, and resourceVersion | +| Timing | written before the object persists | waits up to `--author-attribution-grace` after the watch event | +| Miss | request claims no actor | watch commit has the explicit unresolved author | -The matching `ValidatingWebhookConfiguration` rule — **narrow, `Ignore`, and packaged -in the Helm chart** (this is product behavior now, not the e2e-only `*` observer): +The admission handler records the authenticated submitter in the Redis command-author store and always +allows the request. It is deliberately best-effort: a Redis failure or an unavailable webhook must not +reject a user's command. The controller reads the record when the persisted `CommitRequest` is first +reconciled. The result is **present-or-never**; waiting cannot create a record that admission did not write. -```yaml -webhooks: - - name: validate-operator-types.configbutler.ai - clientConfig: - service: { name: gitops-reverser-webhook, namespace: , path: /validate-operator-types, port: 9443 } - rules: - - operations: [CREATE] - apiGroups: [configbutler.ai] - apiVersions: [v1alpha3] - resources: [commitrequests] # one line per command kind - failurePolicy: Ignore # authorship is best-effort; never block the command - sideEffects: NoneOnDryRun # Redis write on real requests; nothing on dry-run - matchPolicy: Equivalent - timeoutSeconds: 2 - admissionReviewVersions: [v1] -``` +## Attach and final Git author -`failurePolicy: Ignore` here is correct for the opposite reason it is in the e2e -observer: not to avoid a bootstrap deadlock, but because a missed author capture must -degrade to the committer, never reject a user's command. The handler also returns -`Allowed` even when the Redis write errors, so the command's success never depends on -the failure policy — `Ignore` only covers the webhook being entirely unreachable. +`AuthorAttributed=True` means admission captured the command submitter. The request can attach only to an +open window with that same named actor and GitTarget. ---- +`AuthorAttributed=False` (`CommitterFallback`) means no admission record was available. It is not a +failure and does not mean the eventual Git author is necessarily the configured committer. The request +claims no actor and can attach only to an unnamed window. That window determines the actual Git author: -## 5. The controller, before and after +- configured-author mode or a replay/resync write: the configured committer; +- live attribution that ran but found no usable audit fact: + `unknown (attribution unresolved) `. -The state machine loses its first wait entirely. Author resolution becomes a single -synchronous lookup with two outcomes, both immediate and both settled. +The worker never closes another actor's window. If no matching window appears before the close delay +expires, the request ends successfully with `Ready=True`, `Pushed=False`, and `NoWindowInGrace` or +`WindowMismatch`. -**Before** ([commitrequest_controller.go:164-233](../../internal/controller/commitrequest_controller.go#L164-L233)): +## Conditions -``` -first sight ─▶ Attributed=Unknown, WaitingForAuditEvent - │ - ▼ attributeAuthor: poll LookupCommitRequestAuthor every 2s - ├─ hit ───────────────▶ attributionResolved - ├─ within 60s, miss ──▶ attributionPending → requeue 2s (THE WAIT) - └─ past 60s, miss ────▶ attributionTimedOut (Attributed=False) - ▼ - ATTACH + POLL (close-delay) ─▶ terminal -``` +`AuthorAttributed` is binary and settled on the first reconcile; it has no audit-wait or `Unknown` state. -**After:** - -``` -first sight ─▶ attributeAuthor: LookupCommandAuthor (one read) - ├─ hit ─▶ author named, AuthorAttributed=True (AttributedFromAdmission) - └─ miss ─▶ committer, AuthorAttributed=False (CommitterFallback) - ▼ (no Unknown phase, no requeue-for-author) - WaitingForCloseDelay ─▶ ATTACH + POLL ─▶ terminal -``` - -Concretely, `attributeAuthor` collapses to: - -```go -func (r *CommitRequestReconciler) attributeAuthor( - ctx context.Context, cr *configbutleraiv1alpha3.CommitRequest, -) (queue.CommandAuthor, commitRequestAttribution) { - if r.AuthorLookup == nil { // validate-operator-types webhook disabled → configured-author - return queue.CommandAuthor{}, attributionCommitter - } - if a, ok := r.AuthorLookup.LookupCommandAuthor(ctx, cr.UID); ok { - return a, attributionFromAdmission - } - return queue.CommandAuthor{}, attributionCommitter // present-or-never: a miss is committer, now -} -``` - -The reconcile body drops the `waitForAudit` branch and its -`RequeueAfter: commitRequestAttributionRetryDelay` -([commitrequest_controller.go:168-171](../../internal/controller/commitrequest_controller.go#L168-L171)); -first-sight stamping goes straight to `WaitingForCloseDelay`. - -### Status: the `AuthorAttributed` condition - -Rename the domain condition `Attributed` → **`AuthorAttributed`** -([constants.go:59-63](../../internal/controller/constants.go#L59-L63)) and give it a -clean **binary, immediately-settled** meaning — no `Unknown`, no timeout state: - -| `AuthorAttributed` | Reason | Meaning | +| Condition | True | False | |---|---|---| -| `True` | `AttributedFromAdmission` | The submitter was captured at admission and named as the commit author. | -| `False` | `CommitterFallback` | No admission author record — the validate-operator-types webhook is not configured (or did not record one) — so the commit is authored by the configured committer. | - -`AuthorAttributed=False` is **not a failure** and does not affect `Ready`: the command -still commits successfully, just as the committer. The pairing `Ready=True` + -`AuthorAttributed=False` is the precise, honest signal a user reads as *"saved — but as -the bot, because author attribution isn't wired up here."* This is the status surface -for the edge case the whole design hinges on (record absent ⇒ webhook not configured ⇒ -committer), and it is why the absent case earns a real condition rather than being -hidden. - -Removed reasons from -[commitrequest_finalize.go:36-50](../../internal/controller/commitrequest_finalize.go#L36-L50): -`WaitingForAuditEvent` (and its `Attributed=Unknown` first-sight state), -`AttributedFromAuditEvent` (→ `AttributedFromAdmission`), `AuditEventNotObserved`, and -`AttributionNotRequired` (→ `CommitterFallback`). The `commitRequestAttribution` enum -drops `attributionPending`/`attributionTimedOut`, keeping `attributionFromAdmission` -and `attributionCommitter`. e2e specs asserting the old reasons or the -`WaitingForAuditEvent` phase must be updated — call this out in the PR. (v1alpha3 is -alpha, so the status-contract change is acceptable.) - ---- - -## 6. Wiring (`cmd/main.go`) - -Authorship capture is gated by the **validate-operator-types webhook**, not -`--author-attribution`: - -```go -// Command authorship is captured at admission and lives in its own Redis corner, -// independent of --author-attribution (which governs mirrored-resource attribution). -var commandAuthorStore *queue.CommandAuthorStore -if cfg.internalCommandsWebhookEnabled { - commandAuthorStore = redisStore.CommandAuthorStore() - mgr.GetWebhookServer().Register( - webhookhandler.ValidateOperatorTypesPath, - &ctrladmission.Webhook{Handler: &webhookhandler.ValidateOperatorTypesHandler{ - Store: commandAuthorStore, - Decoder: admission.NewDecoder(scheme), - }}, - ) -} -// ... -if err := (&controller.CommitRequestReconciler{ - // ... - AuthorLookup: commandAuthorStore, // nil when the webhook is disabled → configured-author, immediate -}).SetupWithManager(mgr); err != nil { /* ... */ } -``` - -The `AuthorLookup == nil` contract is preserved and *strengthened*: nil → configured-author; -non-nil + miss → configured-author **for that request** — both immediate, neither waits, -both surface `AuthorAttributed=False`. - -**HA.** Admission can hit any replica (it is a `Service`); the controller leader reads -from the shared Redis corner. The synchronous-write-before-persist invariant holds per -request regardless of which replica served admission, so a leader failover between -admission and finalize still finds the record. - -> **Flag naming.** The existing `--admission-webhook` flag and its `admission-webhook-*` -> cert flags ([cmd/main.go](../../cmd/main.go)) gate the e2e `*` observer today. Decide -> whether the validate-operator-types webhook rides the same admission server (one flag, one -> cert, one port — likely yes) or gets its own gate. Recommended: **one admission -> server**, both handlers registered on it, gated by the existing `--admission-webhook`; -> the validate-operator-types handler is always registered when that server is on. - ---- - -## 7. TLS — same story as the other certs - -The validate-operator-types webhook needs a serving cert exactly like the metrics endpoint -and the audit webhook. Follow the **established pattern**: full configuration for those -who need it, and a cert-manager easy-route shipped in the chart for everyone else. - -- **Full configuration (BYO).** Keep the existing serving-cert flags - (`--admission-webhook-cert-path` / `-cert-name` / `-cert-key`, - [cmd/main.go](../../cmd/main.go)) so an operator can mount any cert (their own PKI, a - different issuer, a rotated secret). The cert-watcher already hot-reloads on rotation, - as the audit and metrics certs do. -- **Easy route (chart + config).** The Helm chart and the `config/` kustomize overlay - ship the cert-manager resources that wire it up automatically, the same way the other - endpoints are templated: - - an `Issuer`/`Certificate` minting the admission serving cert into a Secret the - Deployment mounts; - - the `cert-manager.io/inject-ca-from` annotation on the - `ValidatingWebhookConfiguration` so the CA bundle is injected for you — the - config overlay already uses exactly this annotation - ([validating-webhook.yaml](../../config/webhook/validating-webhook.yaml)). - Gate the whole block on a chart value (e.g. `internalCommands.enabled`, - `internalCommands.certManager.enabled`) so a non-cert-manager user can opt into BYO - certs instead. See [audit-webhook-tls-design.md](../finished/audit-webhook-tls-design.md) for the - long-lived-CA + short-lived-serving-cert rotation model to mirror. - -This is the one genuinely new packaging surface (the chart "ships no webhook/cert -templates" today, per the header in -[validating-webhook.yaml](../../config/webhook/validating-webhook.yaml)). It is real, -but it is the standard operator-managed-webhook pattern and far less than the -apiserver-side wiring the audit webhook demands (§9). - ---- - -## 8. Edge cases and degradation - -| Case | Behavior | -|---|---| -| **Internal-commands webhook not configured** | `AuthorLookup == nil` (or record absent) → committer, immediate, **`AuthorAttributed=False (CommitterFallback)`**. The headline edge case: no record means the webhook isn't wired up; fall back to the committer and *say so* in status. | -| Redis write fails at admission | Handler logs, returns `Allowed`; controller finds no record → committer, `AuthorAttributed=False`. | -| Webhook pod unreachable | `failurePolicy: Ignore` admits the command; no record → committer, `AuthorAttributed=False`. | -| `generateName` | Name + uid present in `request.Object` at validating admission; recorded by uid. No response-body recovery (the audit path's `generateName` headache is gone). | -| Impersonation (`--as`) | `req.UserInfo` is the effective (impersonated) user — parity with the audit path's `resolveUserInfo`. | -| Service-account submitter | Recorded by its `system:serviceaccount:…` username like any other; `AuthorAttributed=True`. | -| Object deleted before reconcile | Informer never reconciles it; the orphan record self-cleans via the fixed TTL. Harmless. | -| Recreate same name (new uid) | Different uid → different key; no stale inheritance. | -| Webhook on but record missing | A real anomaly (write miss). `AuthorAttributed=False` like any committer fallback; surface it via a `command_author_lookup{result=miss}` metric so it is observable rather than silent. | - ---- - -## 9. Note for end users — why your *commands* are attributed but your *edits* may not be - -There is an honest asymmetry worth stating plainly, because users will notice it: - -> *"You can name me as the author of a CommitRequest I create, but my normal edits to a -> ConfigMap commit as the bot unless I run the audit webhook. Why?"* - -Because the two are not the same kind of thing, and crediting them the same way would -be wrong: - -- A **command** (a CommitRequest) is something you hand us directly, and we only ever - *act* on it once it has actually persisted. If your command never persists (rejected - by another webhook, a conflict), we simply never act — so attributing it at admission - can never put a wrong author on a real commit. We can safely sign for the parcel you - handed us. -- A **normal edit** attributed at admission could credit you with a change that **never - landed** — a later webhook rejects it, an optimistic-concurrency conflict drops it, a - dry-run was only a rehearsal. If we mirrored that into git, the repository would claim - a change the cluster never accepted. We will not sign for a delivery we only saw - *attempted*. Proof-of-persistence for an arbitrary edit lives only in the **audit - event**, which is why faithful attribution of normal activity needs the audit webhook. - -So the asymmetry is a **feature, not a gap**: gitops-reverser gives you the most -attribution each environment can *correctly* support. On a managed cluster where you -cannot enable an apiserver audit webhook, your commands are still attributed (admission -is enough for them), while your normal edits commit as the configured committer until -audit is available — and the `AuthorAttributed` condition tells you, per command, -exactly which path you got. Nothing is ever attributed on a guess. - -This is the same principle as the rest of the design: **command authorship is "who -asked"; state attribution is "what actually happened, and who."** We never conflate them. - ---- - -## 10. What this removes (forward-facing, not kept in parallel) - -Per the "don't keep the old audit ingestion around for this" directive: - -- **`AttributionIndex.LookupCommitRequestAuthor`** - ([attribution_index.go:410-423](../../internal/queue/attribution_index.go#L410-L423)) - and the `CommitRequestAuthorLookup` interface - ([commitrequest_controller.go:54-59](../../internal/controller/commitrequest_controller.go#L54-L59)) - — deleted; the controller depends on `CommandAuthorStore` instead. -- The **wait machinery**: `commitRequestAttributionTimeout`, - `commitRequestAttributionRetryDelay`, the `waitForAudit` return, the - `WaitingForAuditEvent` progress state, and the `Attributed=Unknown/False` audit - states (§5). -- `commitRequestResolveTimeout` shrinks by its `+60s` attribution component - ([commitrequest_controller.go:92-98](../../internal/controller/commitrequest_controller.go#L92-L98)). -- The audit handler still records facts for every mutating resource generically, so - `commitrequests` facts are *written* but **no longer read** for authorship. - Optionally skip recording the `commitrequests` resource in `RecordFact` to avoid dead - keys (minor). - -The audit webhook itself is **untouched and still required for mirrored-resource -attribution** — this change is scoped to the command path only. - -### Breaking change (alpha) - -CommitRequest authorship now comes **only** from the admission webhook -(`--admission-webhook` / the validate-operator-types webhook). An operator who ran -**`--author-attribution=true` with `--admission-webhook=false`** and relied on -audit-sourced CommitRequest attribution will, after this change, get -`AuthorAttributed=False (CommitterFallback)` — the commit is authored by the configured -committer. To keep CommitRequest authorship, enable the admission webhook. Mitigation: -the `config/` SUT and the Helm chart (`servers.admission.enabled`) both default it **on**, -so a fresh install is unaffected; only an audit-on/admission-off configuration regresses. -Mirrored-resource attribution (normal edits) is unchanged. v1alpha3 is alpha, so the -status-contract and behavior change are acceptable. - ---- - -## 11. Resolved decisions & remaining open questions - -**Resolved (this iteration):** +| `AuthorAttributed` | `AttributedFromAdmission`: the command submitter was captured | `CommitterFallback`: no command-author record; request claims no actor | +| `Pushed` | the attached window was committed and pushed | a benign no-commit or finalize failure | +| `Ready` | a pushed commit or benign no-commit | progress or a finalize failure | -- **Record cleanup:** TTL, fixed internal constant (`commandAuthorRecordTTL`), **no - flag** — the timing is deterministic, so there is nothing to tune (§3). -- **Handler shape:** a dedicated, extensible **validate-operator-types** handler/path, not a - CommitRequest-only one — future command kinds are a table entry plus a webhook rule (§4). -- **Absent record:** fall back to committer and surface it as - **`AuthorAttributed=False (CommitterFallback)`** (§5, §8). -- **TLS:** the established pattern — full BYO-cert configuration plus cert-manager - resources shipped in the chart/`config` for the easy route (§7). +`Reconciling=True` with `WaitingForCloseDelay` is the only normal in-progress state. It covers the +optional collect delay followed by worker finalization and push. -**Decided during implementation:** +## Wiring -- **One admission server.** The validate-operator-types handler rides the existing - `--admission-webhook` server: when that flag is on, `cmd/main.go` builds the - `CommandAuthorStore` and registers both the always-allow observer and the - validate-operator-types handler (`setupAdmissionWebhooks`). The controller's `AuthorLookup` - is that store (nil when the flag is off → configured-author, immediate). -- **Chart value surface.** A new `internalCommands` block in `values.yaml` - (`enabled`, `bindAddress`/`port`, `tls.*`, `certManager.enabled`, `timeoutSeconds`), - defaulting `enabled: true` to mirror `attribution.enabled`. New templates - `validate-operator-types-webhook.yaml` (the `ValidatingWebhookConfiguration` with - `cert-manager.io/inject-ca-from`) and `validate-operator-types-certificates.yaml` (the - serving cert), plus the admission flags/cert mount in `deployment.yaml` and a - `webhook` port on the controller Service. -- **Cleanup:** TTL alone (`commandAuthorRecordTTL`, 1h). No terminal-status delete. +The command-author store is wired when the admission webhook is enabled and Redis is configured. It is +independent of `--author-attribution`, which controls audit-backed authorship for live watched resources. +An installation may therefore have either, both, or neither source of author information. -**Still open (deferred, not blocking):** +Related live contracts: -- **Capture-lag / `command_author_lookup{result}` metric (§8).** Not yet wired — the - `AuthorAttributed=False` condition already gives per-object observability of the - fallback; the aggregate counter/gauge is a follow-up. -- **Window-match coupling.** Command authorship is admission-sourced, but the attach - still matches the open window *by author string* (`openWindow.Author`, set from the - edit's audit attribution). So a named admission author only finalizes a window that - was *also* attributed to the same user — i.e. with audit on. In configured-author mode - the window author is empty and a named command author would not match; the - CommitRequest e2e specs that assert a named author therefore still skip - configured-author mode. Worth a follow-up if command authorship should stand fully - alone. +- [architecture: CommitRequest Finalize](../architecture.md#commitrequest-finalize) +- [configuration: CommitRequest conditions](../configuration.md#commitrequest-finalize) +- [status conditions guide](status-conditions-guide.md) diff --git a/docs/spec/commitrequest-design.md b/docs/spec/commitrequest-design.md index fea2cc08..cc83aecc 100644 --- a/docs/spec/commitrequest-design.md +++ b/docs/spec/commitrequest-design.md @@ -1,757 +1,69 @@ -# CommitRequest: design around two use cases +# CommitRequest window finalization -> **Status & field correction (2026-06-28):** this note predates two renames. -> (1) CommitRequest now reports progress and outcome through kstatus-compatible **conditions** — `Ready` -> (summary), `Reconciling`/`Stalled` (kstatus), and the domain conditions `Attributed` and `Pushed` — with -> no `phase` string. Where the text below says `WaitingForAuditEvent`, read "in-progress (`Reconciling=True`, -> `Attributed` not yet `True`)"; the post-attribution wait the text calls "in grace" / "Finalizing" is now -> the `Reconciling` reason **`WaitingForCloseDelay`**; `Committed`/`Rejected`/`Failed` map to the -> `Ready`/`Pushed`/`Stalled` conditions and the `Ready` reason. -> (2) The spec field `delaySeconds` is now **`closeDelaySeconds`** (it delays closing the open window). -> Read every `delaySeconds` below as `closeDelaySeconds`. See -> [status-conditions-guide.md](status-conditions-guide.md) §CommitRequest and -> `api/v1alpha3/commitrequest_types.go` for the contract. +> **spec** — current behaviour. The code depends on this document; change one, change the other. +> Index: [`../INDEX.md`](../INDEX.md) -> Status: living design note, 2026-06-12. -> Supersedes the speculative redesign in -> [github-e2e-per-type-tail-failure-investigation.md](../finished/signing-snapshot-tail-replay-failure-investigation.md) §11–§15. -> Related: [commitrequest-barrier-timeout-decision.md](commitrequest-design.md) -> (Option A), [commitrequest-multi-finalize-design.md](commitrequest-multi-finalize-design.md) -> (single-worker serialization), [canonical-stream-retirement.md](../architecture.md) -> (the per-type-stream refactor that motivates this note). +A `CommitRequest` is a one-shot “save now” command for one `GitTarget`. It does not mirror the +`CommitRequest` object. Instead, it can attach a message to a matching open commit window and asks the +worker to close that window after the requested collect delay. -## 1. What a CommitRequest is for +## Request and window contract -A CommitRequest is a one-shot **"save my changes now"** signal. Instead of waiting -for a GitTarget's silence timer (the commit window) to expire, a user creates a -CommitRequest naming the GitTarget; the operator finalizes that user's open commit -window into a real Git commit and reports the branch and SHA back in status. +The request identifies the target in `spec.targetRef.name`, may provide `spec.message`, and sets +`spec.closeDelaySeconds` (0–300 seconds). It is handled by the target’s single branch worker, so resource +events and the attach request share one FIFO. -The promise we want to keep is small and precise: +The worker attaches a request only when all of these match an open window: -> The edits **this author** made **before** asking to save are finalized into a -> commit **attributed to that author**, carrying **the author's message**, and the -> commit SHA is reported back. +1. GitTarget name and namespace; +2. the named actor, if either side has one; and +3. whether each side names an actor (`AttributionOutcome.NamesActor`). -This note designs the feature around the **two use cases that must work really -well**, states the simplifying assumptions that make them tractable, and is -explicit about what we cannot promise. +The third rule keeps independently configured command admission and live-resource attribution from being +coupled. A request with no named submitter can attach to either a configured-author or an unresolved live +window, but never to a named actor’s window. A request with a named submitter can attach only to that +actor’s named window. Therefore one user’s request never finalizes another user’s work. -## 2. The two use cases +On its first attach, the worker sets the deadline to receipt plus `closeDelaySeconds`. Repeated reconciles +are idempotent and keep that first deadline. The delay lets a `kubectl apply` bundle whose `CommitRequest` +arrives before its resource mutations collect into the same window. -### UC1 — "Save" button +## Authorship -A user makes a handful of changes, then enters a commit message and presses -**Save**. The UI emits a CommitRequest with `spec.message` set. There is human time -(seconds) between the edits and the save. +The validating admission webhook captures a command submitter before the `CommitRequest` persists. The +controller performs one best-effort, present-or-never lookup: -> **Expected:** the edits made before Save are committed as one commit with the -> typed message; the SHA comes back. Until Save, nothing is pushed (the commit -> window is long). - -### UC2 — `kubectl apply` of a bundle that includes a CommitRequest - -A user runs `kubectl apply -f bundle.yaml`, where the bundle contains several -resources **and** a CommitRequest carrying the intended message. The API server -processes the documents close together but in some order, and they fan out across -**different per-type audit streams**. - -> **Expected:** all the bundle's resources and the CommitRequest land in **one -> commit** with the CommitRequest's message. - -UC2 is the hard one, for one reason: **the CommitRequest may be processed first, in -the middle, or last.** If it is processed first, its "save" intent arrives before -the resources it is meant to save even exist in the pipeline. So the CommitRequest -must not shut the window the instant it is attributed — it needs a **grace period** -during which the rest of the same-author bundle can arrive and join the window. - -## 3. Simplifying assumptions - -This design is written for the two use cases above under two explicit assumptions. -Both are realistic for the use cases and keep the problem tractable. - -- **A1 — The commit window is large.** The GitProvider's silence timer is long - (minutes), so it never rolls the window out from under us during a save or an - apply. Once the author's window opens, it stays open until *we* finalize it. -- **A2 — No concurrent authors.** Only one author is writing to this GitTarget - during the action. We already decided we cannot merge independent authors into - one commit (§4), so we assume that situation away here rather than pretend to - handle it. - -Relaxing either assumption lands you in the "cannot promise" territory of §4 — that -is exactly where it belongs. - -## 4. What we honestly cannot promise - -Stating the limits is half the design. These are consequences of having **one -commit to make** and of giving up the single total order in the per-type refactor. -The design's job is to **bound and report** them, not to pretend they don't exist. - -- **One window, one commit — authors cannot be merged.** A branch worker holds at - most one open commit window. If ten different authors each make a change in one - second, we audit each one and produce **ten commits**, one per author, in arrival - order. We will not, and should not, fold two authors' edits into a single commit: - that would require conflict resolution between independent writers, which is the - complexity a single-window model exists to avoid. A CommitRequest from author X - only ever finalizes author X's window; another author's window is left untouched - (status `Rejected`/`WindowMismatch`, §6.7). **That strictness is a feature.** -- **Cross-type order is already approximate.** Because each type has its own audit - stream and the streams drain independently, even those ten one-event-each commits - may not be in the exact order the API server saw them. For a - **single author writing one bundle (UC2) this does not matter** — all the edits - land in the *same* commit regardless of intra-bundle order. It only surfaces - across commit boundaries, which we accept. -- **An internal writer can still steal the window.** Even with A1+A2, a per-type - **resync** finalizes the open live window before it applies (to preserve arrival - order). If one fires between an edit and the CommitRequest's finalize, the - author's window is gone. We cannot stop this — but §6.4 makes it *graceful* - instead of lossy: once the message is attached to the window, the resync's commit - carries the user's message and the CommitRequest reports `Committed`. The only - irreducible loss is an interrupt that lands *before* attribution, when there is no - message to attach yet. The worker must also ensure the stolen window's commit is - still **pushed** so the data is never lost. (This is the failure analyzed in the - investigation note; the push fix is worker-side, and §6.4 is the semantics that - preserve intent.) -- **"Before the CommitRequest" is bounded by grace, not guaranteed.** An edit made - moments before the save, on a type whose stream lags, may arrive after the grace - window and land in the **next** commit. We make the grace configurable and say so - in status; we do not wait unboundedly. - -## 5. How it works today - -### 5.1 The CRD - -`CommitRequestSpec` ([api/v1alpha3/commitrequest_types.go](../../api/v1alpha3/commitrequest_types.go)): - -- `gitTargetRef.name` — the GitTarget whose window to finalize (same namespace). -- `message` — optional verbatim commit message (1–1024 chars, no control chars). -- `delaySeconds` — the collect-grace window, `0`–`300`, default `0`. -- The spec is **immutable** after creation (CEL `self == oldSelf`). - -`CommitRequestStatus` reports `phase` (`WaitingForAuditEvent`, `Committed`, -`NoOpenWindow`, `Failed`), an optional `message`, and `branch` / `sha` on success. -*(The design renames the `NoOpenWindow` phase to `Rejected` and adds a structured -`reason` — see §6.7.)* - -### 5.2 The controller state machine - -[`CommitRequestReconciler.Reconcile`](../../internal/controller/commitrequest_controller.go) -runs with `MaxConcurrentReconciles=1` — the single worker *is* the -multi-CommitRequest ordering design, so no two finalizes interleave. One object is -advanced through: - -1. **Stamp** `WaitingForAuditEvent` (uncached re-reads guard against a stale cache - echo re-running a finalize that already terminated). -2. **Attribute** — `LookupCommitRequestAuthor` polls the `commitrequests` per-type - audit stream for *this object's own create event* (matched by namespace, name, - UID) and takes the author from it. Not seen yet → requeue every 2s. Past the 60s - attribution timeout → **fail closed** (`Failed`). The author is never guessed. -3. **Delay** — if `delaySeconds > 0`, requeue until `creationTimestamp + - delaySeconds`. Anchored at the object's creation timestamp. *(The design re-anchors - this at the attribution moment — see §6.4.4 — because creation-anchoring is fragile - under a delayed ingestion pipeline.)* -4. **Finalize** — enqueue the author-bound finalize signal and write terminal - status. (Today this step is preceded by the watermark barrier of §5.4, which - §6 argues to remove.) - -### 5.3 Attribution is the keystone - -Step 2 does two jobs at once, and this is the most important property of the design: - -- **Identity** — the author comes from a real audit event, never a guess. -- **Ordering anchor** — every mutation, including this CommitRequest, enters through - the same audit path. The author's earlier edits entered it *before* their - CommitRequest, so by the time we observe the CommitRequest's own create event, - the author's earlier work is — under comparable per-type ingestion delay — already - ingested and enqueued. Attribution buys most of the ordering we want, for free. - This is the basis for both use cases working with very little extra machinery. - -### 5.4 The finalize signal and the open window - -`FinalizeGitTargetWindow` enqueues a `FinalizeSignal` -(finalize_signal.go) onto the branch -worker's **FIFO event queue** — the same queue resource events ride — so by arrival -order it is processed after every earlier write for that worker. -`handleFinalizeSignal`: - -- no open window → `NoOpenWindow`; -- an open window whose **author + GitTarget + namespace** do not match → leave it - open, report `WindowMismatch` (cannot happen under A2 for the author's own work); -- a matching window → finalize into a commit; record branch + SHA. - -### 5.5 The watermark barrier (the part we are removing) - -Today, before finalizing, the controller takes a per-type watermark snapshot and -drains the tails to it (commit_barrier.go), -bounded by `FinalizeBarrierTimeout = 15s`, degrading visibly on timeout (Option A). -Three facts decide its fate: - -- It is **already best-effort** — the 15s timeout means it never was an invariant. -- It is taken **audit-anchored**, in step 4 *after* attribution and delay - ([commitrequest_controller.go:175](../../internal/controller/commitrequest_controller.go#L175)), - despite the stale "creation time" comment in the code. -- It has a **target-local gap**: the tail advances one cursor *per type* even when a - particular GitTarget was skipped - (audit_tail.go:177 vs the skip at - audit_tail.go:200), so it can report - `barrierReached=true` for a target that did not actually receive every protected - entry. A best-effort mechanism that can return a **false positive** launders "we - don't know" into a green status. - -## 6. The design that makes UC1 and UC2 work - -The core is small and, except for removing the barrier, already built: - -1. **Attribute** from the CommitRequest's own create audit event, or fail closed. -2. **Wait** `delaySeconds` from creation — the collect-grace window. -3. **Finalize** the open window bound to the attributed author + GitTarget. - -No snapshot, no drain loop. Here is why that is enough for each use case. - -### 6.1 UC1 walks straight through (delaySeconds = 0) - -The user edits, then — seconds later — presses Save. By the time the CommitRequest's -audit event is observed, the edits (made earlier, on the human timescale) are long -since mirrored into the open window (A1 keeps it open; A2 means it is the author's). -Attribution completes, `delaySeconds: 0` means finalize immediately, the window -matches → **one commit, the typed message, the SHA returned.** The barrier adds -nothing here: the human gap already guarantees the edits are present. - -### 6.2 UC2 needs the collect-grace, and the collect-grace is enough (delaySeconds > 0) - -Lay out the three orderings, all under A1+A2: - -- **CommitRequest last.** Resources ingest first and open/extend the window; the - CommitRequest's event arrives last; finalize after the grace → all in one commit. -- **CommitRequest in the middle.** Some resources are already in the window at - attribution; the rest arrive during the grace and join the same (still-open) - window; finalize at the end of the grace → all in one commit. -- **CommitRequest first.** This is the case that breaks a naive "finalize on - attribution." At attribution there may be **no window yet**. But the finalize is - deferred by the grace (`attribution + delaySeconds`, §6.4.4); during that grace the - bundle's resources arrive, open a window, and fill it; the deferred finalize then - finds the now-open window → all in one commit. - -So the **single mechanism — defer the finalize by `delaySeconds` and then finalize -whatever same-author window is open — covers all three orderings.** "A little bit of -grace before the CommitRequest is really shut down" *is* `delaySeconds`. UC2's -contract is therefore: set `delaySeconds` to comfortably exceed the bundle's -ingestion spread (a few seconds), and all of it lands in one commit. - -The honest boundary (§4): if a resource's stream lags past the grace, that resource -opens a *fresh* window after the finalize and waits for the next close. We do not -wait unboundedly, and we say so. - -### 6.3 Why the barrier goes - -The barrier helped neither use case: UC1 is covered by the human gap, UC2 by the -grace. It defended only a narrow, low-harm, already-degradable case (a same-author -edit on a lagging stream), while doing nothing for the cases that actually bite -(§4), and it could report false-positive success. **Remove it** — and retire the -Option-A timeout wording with it. We do not replace it with a blanket caveat on every -commit: §6.5 makes `Committed` mean "pushed to the remote", which is the honest claim -that matters. The one residual — an edit on a lagging type stream may land in the next -commit — is a documented property of the model (§4), not a per-request status note, -because once the barrier is gone there is nothing left that detects it. - -Keep the *idea* of an audit-anchored picture documented as the first thing to reach -for **if** real workloads ever prove harmful per-type skew — but reintroduce it then -as a **target-local applied watermark**, never the global cursor. - -### 6.4 Window-attach with eager message binding (the recommended target) - -§6.2 makes the **happy paths** of UC1 and UC2 work, but it has one weakness on the -**unhappy path**: between attribution and the finalize deadline, the CommitRequest's -message lives only on the controller side. If anything finalizes the window during -that interval — a resync (§4), a buffer-limit flush, anything — the resulting commit -carries the *generated* message and the CommitRequest later finds no window and -is `Rejected` (`NoWindowInGrace`, §6.7). The work is committed; the user's **intent** -(their message, and a `Committed` result) is lost. - -Window-attach fixes exactly that, and it is the model this feature should grow into: - -> **Bind the CommitRequest's message to the author's open window as early as -> possible. Once bound, whichever path finalizes the window first uses that -> message and resolves the CommitRequest as `Committed`.** - -The grace period stops being a "hold before we act" and becomes a "hold during which -the intent is already safe." That is the durability the feature is really about. - -#### 6.4.1 Worker model: the window carries the message - -There are **two state machines**, at two layers, and they hand off at attribution: - -- **Controller-side lifecycle** (the CRD phase): `Created → WaitingForAuditEvent - (attribution) → [attributed] → attach → poll → terminal`. The attribution wait is a - first-class state — it is where a **delayed ingestion pipeline is absorbed**, and it - is the point the grace is anchored *after* (§6.4.4). Nothing below happens until the - controller has observed the CommitRequest's own create audit event and bound the - author. (So this section's states are *not* missing the "waiting for the audit - event" moment — that moment lives one layer up, in §5.2 step 2 and the §7 table.) -- **Worker-side states** (below), which begin only *after* attribution, when the - controller sends the attach. - -Today the message travels on the one-shot `FinalizeSignal` and is applied only at the -finalize call. Window-attach moves it onto the window itself: - -``` -openWindow: - ... existing fields (author, target, events) ... - pendingMessage string // the CommitRequest's message, once attached - pendingCR *commitRequestRef // namespace/name/uid + result bookkeeping -``` - -A small per-worker table tracks CommitRequests not yet attached and the outcomes of -those already resolved: - -``` -pendingByAuthorTarget map[authorTargetKey][]commitRequestRef // waiting for a window -commitRequestOutcomes map[commitRequestID]Outcome // resolved: phase, sha, branch -``` - -A pending CommitRequest moves through four worker-local states: - -- **WaitingForWindow** — no matching (author + GitTarget + namespace) window is open - yet; the request is parked, with a finalize deadline. -- **Attached** — a matching window is open and now carries this request's - `pendingMessage`/`pendingCR`; a finalize is armed for the deadline. -- **AwaitingPush** — the window was finalized by *some* path; the `pendingCR` moved - onto the resulting `PendingWrite`, which is retained until its push lands (§6.5). -- **Resolved** — recorded in `commitRequestOutcomes`: `Committed` with the **pushed** - SHA once the PendingWrite reaches the remote, or `Rejected` with a reason — the - deadline passed with no window, the window was finalized but produced no diff, or it - belonged to someone else (§6.7). - -Transitions, all on the single worker loop goroutine (so no locking beyond the table): - -- attach request dequeued, matching window open and unclaimed → **Attached** - (set `pendingMessage`, arm finalize timer at the deadline); -- attach request dequeued, no matching window → **WaitingForWindow** (park, arm a - deadline timer); -- a same-author window opens while WaitingForWindow → **Attached**; -- finalize timer fires while Attached → finalize the window with the attached - message → **AwaitingPush**; -- **any other finalize path** runs on a window that is Attached (resync-before-apply, - buffer-limit, author switch, silence timer, shutdown) → it uses the attached - message → **AwaitingPush**; -- the `PendingWrite` carrying the `pendingCR` pushes successfully → **Resolved** - (`Committed` + pushed SHA, §6.5); -- a finalize produces no diff (the change is already present) → **Resolved** - (`Rejected`/`AlreadyPresent`, §6.7) — it does *not* enter AwaitingPush; -- deadline passes while still WaitingForWindow → **Resolved** (`Rejected`/`NoWindowInGrace`). - -#### 6.4.2 One chokepoint makes every cut-off honor the message - -All finalize paths already funnel through a single function -([`finalizeOpenWindowWithMessage`](../../internal/git/branch_worker.go)). That is -the leverage point. Change it to: - -1. use the explicit override message if the caller passed one, **else the window's - `pendingMessage`,** else the generated grouped message; and -2. if the window carries a `pendingCR`, **carry it onto the resulting `PendingWrite`** - alongside the message. Resolution does not happen here — it happens when that - `PendingWrite` pushes (§6.5). - -Because resync-before-apply, the commit-window timer, the buffer-limit flush, the -author-switch close, and shutdown **all** call this one function, every one of them -automatically carries the attached message *and* the `pendingCR` onto the -`PendingWrite` — with no per-path code. This is what makes "if something cuts it off, -the message is still there" true by construction rather than by enumerating closers. - -The companion worker fix from the investigation note still applies and is now a -hard prerequisite: a finalize that closes a window into a pending write **must -schedule its push** (`maybeSchedulePush`), so an attached-then-cut commit actually -reaches the remote. Attach makes the *result* correct; the push fix makes the *data* -land. - -#### 6.4.3 Controller ↔ worker protocol (attach, then poll) - -The synchronous request/reply of today does not fit, because the finalize may now -happen seconds later or be triggered by an unrelated path. Make it asynchronous, in -the same poll-via-requeue shape the attribution step already uses: - -1. **Controller**, the instant it attributes, sends one **AttachCommitRequest**`{crID, - author, target, message, delaySeconds}` onto the worker FIFO — no controller-side - delay. The worker stamps `finalizeAt = receipt + delaySeconds` on first registration - (§6.4.4). Re-sends across requeues are idempotent: the worker keys pending requests - by `crID` and keeps the first deadline. -2. **Worker** attaches (or parks) per §6.4.1; on the push that carries the - request's `PendingWrite`, it records the outcome (§6.5). -3. **Controller** requeues and polls `LookupCommitRequestOutcome(crID)`: - - not resolved and within a safety bound (covers the push cooldown + retries) → - requeue, surfacing any push-retry error in `status.message`; - - resolved → write the terminal status from the recorded outcome; - - no worker exists for the target → `Rejected` (`NoWindowInGrace`), as today. - -Polling (not a held result channel) keeps the controller non-blocking, survives a -reconcile running on a different goroutine, and is restart-tolerant. The worker GCs -`commitRequestOutcomes` entries by age. The controller's existing terminal-status -guards (uncached re-read, UID check) still make the status write at-most-once. - -#### 6.4.4 Timing: anchor the grace at attribution, not at object creation - -`delaySeconds` is a **collect window**, so it must be measured from the moment the -save is *observed*, not from when the object was created. Anchor it at attribution: - -> `finalizeAt = (attribution observed) + delaySeconds`, with `delaySeconds` bounded to -> ≤ 300s. - -This is a deliberate change from today, which anchors at the object's -`creationTimestamp` (§5.2). Object-creation anchoring is fragile under a delayed -ingestion pipeline: if ingestion takes longer than `delaySeconds`, then -`creation + delaySeconds` is already in the past by the time we attribute — the grace -has been entirely eaten by ingestion latency, the window may still be empty, and we -collect nothing (worst case `Rejected`/`NoWindowInGrace`). Creation-anchoring forces -`delaySeconds` to cover the *absolute* pipeline latency; attribution-anchoring lets it -cover only the inter-stream *spread*, which is all UC2 actually needs. Under a slow -pipeline it degrades gracefully — wait for ingestion, *then* collect — instead of -failing to collect. - -Concretely, no new status field is needed: the controller sends the attach the -instant it attributes, and the **worker** stamps `finalizeAt = receipt + delaySeconds` -when it first registers the request (receipt ≈ the attribution moment). Idempotent -re-sends keep the first deadline. - -- **UC1, `delaySeconds: 0`** — window already open at attribution; attach and - finalize immediately → `Committed`, the typed message. -- **UC2, `delaySeconds: N`** — attach the moment the first same-author edit opens the - window (before *or* after attribution, depending on bundle ordering); keep - collecting same-author edits; finalize at `finalizeAt`. - -An optional idle variant — finalize a few seconds after the *last* same-author edit, -capped at `finalizeAt` — would batch a burst more tightly; it adds a knob and the risk -of a chatty author deferring the commit, so treat it as a later tweak, not the -baseline. - -#### 6.4.5 What this buys, and the one residual - -Eager attach shrinks the window of vulnerability from the whole grace period down to -**just the attribution latency**: - -- Before attribution, there is no message to attach; if a resync cuts the window in - that gap, the commit uses the generated message and the CommitRequest finds no - window → `Rejected` (`NoWindowInGrace`). This is irreducible — we cannot attach an - intent we have not yet attributed. -- After attribution, the message is on the window. Any cut-off — resync or - otherwise — produces a commit **with the user's message**, and the CommitRequest - resolves as `Committed` once that commit is pushed (§6.5). The §4 "internal writer - steals the window" case stops being a lost-intent failure and becomes a graceful, - slightly-early commit. - -So we still cannot promise *one* commit in the face of a mid-flight resync (late -same-author edits that arrive after the cut open a fresh window and land later), but -we can promise the thing the user cares about most: **the message and the -work-so-far are committed under the author's name, and the request succeeds.** That -is the honest, defensible contract. - -#### 6.4.6 Edge cases - -- **Two CommitRequests for the same author/target.** A window carries at most one - `pendingCR`. A second request arriving while the window is Attached is parked - WaitingForWindow and binds to the *next* window (the edits after the first - finalize). Each CommitRequest still maps to exactly one commit with its own message. -- **No window ever opens.** Deadline passes WaitingForWindow → `Rejected` - (`NoWindowInGrace`) — pressing save with nothing pending, or a bundle with no - watched resources. -- **Restart.** Explicitly out of scope; see §6.6. - -## 6.5 Resolving the request: on push, from the `PendingWrite` - -The `PendingWrite` is already the system's nicest construct here: it carries the -commit message verbatim ([commit_executor.go:73](../../internal/git/commit_executor.go#L73)), -it is retained across the cooldown wait, and it survives the push-conflict -rebase-replay ([branch_worker.go:1091](../../internal/git/branch_worker.go#L1091)) -because the replay re-commits straight from it. So make it carry the CommitRequest's -result handle too, and resolve the request **when, and only when, its `PendingWrite` -is pushed**: - -``` -PendingWrite: - ... existing fields (events, message, signer, target) ... - CommitRequest *commitRequestRef // who to resolve once this write is pushed - CommitSHA plumbing.Hash // the hash of the commit this write created -``` - -- `executePendingWrite` already calls `worktree.Commit`, which returns the hash; - thread it back onto `CommitSHA`. On a rebase-replay the write is re-executed, so - `CommitSHA` is naturally **refreshed to the post-rebase hash** — no stale SHA. -- On a successful push ([`pushPending`](../../internal/git/branch_worker.go#L910)), - walk the pushed writes and, for each that carries a `CommitRequest`, record - `commitRequestOutcomes[cr] = {Committed, CommitSHA, branch}`. - -This is cheap — a couple of fields and a loop on the success path — and it finishes -the contract: - -- **`Committed` becomes honest.** The request is not resolved at local-commit time; - it is resolved when the commit is genuinely on the remote. A "save" therefore - confirms the push, which is exactly what a save should mean. The cost is latency - bounded by the push cooldown plus retries — seconds, and the right kind of wait. -- **The SHA is the real one.** It is read from the pushed write's own commit - (per-write, not branch HEAD, since a batched push may stack a later commit on top), - *after* any rebase-replay. The SHA-churn problem from the push-honesty discussion - disappears. -- **Push failure is honest too.** A failed push retains the `PendingWrite` - ([branch_worker.go:916-922](../../internal/git/branch_worker.go#L916-L922)) and - does not advance `lastPushAt`, so the request simply stays non-terminal while the - worker retries, with the push error surfaced in `status.message`. It flips to - `Committed` when a retry lands. It genuinely is not saved until then, and the status - says so — no new phase required (the CRD already documents `WaitingForAuditEvent` as - "the finalize it gates has not completed"). - -## 6.6 Restart and replay: explicitly out of scope - -We do **not** engineer durable recovery of an in-flight CommitRequest across an -operator restart. The message itself is never lost — it lives in `spec.message`, a -durable Kubernetes object — and on restart the controller re-reconciles any -non-terminal request, re-attributes from the durable audit event, and re-attaches. -That heals the common cases for free. - -The one case we knowingly accept is: a request whose commit was already **pushed** -but whose terminal status was not yet written when the operator restarted. The -in-memory `commitRequestOutcomes` entry is gone, the re-driven attach finds the work -already mirrored (so the re-finalize is a no-op), and the request resolves -`Rejected` (`AlreadyPresent`, §6.7) even though its commit — with its message — is on -the remote. The data and the message are safe; only the request's own status is -slightly off — and `AlreadyPresent` is at least an honest hint that the change is -already there. We judge this rare and low-harm and choose **not** to add a durable -record (e.g. a commit trailer) to -close it. If that ever changes, the durable system of record would be the commit -itself, stamped with the request identity; until then, restart recovery is best-effort. - -## 6.7 When nothing is committed: the `Rejected` outcome - -Not every CommitRequest produces a commit, and "no commit" is not one thing. Today -these collapse onto a single `NoOpenWindow` phase (with a mismatch *message* for the -foreign case). Rename that terminal phase to **`Rejected`** — one honest "the request -was handled correctly but produced no commit" state — and distinguish the reasons with -a structured `status.reason` plus a human `status.message`: - -| `reason` | When | Note | +| Admission result | `AuthorAttributed` | Request claim | |---|---|---| -| `NoWindowInGrace` | The deadline passed with no matching same-author window. | Benign: nothing was pending to save. | -| `WindowMismatch` | A window was open but belonged to a different author/GitTarget. | Benign and strict (§4): the foreign window is left untouched. | -| `AlreadyPresent` | A matching window was finalized, but its events produced **no diff** — the change already matches the remote, so the commit was dropped (loop prevention). | The theoretical edge this section exists for. | - -`Rejected` is deliberately **not** `Failed`. In all three cases the system behaved -correctly; there was simply no commit to make. `Failed` stays reserved for genuine -faults — attribution failed, the finalize/commit errored, or a push that can never -land — so a dashboard can treat `Failed` as "look at this" and `Rejected` as -informational. (At the worker level the existing `FinalizeNoOpenWindow` / -`FinalizeWindowMismatch` outcomes, plus a new no-change signal, simply roll up to -`Rejected` + the matching reason.) - -Concretely, the CRD gains a typed, validated `reason` enum and the field on status: - -```go -// CommitRequestRejectReason explains a Rejected CommitRequest: the request was -// handled correctly but produced no commit. It is set only when phase is Rejected. -// +kubebuilder:validation:Enum=NoWindowInGrace;WindowMismatch;AlreadyPresent -type CommitRequestRejectReason string - -const ( - // RejectNoWindowInGrace: the grace period elapsed with no matching same-author - // window — nothing was pending to save. - RejectNoWindowInGrace CommitRequestRejectReason = "NoWindowInGrace" - // RejectWindowMismatch: an open window existed but belonged to a different author - // or GitTarget, so it was deliberately left untouched. - RejectWindowMismatch CommitRequestRejectReason = "WindowMismatch" - // RejectAlreadyPresent: a matching window was finalized but produced no diff — the - // change already matches the remote, so the commit was dropped (loop prevention). - RejectAlreadyPresent CommitRequestRejectReason = "AlreadyPresent" -) - -// in CommitRequestStatus: -// // Reason explains a Rejected phase. Empty for non-Rejected phases. -// // +optional -// Reason CommitRequestRejectReason `json:"reason,omitempty"` -``` - -The human `message` still carries the prose; `reason` is the stable, machine-readable -discriminator that status consumers and tests assert on. - -### `AlreadyPresent` must not hang in `AwaitingPush` - -This is the case that interacts with §6.5, and it is why it needs explicit handling. A -finalized window normally creates a commit whose `PendingWrite` resolves the request -on push. But [`executePendingWrite`](../../internal/git/commit_executor.go#L132-L138) -returns `(0, nil)` when the events produce no change — **no commit, nothing to push.** -A request waiting for a push that never comes would sit in `AwaitingPush` until the -controller's safety bound and then mis-resolve. - -So the no-op is detected at finalize time and resolved immediately: - -- when a `PendingWrite` carrying a `pendingCR` executes and reports no change - (`anyChanges == false`), the worker records `commitRequestOutcomes[cr] = {Rejected, - AlreadyPresent}` right there — it never enters `AwaitingPush`; -- only a `PendingWrite` that actually created a commit (a real `CommitSHA`) takes the - `AwaitingPush` → `Committed`-on-push path of §6.5. - -In the §6.4.1 state machine this is one extra terminal edge off the finalize step: -*Attached → finalize produces no diff → Resolved(`Rejected`/`AlreadyPresent`).* - -### Worth a unit test - -Cheap, deterministic, and it guards a path that "never happens": finalize a window -whose event re-asserts state already present in the worktree (so -`applyPendingWriteEvents` returns `false`), then assert the carried request resolves to -`Rejected` with `reason == AlreadyPresent` — **and** that it resolves promptly rather -than blocking on a push. Assert on the structured `reason`, never on message text. - -## 7. State model - -The CRD keeps a small phase set; the richer "moments" are internal. - -| Internal moment | Meaning | User-visible phase | -|---|---|---| -| Created, waiting for audit event | Its own create audit event has not reached the `commitrequests` stream yet — this is where a delayed ingestion pipeline is absorbed. | `WaitingForAuditEvent` | -| Attributed, in grace | Author known; in the `delaySeconds` collect window anchored at attribution (§6.4.4). | `WaitingForAuditEvent` | -| Window matched, collecting | A matching same-author window exists and stays collectable until grace ends. | `WaitingForAuditEvent` | -| Finalized, awaiting push | The window was committed locally and carries the message; its `PendingWrite` is retained until it pushes (push retries surface in `status.message`). | `WaitingForAuditEvent` | -| Pushed | The carrying commit reached the remote; branch + pushed SHA recorded (§6.5). | `Committed` | -| No commit produced | No window in grace, a foreign window, or the change already matched the remote — distinguished by `reason` (§6.7). | `Rejected` (+ `reason`) | -| Attribution failed | The own audit event never arrived within 60s. | `Failed` | - -## 8. End-to-end tests (both use cases must be covered) - -Both use cases get a dedicated e2e spec. They share the existing harness shape -([test/e2e/commit_request_e2e_test.go](../../test/e2e/commit_request_e2e_test.go)): -a dedicated Gitea repo, GitProvider with a **large `commitWindow` (300s)** so the -silence timer can never be what produces the commit, a GitTarget, and a WatchRule. -A large window plus the dedicated repo realize A1; the dedicated namespace/repo -realizes A2. - -### 8.1 E2E-1 — UC1 "Save" (delaySeconds = 0) - -This is essentially the **existing** first spec ("finalizes the open commit window -on demand and reports the resulting SHA") and is the regression anchor for UC1: - -1. Apply a Deployment (opens the window). -2. `Consistently` assert for 10s that `main` does not yet exist — the edit is held. -3. Apply a CommitRequest (`delaySeconds: 0`) with an explicit `message`. -4. `Eventually` assert `status.phase == Committed`, `status.sha` non-empty, - `status.branch == main`. -5. Assert the commit subject equals `spec.message` verbatim, the Deployment file is - present, and `HEAD` equals the reported SHA. - -Strengthen it with one assertion: **exactly one new commit** was produced (HEAD -advanced by one), proving the save did not also trigger a stray second commit. - -### 8.2 E2E-2 — UC2 `kubectl apply` bundle (delaySeconds > 0, CommitRequest first) - -A new spec proving the bundle lands in one commit even when the CommitRequest is -applied **first** (the hard ordering): - -1. Build a single multi-document manifest whose **first** document is a - CommitRequest (`delaySeconds: 8`, explicit `message`) followed by **two or three - Deployments** in the watched namespace. -2. Apply the whole bundle in one `kubectl apply -f -`. -3. `Eventually` assert `status.phase == Committed` and `status.sha` non-empty. -4. Assert **exactly one commit** was produced for the bundle (HEAD advanced by - exactly one), its subject equals the CommitRequest's `message`, and **every** - Deployment in the bundle is present in that single commit. -5. (Stronger variant, optional) Watch a second type so the bundle spans two per-type - streams (e.g. add a ConfigMap to the bundle and a ConfigMap WatchRule), proving - the one-commit guarantee holds across independent streams — mind the pre-existing - `kube-root-ca.crt` ConfigMap noted in the existing suite when asserting commit - counts. - -The `delaySeconds: 8` value must comfortably exceed the bundle's ingestion spread so -the test is deterministic; the assertion is on the **outcome** (one commit, all -files, the message), not on internal ordering. Putting the CommitRequest first in -the file is the deliberately-hard arrangement: it exercises "save intent arrives -before the work" and proves the collect-grace (§6.2) closes that gap. - -### 8.3 Intent durability — a cut-off still carries the message (§6.4) - -The property §6.4 exists for is best pinned by a **deterministic worker/integration -test**, because forcing a resync to interleave is racy in full e2e: - -1. Open an author window (one event), then deliver an `AttachCommitRequest` with a - distinctive `message` and a non-zero `finalizeAt` so the window is **Attached** - but not yet finalized. -2. Before the deadline, invoke a window-closing path directly — the - `resync-before-apply` finalize is the canonical one. -3. Assert the resulting commit's message equals the CommitRequest's `message` (not - the generated grouped message), and that **after the carrying `PendingWrite` - pushes** (§6.5) `LookupCommitRequestOutcome` reports `Committed` with the **pushed** - commit's SHA — and that the reported SHA matches what is actually on the remote. - -Two complementary assertions: (a) `finalizeOpenWindowWithMessage` with no override on -a window carrying a `pendingMessage`/`pendingCR` uses the pending message and moves -the `pendingCR` onto the `PendingWrite`; (b) a push that hits a conflict and -rebase-replays still resolves the request to the *post-replay* SHA (no stale/orphaned -SHA). Add a full e2e only if a reliable resync trigger exists; the integration test is -the dependable pin. +| submitter record found | `True` / `AttributedFromAdmission` | that named actor | +| no record, webhook disabled, or Redis unavailable | `False` / `CommitterFallback` | no actor | -### 8.4 Already-present — a finalize that produces no diff (§6.7) +`AuthorAttributed=False` is not a statement about the eventual Git author. The matched live window decides +that: -The theoretical edge, pinned by a deterministic unit test (see §6.7): finalize a -window whose event re-asserts already-present state so `applyPendingWriteEvents` -returns `false`, and assert the carried request resolves to `Rejected` with -`reason == AlreadyPresent` **promptly** — it must not block waiting for a push that -never comes. Assert on the structured `reason`, not on message text. +- configured-author, replay, and resync windows use the configured committer; +- a resolved live attribution fact uses the authenticated actor; and +- a live attribution miss uses `unknown (attribution unresolved) `. -## 9. Migration / order of work +See [CommitRequest admission authorship](commitrequest-admission-authorship.md) for capture provenance and +[architecture: author identity](../architecture.md#author-and-committer-identity-in-git) for the three Git +author states. -1. Add **E2E-2** (UC2), strengthen **E2E-1** (UC1), and add the **§8.3 intent- - durability** integration test — they pin the behavior before any refactor. -2. Remove the snapshot + drain step from `Reconcile`; keep attribution, delay, and - the author-bound finalize unchanged. -3. Delete (or park behind a clearly-labelled "future, target-local" stub) - `TakeTypeSnapshot` / `DrainTailsToSnapshot` and the per-type tail **cursor** - bookkeeping used only by the barrier. The tail itself, its checkpoint anchor, and - the late-event nudge stay — they are the ingestion path, not the barrier. -4. Retire the Option-A barrier-timeout status wording along with the barrier. Leave - the happy-path status clean: §6.5 makes `Committed` mean "pushed", which is the - honest claim. The lagging-stream caveat (§4) lives in the docs, not on every - request. -5. Worker: ensure **any** finalize that closes a window schedules its push (the - stranded-write fix) — prerequisite for §6.4 to be safe under a cut-off. -6. Build **§6.4 eager attach**: move the message onto `openWindow`, make - `finalizeOpenWindowWithMessage` honor the attached message and carry the - `pendingCR` onto the resulting `PendingWrite`, evolve the `FinalizeSignal` into an - idempotent `AttachCommitRequest`, switch the controller to attach-then-poll, and - **re-anchor the grace at attribution** — the controller drops its own delay and the - worker stamps `finalizeAt = receipt + delaySeconds` (§6.4.4), replacing today's - creation-anchored delay. -7. Build **§6.5 resolve-on-push**: add `CommitRequest` + `CommitSHA` to `PendingWrite`, - capture the commit hash from `executePendingWrite` (refresh it on rebase-replay), - and resolve the request from the push success path with the pushed SHA. This makes - `Committed` mean "on the remote" and removes the stale-SHA case. -8. Adopt **§6.7 `Rejected`**: rename the `NoOpenWindow` phase to `Rejected`, add the - structured `status.reason` (`NoWindowInGrace` / `WindowMismatch` / `AlreadyPresent`), - and resolve a no-diff finalize (`anyChanges == false` carrying a `pendingCR`) - immediately as `Rejected`/`AlreadyPresent` rather than letting it wait on a push. - Update the existing e2e assertions that expect `NoOpenWindow`. (CRD enum change is - acceptable in `v1alpha1`.) -9. Document restart recovery as out of scope (§6.6) — no durable record built for now. -10. Future, only if proven needed: the idle-reset grace variant (§6.4.4), and — if - cross-type skew is ever shown harmful — a **target-local** audit-anchored - watermark, never the global cursor. +## Lifecycle and outcomes -## 10. Bottom line +The controller stamps its conditions on first reconcile, immediately attaches the request, and polls the +worker. There is no audit wait for a `CommitRequest`; a delayed admission record cannot arrive after the +object is visible. -The refactor's one true cost is the loss of a single total order, and we tried to -buy it back with a per-type watermark barrier that is best-effort, partly incorrect, -and aimed at a case neither use case needs. Strip it. The promise users care about — -**save this author's recent work, attributed correctly, with their message, and tell -me the SHA** — is kept by two things we already have: **attribution from the -CommitRequest's own audit event** and a **configurable collect-grace**. UC1 needs -only the first; UC2 needs both, with `delaySeconds` sized to the bundle. Be honest in -status about what "before the CommitRequest" can mean across independent streams, pin -both use cases with e2e tests, and resist rebuilding the total order until something -real proves we need it. +| Outcome | Conditions | +|---|---| +| Commit pushed | `Ready=True`, `Pushed=True`, reason `Committed`; `status.sha` and `status.branch` are set | +| No same window before deadline | `Ready=True`, `Pushed=False`, reason `NoWindowInGrace` or `WindowMismatch` | +| Window produced no diff | `Ready=True`, `Pushed=False`, reason `AlreadyPresent` | +| Finalize or push error | `Ready=False`, `Pushed=False`, `Stalled=True`, reason `FinalizeFailed` | -And grow toward **eager message attachment** (§6.4): bind the author's message to -their open window the instant we can, so the grace period protects intent rather than -merely deferring action. Then the worst realistic interruption — a resync cutting the -window — yields a commit *with the user's message*, and the only thing we cannot save -is an interrupt that beats attribution itself. +`Reconciling=True` with reason `WaitingForCloseDelay` is the normal in-progress state. The controller fails +with `FinalizeFailed` only if the worker does not resolve the request within its bounded safety window; it +never polls indefinitely. -Finally, let the `PendingWrite` carry the request to its end (§6.5): it already holds -the message verbatim and survives every replay, so it is the natural place to also -hold the result handle and the commit SHA. Resolve the request **on push**, with the -pushed SHA — and `Committed` stops meaning "written locally" and starts meaning "on -the remote," with no stale SHA and at near-zero cost. Restart recovery we knowingly -leave best-effort (§6.6). That is a contract worth being proud of: simple at the core, -honest at the edges, and durable exactly where it counts. +The complete status vocabulary is in the [status conditions guide](status-conditions-guide.md). diff --git a/docs/spec/commitrequest-multi-finalize-design.md b/docs/spec/commitrequest-multi-finalize-design.md deleted file mode 100644 index 792a85e5..00000000 --- a/docs/spec/commitrequest-multi-finalize-design.md +++ /dev/null @@ -1,405 +0,0 @@ -# C-B2: per-GitTarget finalize coordinator (Option A design) - -> **spec** — current behaviour. The code depends on this document; change one, change the other. Index: [`../INDEX.md`](../INDEX.md) - -> Status: **superseded in part — C-B2 LANDED 2026-06-11 in a simpler form; see §12.** -> The per-type snapshot analysis (§3) and the ordering invariant (§1) are the parts that -> shipped; the coordinator goroutine, the annotation persistence, and the requeue-poll -> state machine did not — `MaxConcurrentReconciles=1` plus a fresh snapshot per attempt -> deliver the same guarantees with less machinery. -> Context: [canonical-stream-retirement.md](../architecture.md) §8 C-B2 row -> + the multi-CommitRequest ordering hazard raised 2026-06-11 - -## 1. The problem the design must solve - -The planned C-B2 puts finalize logic into the `CommitRequestReconciler`. The immediate -question is: **what happens when two CommitRequests for the same GitTarget exist at the -same time?** - -The root cause is that controller-runtime reconciles independent objects concurrently. -Two reconcile goroutines — one for CR-1 (rv=100), one for CR-2 (rv=110) — can both -pass their watermark barrier and then race to call `FinalizeGitTargetWindow`. The -BranchWorker's FIFO determines ordering; whichever goroutine enqueues first wins. If -CR-2 wins the race: - -``` -BranchWorker FIFO (wrong order): - [upserts@<100] [upserts@100-110] [finalize_CR2] [finalize_CR1] - ← CR-2's barrier ← CR-1's barrier, loses race -``` - -Result: **one commit attributing everything to CR-2**, then CR-1 gets `NoOpenWindow`. -The author who created CR-1 first gets no commit, even though every mutation was -present. The correctness guarantee from §6 of the retirement doc is violated. - -**The invariant that must hold:** -> For any two CommitRequests CR-i and CR-j targeting the same GitTarget where -> `rv_i < rv_j`, the sequence `barrier(CR-i) → enqueue(finalize_i)` must complete -> **before** `barrier(CR-j) → enqueue(finalize_j)` begins. - ---- - -## 2. Solution: per-GitTarget FinalizeCoordinator - -A single goroutine per active GitTarget owns the (barrier → finalize-enqueue) pair. -CommitRequest reconcilers hand off their work via a **buffered channel** and poll for -the result. The goroutine serializes naturally: it processes one finalize request at a -time, in FIFO order. Because it is the only one calling `FinalizeGitTargetWindow` for -its GitTarget, the invariant above holds by construction. - -```mermaid -graph TD - subgraph controller-runtime - CRC1["CommitRequestReconciler\ngoroutine A (CR-1)"] - CRC2["CommitRequestReconciler\ngoroutine B (CR-2)"] - end - - subgraph EventRouter - FC["FinalizeCoordinator\n(one goroutine per GitTarget)\n\nchan finalizeRequest ← buffered"] - FAW["FinalizeAtWatermark\n(DrainTailsToWatermark + Enqueue)"] - end - - subgraph Manager - TC["auditTailCursors\n(per-GVR, updated after apply)"] - end - - BW["BranchWorker\n(FIFO per GitTarget)"] - - CRC1 -- "Enqueue(rv_C=100, types_snapshot, ...)" --> FC - CRC2 -- "Enqueue(rv_C=110, types_snapshot, ...)" --> FC - FC -- "one at a time, in order" --> FAW - FAW -- "DrainTailsToWatermark(rv)" --> TC - FAW -- "EnqueueFinalize(signal)" --> BW - FC -- "result written to k8s status" --> API["kube-apiserver\n(CommitRequest status)"] - BW -- "flushed commit" --> Git["Git remote"] -``` - ---- - -## 3. What a finalizeRequest carries - -The request is assembled by the reconciler at the moment it first sees the -CommitRequest (`phase == ""`), **before** the status stamp bumps the object's -ResourceVersion: - -```go -type finalizeRequest struct { - // typeSnapshot maps each claimed GVR to the current top of its stream, - // read via TakeTypeSnapshot at CR creation time. This is the per-type - // watermark the barrier will wait on — one independent comparison per type, - // no cross-type RV ordering assumed (see §4). - typeSnapshot map[schema.GroupVersionResource]string - - // rv_C is the CommitRequest's create ResourceVersion, stored only for the - // restart-safety annotation (see §8). It is NOT used as a barrier watermark. - rv_C string - - // commit identity - author string - message string - - // crRef identifies the CommitRequest to write status back to. - crNamespace string - crName string - crUID types.UID // guards against delete+recreate between enqueue and writeback -} -``` - -### Why per-type watermarks, not a single rv_C? - -Kubernetes only guarantees ResourceVersion monotonicity **within a single resource -type**. In practice, core types all share the same etcd global counter, but: - -1. Aggregated-API types (served by a separate APIService) have RVs in an entirely - different counter. A CRD type might have entries at rv `8, 9, 10` while the - CommitRequest has rv `200` (core etcd). Comparing `cursor_T` (agg RV=10) against - `rv_C` (core RV=200) gives 10 < 200 — the barrier would stall until the 15 s - timeout. - -2. Even for core types, a type claimed **after** rv_C (say at checkpoint rv=150) gets - its tail cursor anchored at `"150-maxuint64"`. If rv_C=100, the barrier would wait - for that cursor to reach 100, which it has already passed — but could also stall if - the tail hasn't started yet. - -The per-type snapshot avoids both problems: for each type T the GitTarget was already -watching when the CR was created, record the current top of T's stream. The barrier -waits until `cursor_T >= snapshot[T]` — a comparison within T's own RV space. -Correctness argument: any mutation the author made to type T before creating the CR -was already in T's stream when the snapshot was taken, so `snapshot[T] >= rv(mutation)`, -and the barrier guarantees the tail consumed it. - -**The snapshot is built by `EventRouter.TakeTypeSnapshot(ctx, gitTargetName, ns)`** -which calls `Manager.TakeTypeSnapshot` → `TypeAuditHighWater` (per claimed type via the -`auditHighWaterReader` optional capability on the reader). The barrier -(`DrainTailsToSnapshot`) then polls only the in-memory cursor map — no Redis calls -during the wait loop. - ---- - -## 4. Full sequence: single CommitRequest - -```mermaid -sequenceDiagram - autonumber - participant Author - participant API as kube-apiserver - participant WH as webhook - participant Streams as :audit:stream (×N types) - participant Tails as per-type tails - participant BW as BranchWorker (FIFO) - participant CRC as CommitRequestReconciler - participant FC as FinalizeCoordinator - participant Git as Git remote - - Author->>API: PUT cm/a (rv=101), PUT deploy/b (rv=102) - API-->>WH: audit events - WH-->>Streams: mirror cm@101-*, deploy@102-* - Streams-->>Tails: deliver entries - Tails->>BW: enqueue upsert(cm/a), upsert(deploy/b) - Tails->>Tails: cursor[configmaps]=101-0, cursor[deployments]=102-0 - - Author->>API: CREATE CommitRequest (rv_C = 105) - API-->>CRC: watch event - - Note over CRC: phase == "", first reconcile - CRC->>CRC: capture rv_C = 105 (object.ResourceVersion) - CRC->>CRC: snapshot typesAtRV = [configmaps, deployments] - CRC->>CRC: store rv_C in annotation (restart safety) - CRC->>API: stamp WaitingForAuditEvent (RV now 106) - CRC->>FC: Enqueue(rv_C=105, types=[cm,deploy], author, message) - CRC->>CRC: requeueAfter(200ms) to poll for result - - Note over FC: goroutine owns the barrier+enqueue pair - FC->>FC: DrainTailsToWatermark(rv_C=105, types=[cm,deploy]) - Note over FC: cursor[cm]=101≥100? ✓ cursor[deploy]=102≥100? ✓ → passes immediately - FC->>BW: EnqueueFinalize(author, message) ← after all upserts in FIFO - BW->>Git: flush open window → commit with cm/a, deploy/b - BW-->>FC: FinalizeResult{Committed, sha=abc123} - - FC->>API: write status Committed / sha=abc123 - - CRC->>API: re-read CommitRequest (poll on requeue) - Note over CRC: phase=Committed → terminal, nothing more to do -``` - ---- - -## 5. Full sequence: two concurrent CommitRequests - -This is the critical scenario. Both CRs target the same GitTarget. - -```mermaid -sequenceDiagram - autonumber - participant CRC1 as Reconciler (CR-1, rv=100) - participant CRC2 as Reconciler (CR-2, rv=110) - participant FC as FinalizeCoordinator\n(single goroutine) - participant Tails - participant BW as BranchWorker (FIFO) - - CRC1->>FC: Enqueue({rv_C=100, types, author1, msg1}) - CRC2->>FC: Enqueue({rv_C=110, types, author2, msg2}) - Note over FC: channel is FIFO, CR-1 was enqueued first - - Note over FC: START processing CR-1 - FC->>Tails: DrainTailsToWatermark(rv_C=100) - Tails-->>FC: cursor[cm]=101≥100 ✓, cursor[deploy]=102≥100 ✓ → pass - FC->>BW: EnqueueFinalize(author1, msg1, rv_C=100) - BW->>BW: flush window → commit-1 (cm/a, deploy/b at rv<100) - BW-->>FC: Committed, sha=aaa - FC->>API: write CR-1 status = Committed - - Note over FC: START processing CR-2 (only now) - FC->>Tails: DrainTailsToWatermark(rv_C=110) - Note over Tails: tails continue delivering entries 100-110 while FC waited - Tails-->>FC: cursor[cm]=108≥110? maybe not yet → FC waits - Tails->>BW: enqueue more upserts (changes 100-110) - Tails-->>FC: cursor[cm]=111≥110 ✓ → pass - FC->>BW: EnqueueFinalize(author2, msg2) - BW->>BW: flush window → commit-2 (changes between commit-1 and now) - BW-->>FC: Committed, sha=bbb - FC->>API: write CR-2 status = Committed -``` - -The coordinator is the single point that prevents the race: CR-2's barrier cannot -start until CR-1's barrier + enqueue is complete. The BranchWorker FIFO then -naturally holds `[upserts@<100] [finalize_1] [upserts@100-110] [finalize_2]`. - ---- - -## 6. Reconciler state machine - -```mermaid -stateDiagram-v2 - [*] --> FirstReconcile : watch event (phase == "") - - FirstReconcile --> WaitingForResult : call TakeTypeSnapshot (per-type stream tops),\nstore rv_C + snapshot in annotation,\nstamp WaitingForAuditEvent,\nEnqueue(typeSnapshot, rv_C, author, msg) to coordinator,\nrequeueAfter(200ms) - - WaitingForResult --> WaitingForResult : poll: status still WaitingForAuditEvent\n→ requeueAfter(200ms) - - WaitingForResult --> Terminal : coordinator wrote terminal status\n→ nothing more to do - - FirstReconcile --> WaitingForResult : restart — phase=WaitingForAuditEvent,\nread snapshot+rv_C from annotation,\nre-Enqueue to coordinator - - Terminal --> [*] -``` - -The reconciler itself **never blocks** for more than a single API call. All blocking -(the ≤15 s barrier wait) happens in the coordinator goroutine, which calls -`DrainTailsToSnapshot` — a pure in-memory poll of cursor values, no Redis calls during -the wait loop. - ---- - -## 7. Coordinator lifecycle - -| Event | Action | -|---|---| -| First `Enqueue` call for a GitTarget | Start coordinator goroutine, create buffered channel (capacity=8) | -| GitTarget deleted (`UnregisterGitTargetEventStream`) | Close channel → goroutine exits after draining | -| Process restart | All coordinators are gone. Reconciler re-fires for each WaitingForAuditEvent CR and re-enqueues (rv_C re-read from annotation). | -| A CR is deleted before coordinator processes it | Coordinator reads the CR before writing status; `IsNotFound` → skip. | -| UID mismatch (delete + recreate with same name) | Coordinator guards with `crUID` check before writing status → skip stale result. | - ---- - -## 8. Restart safety: persisting the snapshot across pod restarts - -**The problem:** The reconciler calls `TakeTypeSnapshot` and captures -`rv_C = commitRequest.ResourceVersion` on the first reconcile, *before* stamping -status. Once it stamps `WaitingForAuditEvent`, the object's `ResourceVersion` changes -(to e.g. 106). If the pod restarts, the reconciler sees `phase=WaitingForAuditEvent` -but both `rv_C` and the snapshot are gone. - -**Solution: store rv_C and snapshot in an annotation before the status stamp.** - -``` -Annotation key: configbutler.ai/finalize-snapshot -Value: JSON: {"rv":"105","snapshot":{"":"secrets:101","apps":"deployments:50"}} -``` - -Reconciler on restart: -``` -if phase == WaitingForAuditEvent: - rv_C, snapshot = parseAnnotation("configbutler.ai/finalize-snapshot") - if annotation missing: call TakeTypeSnapshot again (stale, waits for a bit more but safe) - re-Enqueue(snapshot, rv_C, ...) to coordinator -``` - -**On restart without the annotation** (annotation somehow lost): the reconciler calls -`TakeTypeSnapshot` again. The new snapshot will have stream tops at the restart time, -which are ≥ the original snapshot values — the barrier waits for a superset of what the -original snapshot required. This is safe (correct, slightly more conservative) and -bounded by the 15 s timeout. - -**Why an annotation and not a status field?** Adding a status field requires a CRD -schema change and regeneration. An annotation is a metadata write that can be applied -in the same `Patch` as the status stamp — no schema bump needed. - -**Order of operations on first reconcile:** - -``` -1. TakeTypeSnapshot → snapshot (one Redis call per claimed type) -2. Patch: set annotation with rv_C + snapshot (metadata write) -3. Status().Update: phase = WaitingForAuditEvent (status write) -4. Enqueue(snapshot, rv_C, author, message) to coordinator -5. requeueAfter(200ms) -``` - -If the pod dies between steps 2 and 3: on restart, annotation is present but phase is -still `""` → first-reconcile path, reads annotation to avoid a second `TakeTypeSnapshot`. - -If the pod dies between steps 3 and 4: on restart, phase=WaitingForAuditEvent, reads -annotation, re-enqueues. The coordinator is idempotent for the same CR+UID. - ---- - -## 9. Where does the coordinator live? - -**`EventRouter`** — it already owns: -- `WatchManager` (→ `DrainTailsToWatermark`) -- `WorkerManager` (→ `FinalizeGitTargetWindow`) -- `Client` (→ status writeback) -- `gitTargetStreams` map with the same per-GitTarget keying pattern - -It adds a `finalizeCoordinators map[types.ResourceReference]chan finalizeRequest` with -the same mutex discipline. The coordinator goroutine calls -`r.FinalizeAtWatermark(...)` (already implemented, C-B1) and then writes status -directly using `r.Client`. - -The `CommitRequestReconciler` gets an `EventRouter` reference (it already has none -today; it would be wired in `cmd/main.go` where the reconciler is registered). - ---- - -## 10. What changes in the audit consumer - -*(Landed with C-B2 — `commit_request.go` was deleted outright; `applyFinalizeResultToStatus` -and friends moved to `internal/controller/commitrequest_finalize.go`.)* - -After C-B2 lands: - -- `isCommitRequestCreate` — **deleted** -- `handleCommitRequest` / `writeCommitRequestStatus` — **deleted** -- `applyFinalizeResultToStatus` — **moved** to `internal/watch/` (or kept in - `internal/queue/` and imported by the coordinator) - -The consumer's only remaining job before C-C is: route `AuditConsumer.routeAuditEvent` -— which today already returns early for everything except scale (already deleted in -C-A2). After C-B2 it returns early for everything. C-C then deletes the file entirely. - ---- - -## 11. Open questions before implementation - -| # | Question | Impact | -|---|---|---| -| Q1 | Should the coordinator goroutine also be responsible for the `barrierTimedOut` metric (`commitrequest_barrier_timeouts_total`) or should it be in `DrainTailsToWatermark`? | Logging/metric placement | -| Q2 | Channel capacity: 8 per GitTarget feels generous. Should it be bounded differently? | Backpressure vs. blocking reconciler on `Enqueue` | -| Q3 | Should the coordinator skip the barrier entirely if rv_C is missing from the annotation (restart + annotation lost)? Or finalize with `barrierReached=false`? | Restart degrade behavior | -| Q4 | The annotation `configbutler.ai/finalize-watermark-rv` — is a separate `metadata` Patch acceptable, or should rv_C go into the CRD status (requires schema bump)? | CRD versioning | - -The answers to Q3 and Q4 are the most load-bearing before writing the implementation. - ---- - -## 12. As landed (C-B2, 2026-06-11): the simplifications - -The implementation kept this doc's per-type snapshot (§3) and ordering invariant (§1) -but dropped three pieces of machinery. The reasoning mirrors the Option-A timeout memo's -own rejection of its option C: CommitRequests are rare, so blocking one dedicated -reconcile worker for ≤15 s is fine, and everything built to avoid that blocking was -weight without benefit. - -1. **No FinalizeCoordinator.** The reconciler runs barrier → finalize → status **inline** - in one `Reconcile` invocation, and the controller is registered with - `MaxConcurrentReconciles: 1`. A single worker serializes concurrent CommitRequests in - workqueue (≈ creation) order — exactly the FIFO the coordinator channel would have - provided, because the channel was filled in reconcile order anyway. The §1 invariant - holds by construction: no two barrier/finalize pairs ever interleave. -2. **No annotation, no persisted rv_C.** Every attempt (first reconcile, conflict retry, - post-restart redelivery) calls `TakeTypeSnapshot` fresh. A later snapshot waits on a - **superset** of the create-time entries — conservative, still bounded by the 15 s - timeout, and correct without any cross-restart state. (§8's own fallback analysis - already proved this safe; landing made the fallback the only path.) -3. **The author comes from the CR's own audit event — attribution as a state.** §3's - `finalizeRequest.author` field never named its source; the landed design makes it - explicit: the reconciler polls `…commitrequests:audit:stream` (which `mirrorByType` - already fills) for the CR's create event, takes `resolveUserInfo(event).Username`, - and only then finalizes with the strict author match. Unattributable within 60s → - `Failed`, fail closed. The same wait re-anchors the trigger on the audit ingest path, - closing the watch-event-outruns-ingest race the first e2e run exposed — see the DEC-B - "as landed" note in [canonical-stream-retirement.md](../architecture.md) §4. - A new `spec.delaySeconds` (0–300) optionally holds the finalize as an extra collect - window after creation. - -The stale-cache hazard that motivated the coordinator's UID bookkeeping is handled by an -**uncached APIReader re-read** before any work: a watch echo whose cached object still -says `WaitingForAuditEvent` after the terminal phase was written short-circuits on the -live read. A crash between finalize and status write re-runs the idempotent flow on -redelivery and lands `NoOpenWindow` — the same behaviour the consumer's XAUTOCLAIM -redelivery had. - -The §11 questions resolve as: **Q1** — the metric increments in -`EventRouter.FinalizeAtWatermark` next to the degrade log (one place, covers any future -caller); **Q2/Q3/Q4** — moot (no channel, no annotation, no persisted watermark). diff --git a/docs/spec/deletecollection-attribution-expander.md b/docs/spec/deletecollection-attribution-expander.md index cee3d8f4..6d70bdc8 100644 --- a/docs/spec/deletecollection-attribution-expander.md +++ b/docs/spec/deletecollection-attribution-expander.md @@ -27,8 +27,8 @@ When someone runs `kubectl delete configmaps --all -n team-a`, the API server em with no special code. Two things are *not* free: - **Attribution.** Each per-object removal runs the resolver, finds no fact (the collection event is name-less, so - it stores nothing today — §4), and ships as **committer**. "Alice deleted these 12 configmaps" is recorded as - the operator. The **expander** (§5) closes this. + it stores nothing today — §4), and ships with the explicit **unresolved** author. "Alice deleted these 12 + configmaps" is visibly unresolved rather than silently credited to the operator. The **expander** (§5) closes this. - **Finalizers — and this is where the design got interesting.** An object with a finalizer is *not* removed by the delete; it gets a `deletionTimestamp` and lingers in `Terminating` until a controller clears the finalizer. An earlier revision of this doc tried to defer attribution to that eventual removal and fight the resulting @@ -149,9 +149,9 @@ variants are skipped when no RV is supplied, which is precisely right since the has a name, so `RecordFact` already writes its uid-only fact; with §2's intent rule, the finalizer single-delete is now removed and attributed at intent time too — for free, no expander needed. The expander exists **only** for the name-less collection case. -- **The conservative resolver fails closed** — multiple authors on one key → `AttributionConflict` → committer - ([storeFactKey](../../internal/queue/attribution_index.go#L403)). Governing rule: **a wrong author is worse than - no author.** +- **The conservative resolver fails closed** — multiple authors on one key → no usable attribution fact → the + explicit unresolved author ([storeFactKey](../../internal/queue/attribution_index.go#L403)). Governing rule: + **a wrong author is worse than no author.** - **The grace window** absorbs a watch event that arrives before its audit fact ([author_resolver.go:40](../../internal/watch/author_resolver.go#L40)). @@ -206,19 +206,19 @@ list**. We know the actor, type, maybe namespace, maybe a label selector, and ro objects**. The owner's trap: *in a few seconds you could see more than one `deletecollection` that "fits."* - **Option A — keep it around and guess by scope.** Reject. Two independent mis-attribution modes: (1) two - actors deleting in the same `(type, namespace)` window → honest answer is conflict→committer, so it degrades to - committer under exactly the load that makes the case interesting; (2) even a single collection delete would + actors deleting in the same `(type, namespace)` window → honest answer is conflict→unresolved, so it degrades to + the explicit unresolved author under exactly the load that makes the case interesting; (2) even a single collection delete would capture an *unrelated* plain `kubectl delete configmap x` in the same window. Selector re-matching narrows but doesn't remove it (selectors overlap; empty selector matches all). Violates "a wrong author is worse than no author." -- **Option B — `Co-authored-by` floor.** Honest credit under ambiguity (multiple trailers), git author stays - committer. But in watch-first the deletes are driven by watch events, not the audit cause, so it needs real +- **Option B — `Co-authored-by` floor.** Honest credit under ambiguity (multiple trailers), Git author stays + unresolved. But in watch-first the deletes are driven by watch events, not the audit cause, so it needs real plumbing to carry a scope-cause into the commit-window builder. A deliberate fast-follow. -- **Option C — commit as committer, document the limit.** v1. +- **Option C — commit as the explicit unresolved author, document the limit.** v1. **v1: Option C.** The hard case is a narrow intersection (aggregated/hollow body ∧ supports `deletecollection` ∧ -watched ∧ attributed-author mode), and the failure is *degraded attribution*, not wrong state or wrong author — which is -the correct conservative outcome. **Ship §2 + §5 now; Option B is the named fast-follow; Option A is rejected.** +watched ∧ attributed-author mode), and the failure is *degraded attribution*, not wrong state or a guessed author — which +is the correct conservative outcome. **Ship §2 + §5 now; Option B is the named fast-follow; Option A is rejected.** ## 7. Recommendation at a glance @@ -227,14 +227,14 @@ the correct conservative outcome. **Ship §2 + §5 now; Option B is the named fa | Any delete (single or collection member) | **Remove file at intent time; never commit `deletionTimestamp`** (§2) | Git is intent; reversible invariant; manifests stay re-appliable. | | Finalizer object | **Removed immediately, attributed to the delete-requester**; finalizer cleanup is runtime no-op in Git | Reframe dissolves the old delay/conflict; controllers still finalize in-cluster. | | Collection delete, body present | **Per-UID expander (§5)** credits the actor on each removal | API server states "these exact objects, by this user." | -| Collection delete, hollow body | **Committer (Option C, §6)** | Narrow; degraded attribution beats a wrong author. | +| Collection delete, hollow body | **Explicit unresolved author (Option C, §6)** | Narrow; degraded attribution beats a wrong author. | | Stuck `Terminating` | **Operational status/metric** (§2.5), file already absent | Don't pollute intent with runtime state. | ## 8. Observability & diagnostics - **`AttributionExactDeleteCollectionItem`** flows onto `AttributionResolutionsTotal{result=…}` via `recordAttributionResolution` ([author_resolver.go:169](../../internal/watch/author_resolver.go#L169)) — a - dashboard can show collection-member precise attributions vs. committer fallbacks. + dashboard can show collection-member precise attributions vs. unresolved outcomes. - **Expander write counter** (`op="deletecollection_expanded"`) on `AttributionFactEventsTotal` via the existing `recordFactEvent` hook ([attribution_index.go:433](../../internal/queue/attribution_index.go#L433)). - **Stuck-finalizer / terminating diagnostics** (§2.5): surface long-`Terminating` watched objects whose files we diff --git a/docs/spec/status-conditions-guide.md b/docs/spec/status-conditions-guide.md index 2f6273a8..84c8d4b0 100644 --- a/docs/spec/status-conditions-guide.md +++ b/docs/spec/status-conditions-guide.md @@ -78,28 +78,29 @@ trio; the bounded "did the work actually happen" signal lives on the `Pushed` do ```go const ( - TypeReady = "Ready" // True at a non-error terminal outcome (committed, or a benign no-commit) - TypeReconciling = "Reconciling" // True through both progress waits (reason names which) - TypeStalled = "Stalled" // True when the finalize failed (kstatus Failed) - TypeAttributed = "Attributed" // True once the author is settled (immediately True when not required) - TypePushed = "Pushed" // True once the commit is in the remote repository + TypeReady = "Ready" // True at a non-error terminal outcome + TypeReconciling = "Reconciling" // True while the close delay/finalize is in progress + TypeStalled = "Stalled" // True when the finalize failed (kstatus Failed) + TypeAuthorAttributed = "AuthorAttributed" // Whether admission captured the command submitter + TypePushed = "Pushed" // True once the commit is in the remote repository ) ``` -A request passes through two distinct, observable progress waits, both `Reconciling=True` but told apart -by the `Reconciling`/`Ready` reason and the `Attributed` condition. +A request has one progress wait: its optional `closeDelaySeconds` collect window, followed by finalization +and push. `AuthorAttributed` settles at first sight because command authorship is captured synchronously +at admission; it never has an `Unknown` or audit-wait state. Canonical reads: -* waiting for the create audit event (attributed-author mode only): `Reconciling=True` reason - `WaitingForAuditEvent`, `Attributed=Unknown` → kstatus InProgress -* waiting for the close delay (author settled, attached — the `closeDelaySeconds` collect window, then - commit and push): `Reconciling=True` reason `WaitingForCloseDelay`, `Attributed` settled, `Pushed=False` - → kstatus InProgress +* waiting for the close delay: `Reconciling=True` reason `WaitingForCloseDelay`, `AuthorAttributed` + settled, `Pushed=Unknown` → kstatus InProgress * committed: `Ready=True`, `Pushed=True`, `Stalled=False`, reason `Committed` → kstatus Current * benign no-commit (nothing to save / already present / foreign open window): `Ready=True`, `Pushed=False`, `Stalled=False`, with the specific reason on `Ready` → kstatus Current (a correct, non-error outcome) * failed finalize: `Ready=False`, `Stalled=True`, reason `FinalizeFailed` → kstatus Failed -* configured-author mode sets `Attributed=True` (`AttributionNotRequired`) immediately; attributed-author mode leaves - it `Unknown` until the create audit event names the author, or `False` (`AuditEventNotObserved`) if it - never arrives and the commit is authored by the configured committer + +`AuthorAttributed=True` (`AttributedFromAdmission`) means the command submitter was captured. `False` +(`CommitterFallback`) means no admission author record exists, so the request claims no actor and can attach +only to an unnamed window. That condition does not itself determine the Git author: the attached watch +window is configured-author (the committer) when watch attribution is disabled, or explicitly unresolved +when watch attribution ran but could not name an actor. diff --git a/docs/tasks/bounded-queue.md b/docs/tasks/bounded-queue.md deleted file mode 100644 index 2180c800..00000000 --- a/docs/tasks/bounded-queue.md +++ /dev/null @@ -1,221 +0,0 @@ -# gitops-reverser feature request - -## Title - -Bound and monitor the durable audit queue by default - -## Problem - -The durable audit Redis/Valkey stream can grow without bound. On the demo -cluster this produced a multi-GB Valkey dataset and repeated Valkey restarts, -which temporarily broke audit ingestion. - -Live observation: - -```text -namespace: gitops-reverser -pod: valkey-867df97d8b-cckmm -restartCount: 9 -lastState.terminated.exitCode: 137 -lastState.terminated.reason: Error -``` - -Valkey startup log after one restart: - -```text -RDB memory usage when created 6124.68 Mb -Done loading RDB, keys loaded: 13359, keys expired: 645. -DB loaded from disk: 14.964 seconds -Ready to accept connections tcp -``` - -During that loading window, `gitops-reverser` could not enqueue or consume -audit events: - -```text -XREADGROUP failed: dial tcp 10.96.98.248:6379: connect: connection refused -failed to claim audit decision ... LOADING Valkey is loading the dataset in memory -``` - -This is operationally risky because the audit queue is the handoff between the -Kubernetes API server audit webhook and the controller. A bloated queue can -turn normal demo traffic into a queue outage, and queue outages can lead to -missed or delayed Git commits. - -## Current behavior - -The chart default is unbounded: - -```yaml -queue: - redis: - maxLen: 0 -``` - -This renders: - -```text ---audit-redis-max-len=0 -``` - -`0` disables stream trimming. The debug stream also defaults to unbounded when -enabled: - -```yaml -webhook: - audit: - debugStream: - maxLen: 0 -``` - -For a long-running cluster or a noisy audit policy, this allows unbounded data -growth in Valkey. - -## Expected behavior - -`gitops-reverser` should have a bounded, production-safe default for audit -stream storage. A default install should not be able to grow Valkey to several -GB without the operator explicitly opting into that behavior. - -The queue should provide enough retention for normal controller catch-up and -short outages, but it should not act as an infinite audit archive. Kubernetes -audit logs or object storage are better places for long-term raw audit -retention. - -## Suggested implementation - -### 1. Change chart defaults - -Set a bounded default for the hot audit stream, for example: - -```yaml -queue: - redis: - maxLen: 10000 -``` - -Treat the default as an operational starting point rather than a universal -capacity answer. Sizing should be based on expected audit events per second, -the outage/catch-up window the queue should tolerate, and a safety factor. - -If exact retention is not required, keep using approximate Redis stream -trimming for performance: - -```text -XADD ... MAXLEN ~ 10000 -``` - -This is a deliberate tradeoff: bounded streams protect Valkey memory and reload -time, but they can trim entries before a slow or stopped consumer has processed -them. The implementation should make that tradeoff visible through metrics and -documentation. - -Keep `0` available as an explicit opt-out for users who really want unbounded -retention: - -```yaml -queue: - redis: - maxLen: 0 # explicit unbounded mode -``` - -### 2. Bound the debug stream too - -If `webhook.audit.debugStream.enabled=true`, the debug stream should also have -a bounded default. For example: - -```yaml -webhook: - audit: - debugStream: - enabled: false - maxLen: 10000 -``` - -The debug stream is especially dangerous because it stores every decoded audit -event before filtering, joining, or dropping. - -### 3. Surface a warning for unbounded mode - -At startup, log a clear warning when either stream is unbounded, whether the -configuration came from Helm values or direct binary flags: - -```text -audit redis stream max length is 0; queue retention is unbounded -``` - -This makes accidental unbounded production installs visible in logs. - -### 4. Add metrics - -Expose queue size and trimming-relevant metrics, for example: - -- `gitops_reverser_audit_queue_stream_length` -- `gitops_reverser_audit_queue_consumer_lag` -- `gitops_reverser_audit_queue_pending_entries` -- `gitops_reverser_audit_queue_oldest_entry_age_seconds` -- `gitops_reverser_audit_queue_oldest_pending_age_seconds` -- `gitops_reverser_audit_debug_stream_length` - -These should be low-cardinality and scoped to the configured stream names. -`pending_entries` alone is not enough: Redis pending entries are claimed but -unacked messages, while consumer-group lag captures unread backlog that may be -trimmed before consumption. - -### 5. Document retention semantics - -Document that: - -- the Redis/Valkey stream is a durable work queue, not the long-term audit log; -- `maxLen` is an approximate upper bound if Redis `MAXLEN ~` is used; -- too-low values can drop events before slow consumers catch up; -- too-high values can create memory pressure and long restart/reload times. - -## Configuration recommendation for current deployments - -Until defaults change, set this explicitly: - -```yaml -queue: - redis: - maxLen: 10000 -``` - -For demo/dev deployments with debug stream enabled: - -```yaml -webhook: - audit: - debugStream: - enabled: true - maxLen: 10000 -``` - -## Acceptance criteria - -- A fresh chart install renders a non-zero `--audit-redis-max-len` by default. -- Debug stream, when enabled, renders a non-zero - `--audit-debug-redis-max-len` by default. -- Setting either value to `0` still works, but logs a startup warning. -- Unit tests cover bounded `MAXLEN ~` behavior and explicit unbounded `0` - behavior for the canonical and debug streams. -- Chart tests or rendered-manifest assertions cover the bounded defaults and - explicit `0` opt-out. -- Queue stream length, consumer-group lag, pending entries, and oldest - entry/pending ages are observable via metrics. -- Helm README and project documentation explain the retention tradeoff and - recommend sizing guidance. -- A long-running demo cluster under normal audit traffic does not grow Valkey - into multi-GB RDB files without explicit unbounded configuration. - -## Related bug - -This was discovered while investigating generated-name `CommitRequest` objects -stuck in `WaitingForAuditEvent`. The queue growth was not the root cause of -that bug, but it made the failure mode worse by causing audit ingestion outages. - -See: - -```text -docs/tasks/generated-name-support.md -``` diff --git a/hack/attribution-diagnostics.sh b/hack/attribution-diagnostics.sh index 6135ad40..850efc59 100755 --- a/hack/attribution-diagnostics.sh +++ b/hack/attribution-diagnostics.sh @@ -1,11 +1,10 @@ #!/usr/bin/env bash # Post-run diagnostics for the author-attribution path. # -# Attribution fails SILENTLY: when no audit fact matches within the grace window -# (watch.DefaultAttributionGraceWindow, 3s), the commit is authored by the configured -# committer instead of the real actor. Because git.DefaultCommitterName is ALSO the -# configured-author identity, the resulting commit is indistinguishable from a correct -# configured-author commit by inspection alone. This script makes the difference visible. +# When no audit fact matches within the grace window (watch.DefaultAttributionGraceWindow, +# 3s), the commit is visibly authored as `unknown (attribution unresolved)` rather than as +# the configured committer. For a live change that should be attributable, that author is a +# concrete signal to check audit-attribution configuration and delivery. # # It answers the one question that splits the diagnosis in two: # @@ -30,7 +29,7 @@ FILTER="${1:-}" kc() { kubectl --context "${CTX}" "$@"; } echo "== attribution resolution outcomes (Prometheus) ==" -echo " absent = fell back to the configured committer; the actor's name was LOST." +echo " absent = Git author was unknown (attribution unresolved); no actor was named." prom_pod_port=9097 kc -n "${PROM_NS}" port-forward svc/prometheus-operated ${prom_pod_port}:9090 >/dev/null 2>&1 & pf=$! @@ -48,8 +47,9 @@ for k,v in sorted(rows, key=lambda x:-x[1]): print(f" {'TOTAL':32s} {total:8.0f}") absent=dict(rows).get("absent",0) if absent: - print(f"\n >> {absent:.0f} resolution(s) lost the actor. Each is a commit authored by the") - print( " >> committer instead of the human/SA that caused the change.") + print(f"\n >> {absent:.0f} resolution(s) produced the explicit unresolved Git author.") + print( " >> For a change that should be attributable, inspect the audit policy, route,") + print( " >> source identity, and Redis delivery below.") ' || echo " (query failed)" echo @@ -73,7 +73,7 @@ vc() { kc -n "${VALKEY_NS}" exec "${pod}" -c valkey -- valkey-cli -a "${pw}" --n keys="$(vc --scan --pattern '*author:v1:audit:*' | { [ -n "${FILTER}" ] && grep -F "${FILTER}" || cat; } | head -40)" if [ -z "${keys}" ]; then echo " no facts matching ${FILTER:-}" - echo " >> If a commit lost its author AND no fact exists, the apiserver never delivered it:" + echo " >> If a commit has the unresolved author AND no fact exists, check audit delivery:" echo " >> check audit-webhook-batch-max-wait/-size and the /audit-webhook/ route." exit 0 fi @@ -100,6 +100,6 @@ PY done echo -echo " Reading this: a fact PRESENT here for an object whose commit lost its author means" +echo " Reading this: a fact PRESENT here for an object whose commit has the unresolved author means" echo " the fact arrived, just not within the 3s grace — the grace window (or the audit" -echo " batching delay ahead of it) is the problem, not delivery." +echo " batching delay ahead of it) is the problem, not fact delivery." diff --git a/internal/controller/commitrequest_controller.go b/internal/controller/commitrequest_controller.go index d4336f13..37a643da 100644 --- a/internal/controller/commitrequest_controller.go +++ b/internal/controller/commitrequest_controller.go @@ -22,15 +22,15 @@ import ( ) // CommitRequestFinalizer is the EventRouter seam the reconciler drives, using the -// attach-then-poll protocol (docs/spec/commitrequest-design.md §6.4.3): +// attach-then-poll protocol (docs/spec/commitrequest-design.md): // ServiceCommitRequest registers the attach idempotently on the GitTarget's branch // worker (bind the message to the author's open window, finalize after the grace) // and returns the request's current outcome — resolved=false means keep polling. // watch.EventRouter satisfies it without adaptation. // -// There is no watermark barrier (§6.3): UC1 is covered by the human gap between the +// There is no watermark barrier: the interactive case is covered by the human gap between the // edit and the save, UC2 by the collect-grace. The grace is anchored at attribution -// — the worker stamps finalizeAt = receipt + closeDelaySeconds (§6.4.4) — so the +// — the worker stamps finalizeAt = receipt + closeDelaySeconds — so the // controller no longer holds the finalize itself. type CommitRequestFinalizer interface { ServiceCommitRequest(ctx context.Context, attach git.AttachCommitRequest) (git.FinalizeResult, bool, error) @@ -39,9 +39,9 @@ type CommitRequestFinalizer interface { // CommandAuthorLookup resolves the author of a CommitRequest from the submitter // captured at admission by the validate-operator-types webhook, keyed by the persisted // object's UID. *queue.CommandAuthorStore satisfies it without adaptation. The lookup -// is present-or-never (docs/spec/commitrequest-admission-authorship.md §2): a miss is +// is present-or-never (docs/spec/commitrequest-admission-authorship.md): a miss is // immediate and final — the webhook is not configured (or a best-effort write missed) — -// and the controller finalizes as the committer with no wait. +// and the request claims no actor with no wait. type CommandAuthorLookup interface { LookupCommandAuthor(ctx context.Context, uid types.UID) (queue.CommandAuthor, bool) } @@ -58,26 +58,26 @@ const resolveTimeoutMessage = "the CommitRequest finalize did not resolve within const ( // commitRequestPollInterval is the requeue cadence while polling the worker - // for the attached request's outcome (attach-then-poll, §6.4.3). + // for the attached request's outcome (attach-then-poll). commitRequestPollInterval = 2 * time.Second // commitRequestResolveTimeout bounds the attach-then-poll wait, measured from // object creation: it must cover the maximum collect-grace (closeDelaySeconds ≤ 300s, // anchored at attribution) and the push cooldown plus retries. Authorship is now - // settled synchronously at first sight (no attribution wait, §2), so the former + // settled synchronously at first sight (no attribution wait), so the former // +60s attribution component is gone. Past it, a request the worker never resolved // (e.g. a vanished worker) fails closed instead of polling forever. commitRequestResolveTimeout = 300*time.Second + 120*time.Second ) // CommitRequestReconciler drives a CommitRequest through its state machine -// (docs/spec/commitrequest-design.md §6.4 and -// docs/spec/commitrequest-admission-authorship.md §5): +// (docs/spec/commitrequest-design.md and +// docs/spec/commitrequest-admission-authorship.md): // // 1. ATTRIBUTE — a single synchronous read of the submitter captured at admission -// (present-or-never, §2). A hit names that submitter as the author -// (AuthorAttributed=True); a miss falls back to the configured committer -// immediately (AuthorAttributed=False). There is no wait and no requeue for the +// (present-or-never). A hit names that submitter as the author +// (AuthorAttributed=True); a miss claims no actor immediately +// (AuthorAttributed=False). There is no wait and no requeue for the // author: the record is written before the object is visible, so waiting cannot // help. // 2. ATTACH + POLL — the instant the author is settled, send the attach to the @@ -99,7 +99,8 @@ type CommitRequestReconciler struct { // Finalizer attaches the request to the author-bound open window and reports // its outcome; AuthorLookup resolves the submitter captured at admission. When // AuthorLookup is nil (the validate-operator-types webhook is disabled), requests - // finalize as the configured committer — immediately, with AuthorAttributed=False. + // claim no actor immediately, with AuthorAttributed=False. The attached window + // determines the eventual Git author. Finalizer CommitRequestFinalizer AuthorLookup CommandAuthorLookup } @@ -125,9 +126,9 @@ func (r *CommitRequestReconciler) Reconcile(ctx context.Context, req ctrl.Reques return ctrl.Result{}, nil } - // 1. ATTRIBUTE: settle the commit author synchronously (present-or-never, §2). - // A hit names the admission submitter; a miss is the configured committer. Either - // way the decision is final — there is no wait and no requeue for the author. + // 1. ATTRIBUTE: settle the request's actor synchronously (present-or-never). + // A hit names the admission submitter; a miss claims no actor. Either way the + // decision is final — there is no wait and no requeue for the author. author, attribution := r.attributeAuthor(ctx, commitRequest) // First sight: stamp the still-running conditions so the object reports its @@ -143,7 +144,7 @@ func (r *CommitRequestReconciler) Reconcile(ctx context.Context, req ctrl.Reques // 2. ATTACH + POLL: register the attach idempotently the instant we attribute // (no controller-side delay — the worker anchors the grace at attribution, - // §6.4.4) and poll the outcome. + // close-delay contract) and poll the outcome. result, resolved, serviceErr := r.Finalizer.ServiceCommitRequest(ctx, git.AttachCommitRequest{ Namespace: commitRequest.Namespace, Name: commitRequest.Name, @@ -182,21 +183,21 @@ func (r *CommitRequestReconciler) Reconcile(ctx context.Context, req ctrl.Reques } // attributeAuthor settles the commit author with a single synchronous lookup of the -// submitter captured at admission (present-or-never, §2). It never waits: a nil -// AuthorLookup (the validate-operator-types webhook is disabled) or a miss both resolve to -// the configured committer immediately. The miss case is final — the record is written -// before the object is visible, so there is no asynchronous arrival to wait for. +// submitter captured at admission (present-or-never). It never waits: a nil +// AuthorLookup (the validate-operator-types webhook is disabled) or a miss both claim no +// actor immediately. The miss case is final — the record is written before the object is +// visible, so there is no asynchronous arrival to wait for. // // The lookup result is logged at Info: it is the counterpart to the admission handler's // "recorded command author" line, so a hit/miss pair makes the whole capture→read path -// legible (the first thing to check when a CommitRequest commits as the committer). +// legible (the first thing to check when a request unexpectedly claims no actor). func (r *CommitRequestReconciler) attributeAuthor( ctx context.Context, commitRequest *configbutleraiv1alpha3.CommitRequest, ) (queue.CommandAuthor, commitRequestAttribution) { log := logf.FromContext(ctx).WithName("CommitRequestReconciler") if r.AuthorLookup == nil { - log.Info("command-author lookup disabled (validate-operator-types webhook off); committing as committer", + log.Info("command-author lookup disabled (validate-operator-types webhook off); request claims no actor", "name", client.ObjectKeyFromObject(commitRequest), "uid", commitRequest.UID) // Capture is off, so nothing was attempted — distinct from a capture that ran and // found no record, which is what attributionCommitter now means. @@ -207,7 +208,7 @@ func (r *CommitRequestReconciler) attributeAuthor( "name", client.ObjectKeyFromObject(commitRequest), "uid", commitRequest.UID, "author", author.Author) return author, attributionFromAdmission } - log.Info("no admission command-author record found; committing as committer", + log.Info("no admission command-author record found; request claims no actor", "name", client.ObjectKeyFromObject(commitRequest), "uid", commitRequest.UID) return queue.CommandAuthor{}, attributionCommitter } @@ -344,9 +345,9 @@ func (r *CommitRequestReconciler) writeTerminalStatus( // multi-CommitRequest ordering design — concurrent CommitRequests for the same // GitTarget are serialized exactly as a dedicated finalize-coordinator // goroutine would serialize them, without the extra moving parts (see -// docs/spec/commitrequest-multi-finalize-design.md). +// docs/spec/commitrequest-design.md). // -// Restart recovery is best-effort by design (commitrequest-design.md §6.6): the +// Restart recovery is best-effort by design: the // message is durable in spec.message, so on restart any non-terminal request is // re-reconciled — author-resolved from the admission cache when present and // re-attached — which heals the common cases. The one knowingly-accepted gap is a request whose diff --git a/internal/controller/commitrequest_controller_unit_test.go b/internal/controller/commitrequest_controller_unit_test.go index 490868ec..5688c843 100644 --- a/internal/controller/commitrequest_controller_unit_test.go +++ b/internal/controller/commitrequest_controller_unit_test.go @@ -258,11 +258,10 @@ func TestCommitRequestReconcile_FinalizeErrorBecomesFailed(t *testing.T) { requireCondition(t, got, ConditionTypePushed, metav1.ConditionFalse, "") } -// A lookup miss (the validate-operator-types webhook recorded no author for this object) -// resolves to the configured committer immediately — present-or-never, no wait. Even a +// A lookup miss means the request claims no actor immediately — present-or-never, no wait. Even a // freshly created CommitRequest attaches at once with a blank author and records -// AuthorAttributed=False (CommitterFallback). -func TestCommitRequestReconcile_LookupMissFallsBackToCommitter(t *testing.T) { +// AuthorAttributed=False (CommitterFallback). The matching window determines the Git author. +func TestCommitRequestReconcile_LookupMissClaimsNoActor(t *testing.T) { cr := newCommitRequest("save-fresh") cr.CreationTimestamp = metav1.Now() // fresh: the old path would have requeued here c := newCommitRequestClient(t, nil, cr) @@ -278,7 +277,7 @@ func TestCommitRequestReconcile_LookupMissFallsBackToCommitter(t *testing.T) { assert.Zero(t, res.RequeueAfter, "a miss is final; the request must not poll for an author") assert.Equal(t, 1, lookup.calls, "the author is looked up exactly once, synchronously") require.Len(t, f.calls, 1, "the attach is sent immediately on a miss") - assert.Empty(t, f.calls[0].Author, "a miss attaches with a blank author (the configured committer)") + assert.Empty(t, f.calls[0].Author, "a miss attaches with a blank, unnamed actor") got := fetchCommitRequest(t, c, "save-fresh") requireCondition(t, got, ConditionTypeReady, metav1.ConditionTrue, crReasonCommitted) @@ -380,8 +379,8 @@ func TestCommitRequestReconcile_StaleCacheEchoDoesNotRefinalize(t *testing.T) { } // Webhook-disabled mode (no AuthorLookup) never waits: a freshly created CommitRequest -// attaches immediately with a blank author and commits as the configured committer, -// recording AuthorAttributed=False (CommitterFallback). +// attaches immediately with a blank author and records AuthorAttributed=False +// (CommitterFallback). The matching window determines the Git author. func TestCommitRequestReconcile_ConfiguredAuthorCommitsWithoutWaiting(t *testing.T) { cr := newCommitRequest("save-configured-author") cr.CreationTimestamp = metav1.Now() // fresh: a waiting path would requeue instead of commit @@ -514,7 +513,7 @@ func TestApplyFinalizeResultToStatus(t *testing.T) { assert.Equal(t, "main", cr.Status.Branch) }) - t.Run("no window in grace is a benign ready (committer fallback)", func(t *testing.T) { + t.Run("no window in grace is a benign ready (no admission author)", func(t *testing.T) { var cr configv1alpha3.CommitRequest applyFinalizeResultToStatus(&cr, git.FinalizeResult{Outcome: git.FinalizeNoOpenWindow, Branch: "main"}, nil, attributionCommitter) diff --git a/internal/controller/commitrequest_finalize.go b/internal/controller/commitrequest_finalize.go index c9d77268..f934b10a 100644 --- a/internal/controller/commitrequest_finalize.go +++ b/internal/controller/commitrequest_finalize.go @@ -55,7 +55,7 @@ const ( attributionFromAdmission commitRequestAttribution = iota // attributionCommitter means no admission author record exists — the // validate-operator-types webhook is not configured (or did not record one) — so the - // commit is authored by the configured committer. + // request does not claim an actor. attributionCommitter // attributionNotAttempted means command-author capture is switched off entirely (the // validate-operator-types webhook is not running), so no submitter was ever sought. It is @@ -136,13 +136,13 @@ func setCommitRequestAttributed(cr *configv1alpha3.CommitRequest, attribution co // webhook that is configured and simply missed sends operators to the wrong place. setCommitRequestCondition(cr, ConditionTypeAuthorAttributed, metav1.ConditionFalse, crReasonCommitterFallback, - "no admission author record was found for this request; committed as the "+ - "configured committer") + "no admission author record was found for this request; it can attach only to a "+ + "window with no named actor") case attributionNotAttempted: setCommitRequestCondition(cr, ConditionTypeAuthorAttributed, metav1.ConditionFalse, crReasonCommitterFallback, "command-author capture is disabled (the validate-operator-types webhook is not "+ - "configured); committed as the configured committer") + "configured); the request can attach only to a window with no named actor") } } diff --git a/internal/controller/commitrequest_kstatus_test.go b/internal/controller/commitrequest_kstatus_test.go index ba13756b..d77e210f 100644 --- a/internal/controller/commitrequest_kstatus_test.go +++ b/internal/controller/commitrequest_kstatus_test.go @@ -22,7 +22,7 @@ func TestCommitRequestKstatusContract(t *testing.T) { wantMsg string }{ { - name: "committer fallback in the close-delay wait (AuthorAttributed=False is not a failure)", + name: "missing admission author in the close-delay wait (AuthorAttributed=False is not a failure)", conds: []map[string]interface{}{ conditionMap(ConditionTypeReady, "False", crReasonWaitingForCloseDelay, closeDelayMessage), conditionMap(ConditionTypeReconciling, "True", crReasonWaitingForCloseDelay, closeDelayMessage), diff --git a/internal/controller/constants.go b/internal/controller/constants.go index f5146534..770c77b2 100644 --- a/internal/controller/constants.go +++ b/internal/controller/constants.go @@ -46,9 +46,9 @@ const ( // was named from the submitter captured at admission. It is binary and immediately // settled (no Unknown, no timeout): True (AttributedFromAdmission) when the // validate-operator-types webhook recorded the submitter, False (CommitterFallback) when - // no admission record exists — the webhook is not configured — and the commit is - // authored by the configured committer. False is not a failure and does not affect - // Ready (docs/spec/commitrequest-admission-authorship.md §5). + // no admission record exists. False is not a failure and does not affect Ready: the + // request claims no actor and can attach only to an unnamed watch window + // (docs/architecture.md#commitrequest-finalize). ConditionTypeAuthorAttributed = "AuthorAttributed" // ConditionTypePushed indicates whether a CommitRequest's commit reached the // remote repository. diff --git a/internal/git/attribution_outcome_test.go b/internal/git/attribution_outcome_test.go index 4b20e6b3..9218d36d 100644 --- a/internal/git/attribution_outcome_test.go +++ b/internal/git/attribution_outcome_test.go @@ -170,13 +170,13 @@ func TestPendingCommitRequest_UnresolvedRequestAttachesToUnresolvedWindow(t *tes "an unresolved window carries no author; the sentinel is applied at the write path") request := &pendingCommitRequest{ - author: "", // a fallback CommitRequest has no author + author: "", // an unnamed CommitRequest claims no actor attribution: AttributionUnresolved, gitTargetName: "target", gitTargetNamespace: "default", } assert.True(t, request.matchesWindow(window), - "a fallback CommitRequest must attach to the unresolved window it was meant for") + "an unnamed CommitRequest must attach to the unresolved window it was meant for") // The default deployment: command-author capture is off, so the request says "not // attempted" while the window says "unresolved". Neither names an actor, so they match. diff --git a/internal/git/commit_request_attach.go b/internal/git/commit_request_attach.go index c4f505c8..d0368d4b 100644 --- a/internal/git/commit_request_attach.go +++ b/internal/git/commit_request_attach.go @@ -43,8 +43,7 @@ type FinalizeResult struct { } // AttachCommitRequest is the "bind this CommitRequest's message to the author's -// open window, then finalize that window after the grace" work item (§6.4 of -// docs/spec/commitrequest-design.md). It rides the same per-worker FIFO +// open window, then finalize that window after the grace" work item. It rides the same per-worker FIFO // event queue as resource events, so by audit-stream ordering it is processed // after every earlier write for that worker. Re-sends are idempotent: the worker // keys pending requests by identity and keeps the first finalize deadline. @@ -56,8 +55,8 @@ type AttachCommitRequest struct { Name string UID string - // Author is the effective user that requested the finalize, attributed from - // the CommitRequest's own create audit event. Only a window whose author + // Author is the effective user that requested the finalize, captured from + // validating admission. Only a window whose author // matches is attached; this binds "the open window" to "the requesting // author's open window". Author string @@ -76,7 +75,7 @@ type AttachCommitRequest struct { Message string // CloseDelaySeconds is the close-delay collect window: the worker closes the // attached window and finalizes it at receipt + CloseDelaySeconds (the delay is - // anchored at attribution, §6.4.4). + // anchored at attach receipt). CloseDelaySeconds int32 } diff --git a/internal/git/commit_request_attach_loop.go b/internal/git/commit_request_attach_loop.go index 8ff62a75..d3658eb3 100644 --- a/internal/git/commit_request_attach_loop.go +++ b/internal/git/commit_request_attach_loop.go @@ -6,8 +6,7 @@ import ( "time" ) -// This file holds the worker-loop side of CommitRequest eager attach (§6.4 of -// docs/spec/commitrequest-design.md). All methods run on the single event +// This file holds the worker-loop side of CommitRequest eager attach. All methods run on the single event // loop goroutine, so pendingCRs needs no locking; only the resolved-outcome table // (BranchWorker.crOutcomes) crosses goroutines and is mutex-guarded. // @@ -77,7 +76,7 @@ func (l *branchWorkerEventLoop) handleAttachCommitRequest(req *AttachCommitReque if _, exists := l.pendingCRs[id]; exists { return // idempotent re-send: keep the first finalize deadline. } - // Anchor the grace at receipt (≈ the attribution moment, §6.4.4), not at object + // Anchor the grace at receipt (≈ the attribution moment), not at object // creation: under a delayed ingestion pipeline this lets delaySeconds cover only // the inter-stream spread instead of the absolute latency. l.pendingCRs[id] = &pendingCommitRequest{ @@ -111,7 +110,7 @@ func (l *branchWorkerEventLoop) serviceCommitRequests() { // attachWaitingCommitRequests binds the oldest waiting same-author request to the // open window when it is unclaimed. A window carries at most one request; a second -// waits for the next window (§6.4.6). +// waits for the next window. func (l *branchWorkerEventLoop) attachWaitingCommitRequests() { if l.openWindow == nil || l.openWindow.pendingCR != nil { return diff --git a/internal/git/resync_flush.go b/internal/git/resync_flush.go index f546edf3..aaab9988 100644 --- a/internal/git/resync_flush.go +++ b/internal/git/resync_flush.go @@ -128,8 +128,7 @@ func (l *branchWorkerEventLoop) applyResync(req *ResyncRequest) { // commit is now in pendingWrites and must reach the remote — or the resync itself // committed. Any finalize that closes a window must schedule its push, or the // window's commit is stranded: committed locally but never pushed (the - // stranded-write fix, docs/spec/commitrequest-design.md §6.4.2; the - // failure analyzed in github-e2e-per-type-tail-failure-investigation.md). + // stranded-write fix in the CommitRequest window contract). // maybeSchedulePush no-ops when nothing is pending, so a pure no-op resync that // closed no window stays a no-op and does not disturb the push cooldown. if committed || closedWindow { diff --git a/internal/git/resync_push_test.go b/internal/git/resync_push_test.go index 74df2c5c..ad3460cd 100644 --- a/internal/git/resync_push_test.go +++ b/internal/git/resync_push_test.go @@ -38,7 +38,7 @@ func TestEnqueueResync_ReportsEnqueueOutcome(t *testing.T) { } // TestHandleResyncRequest_ClosedWindowIsPushedEvenWhenNoOpResync pins the -// stranded-write fix (docs/spec/commitrequest-design.md §6.4.2, §9.5): a +// stranded-write fix in the CommitRequest window contract: a // resync that closes a live commit window but commits nothing of its own must // still schedule the window's commit for push. Before the fix maybeSchedulePush // ran only when the resync itself committed, so a window the resync closed was diff --git a/internal/git/types.go b/internal/git/types.go index ed0842be..6135b030 100644 --- a/internal/git/types.go +++ b/internal/git/types.go @@ -22,7 +22,7 @@ import ( // author string is load-bearing in several places (window grouping, commit-message templates, // the author_kind metric) and overloading it to also mean "attribution failed" made a silent // failure indistinguishable from correct configured-author behaviour. See -// docs/design/attribution-outcome-and-author.md. +// docs/architecture.md#author-and-committer-identity-in-git. type AttributionOutcome string const ( @@ -285,8 +285,7 @@ type PendingWrite struct { Committed *bool // CommitRequest, when set, is the CommitRequest claiming this write: it is - // resolved Committed (with CommitSHA) once this write is pushed (§6.5 of - // docs/spec/commitrequest-design.md). It rides the write through the + // resolved Committed (with CommitSHA) once this write is pushed. It rides the write through the // push cooldown and the conflict rebase-replay, so the result follows the data. CommitRequest *commitRequestID // CommitSHA is the hash of the commit this write created, captured in diff --git a/internal/queue/attribution_index_deletecollection_test.go b/internal/queue/attribution_index_deletecollection_test.go index f6c73d6b..b7f792f0 100644 --- a/internal/queue/attribution_index_deletecollection_test.go +++ b/internal/queue/attribution_index_deletecollection_test.go @@ -123,7 +123,8 @@ func TestRecordDeleteCollectionFacts_HollowBodyWritesNothing(t *testing.T) { // TestRecordDeleteCollectionFacts_PartialListOnlyWritesPresent proves a partial body // (a large collection delete that returned only some items) attributes what it has and -// leaves the rest to the committer fallback (the watch + sweep backstop state). +// leaves the rest without a usable fact; live attributed writes use the explicit +// unresolved author for those events. func TestRecordDeleteCollectionFacts_PartialListOnlyWritesPresent(t *testing.T) { idx := newTestAttributionIndex(t) ctx := context.Background() diff --git a/internal/watch/target_watch.go b/internal/watch/target_watch.go index 867863db..40006c05 100644 --- a/internal/watch/target_watch.go +++ b/internal/watch/target_watch.go @@ -743,7 +743,7 @@ func (m *Manager) routeLiveTargetWatchEvent( // attribution index. The live object still carries its UID and resourceVersion // here (sanitize strips them inside targetWatchGitEvent), so the resolver joins on // the strongest available key. Configured-author mode (nil resolver) leaves UserInfo -// zero, so the writer authors the commit as the configured committer. +// zero, so the writer authors that commit as the configured committer. func (m *Manager) attachAuthor( ctx context.Context, event *git.Event, diff --git a/internal/watch/target_watch_test.go b/internal/watch/target_watch_test.go index 89d6ee17..18cca748 100644 --- a/internal/watch/target_watch_test.go +++ b/internal/watch/target_watch_test.go @@ -166,7 +166,7 @@ func TestRouteLiveTargetWatchEvent_ForwardsObjectEventsAsCommitter(t *testing.T) assert.Equal(t, "CREATE", event.Operation) assert.Equal(t, "target", event.GitTargetName) assert.Equal(t, "default", event.GitTargetNamespace) - assert.Empty(t, event.UserInfo.Username, "unattributed watch events commit as the configured committer") + assert.Empty(t, event.UserInfo.Username, "configured-author watch events leave the actor empty") assert.NotNil(t, event.Object) assert.Empty(t, event.Object.GetResourceVersion(), "live events are sanitized before entering Git") } diff --git a/internal/webhook/validate_operator_types_handler.go b/internal/webhook/validate_operator_types_handler.go index a9a843bc..6637876f 100644 --- a/internal/webhook/validate_operator_types_handler.go +++ b/internal/webhook/validate_operator_types_handler.go @@ -59,8 +59,8 @@ type CommandAuthorRecorder interface { // does one thing: for a command kind (a CommitRequest) it captures the authenticated // submitter into the CommandAuthorStore and always allows — pure observation with a // single side effect (a Redis upsert), never a rejection, so a user's command never -// depends on it succeeding (a missed capture degrades to a committer-authored commit; -// see docs/spec/commitrequest-admission-authorship.md §2, §4). It dispatches on the +// depends on it succeeding (a missed capture leaves the request without a claimed actor; +// see docs/spec/commitrequest-admission-authorship.md). It dispatches on the // resource (isCommandKind today), so a future config-validation branch for non-command // kinds slots in alongside without disturbing this one. type ValidateOperatorTypesHandler struct { @@ -70,7 +70,7 @@ type ValidateOperatorTypesHandler struct { // Handle records {uid → author} for an admitted command CREATE before the object // persists (the authorship invariant, §2), then allows. Every early return still // allows: a non-command kind, a dry-run, a missing uid, or an unauthenticated request -// simply records nothing and falls back to the committer downstream. +// simply records nothing and leaves the request without a claimed actor downstream. func (h *ValidateOperatorTypesHandler) Handle(ctx context.Context, req admission.Request) admission.Response { log := logf.FromContext(ctx).WithName("validate-operator-types") @@ -87,7 +87,7 @@ func (h *ValidateOperatorTypesHandler) Handle(ctx context.Context, req admission // No store means the webhook is running without a Redis backend (admission is on by // default, but command-author capture is Redis-backed). Allow without recording; the - // controller then finalizes as the committer (AuthorAttributed=False). + // controller then uses the no-actor path (AuthorAttributed=False). if h.Store == nil { return admission.Allowed("no author store: not recorded") } @@ -112,16 +112,16 @@ func (h *ValidateOperatorTypesHandler) Handle(ctx context.Context, req admission } // Synchronous, best-effort: the write completes before we return, so it is present - // before the object is visible (the authorship invariant, §2). A failure degrades to - // the committer — never block the command. + // before the object is visible. A failure leaves the request without a claimed actor — + // never block the command. if err := h.Store.RecordCommandAuthor(ctx, uid, author); err != nil { - log.Error(err, "record command author failed; will fall back to committer", + log.Error(err, "record command author failed; request will claim no actor", "resource", gr.String(), "namespace", req.Namespace, "name", req.Name, "uid", uid) return admission.Allowed("author record failed") } // Info (not V(1)) on purpose: command CREATEs are rare, and this line proves the // webhook was actually invoked and the author captured — the first thing to check - // when a CommitRequest unexpectedly commits as the committer. + // when a CommitRequest unexpectedly reports AuthorAttributed=False. log.Info("recorded command author at admission", "resource", gr.String(), "namespace", req.Namespace, "name", req.Name, "uid", uid, "author", author.Author) diff --git a/test/e2e/commit_author_attribution_e2e_test.go b/test/e2e/commit_author_attribution_e2e_test.go index 09cb140f..0789f416 100644 --- a/test/e2e/commit_author_attribution_e2e_test.go +++ b/test/e2e/commit_author_attribution_e2e_test.go @@ -86,7 +86,7 @@ var _ = Describe("Commit Author Attribution", Label("manager"), Ordered, func() // Authorship only flows through the per-event audit tail. A ConfigMap created // while the configmaps type is still building its first checkpoint would land in - // the unattributed baseline splice instead (fallback author), so wait until the + // the configured-author baseline splice instead, so wait until the // GitTarget reports StreamsRunning before producing the impersonated writes. waitForStreamsRunning(gitTargetName, testNs) }) diff --git a/test/e2e/commit_request_e2e_test.go b/test/e2e/commit_request_e2e_test.go index 0331bbbb..5823fbde 100644 --- a/test/e2e/commit_request_e2e_test.go +++ b/test/e2e/commit_request_e2e_test.go @@ -144,7 +144,7 @@ var _ = Describe("Commit Request", Label("commit-request", "audit-consumer"), Or "status.sha should match the SHA of the commit on the branch\n%s", recentCommitDiagnostics(repo.CheckoutDir, basePath)) - // Exactly one new commit (UC1, §8.1): this is the first commit on a fresh + // Exactly one new commit: this is the first commit on a fresh // repo, so main holds exactly one commit — the save did not also trigger a // stray second commit. g.Expect(mustCommitCount(repo.CheckoutDir)).To(Equal(1), @@ -214,7 +214,7 @@ var _ = Describe("Commit Request", Label("commit-request", "audit-consumer"), Or // validating admission runs, so the validate-operator-types webhook records the // submitter keyed by uid with no response-body name recovery (the old audit // generateName headache is gone, see - // docs/spec/commitrequest-admission-authorship.md §8). This spec proves a + // docs/spec/commitrequest-admission-authorship.md). This spec proves a // generateName CommitRequest still finalizes and becomes Ready. It is skipped in // configured-author mode, where the edit's window is committer-authored and the // named admission author would not match it end to end. @@ -267,8 +267,8 @@ var _ = Describe("Commit Request", Label("commit-request", "audit-consumer"), Or // The UC2 suite exercises a `kubectl apply` bundle that includes a CommitRequest // as its FIRST document — the deliberately-hard ordering where the save intent -// arrives before the work it is meant to save (docs/spec/commitrequest-design.md -// §2 UC2, §6.2, §8.2). A non-zero spec.closeDelaySeconds is the close-delay collect +// arrives before the work it is meant to save (docs/spec/commitrequest-design.md). A non-zero +// spec.closeDelaySeconds is the close-delay collect // window that lets the bundle's resources arrive and join the same window after the // CommitRequest is attributed, so the whole bundle lands in ONE commit carrying // the CommitRequest's message. @@ -352,7 +352,7 @@ var _ = Describe("Commit Request Bundle (UC2)", Label("commit-request", "audit-c By("applying a bundle whose FIRST document is a CommitRequest, then three Deployments") // closeDelaySeconds is sized to comfortably exceed the bundle's per-type ingestion - // spread so the close-delay collect window (§6.2) is deterministic. + // spread so the close-delay collect window is deterministic. var bundle strings.Builder bundle.WriteString(commitRequestManifest(testNs, commitRequestName, gitTargetName, message, 8)) for _, name := range deployNames { @@ -382,7 +382,7 @@ var _ = Describe("Commit Request Bundle (UC2)", Label("commit-request", "audit-c // Exactly one commit on the fresh repo's main: the entire bundle — applied // across the CommitRequest's attribution and the per-type Deployment stream — - // collapsed into a single commit (§8.2 step 4). + // collapsed into a single commit. g.Expect(mustCommitCount(repo.CheckoutDir)).To(Equal(1), "the whole bundle must land in exactly one commit\n%s", recentCommitDiagnostics(repo.CheckoutDir, basePath)) diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index 3e961701..00a219bf 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -133,16 +133,15 @@ var attributionWaitBuckets = []float64{0.5, 1, 2, 3, 5, 10} // reportAttributionStats prints the run's author-attribution outcome and timing distribution. // -// It is DIAGNOSTIC, not gating. It exists because attribution fails silently: when no audit fact -// matches within the grace window the commit is authored by the configured committer, and -// git.DefaultCommitterName ("GitOps Reverser") is ALSO the configured-author identity — so a lost -// actor is indistinguishable from a correct configured-author commit by inspection. The only way -// to see it is to count it. +// It is DIAGNOSTIC, not gating. When no audit fact matches within the grace window, the commit is +// authored as `unknown (attribution unresolved)`, making the missing actor visible in Git history +// and in the metrics below. // // The two numbers to read: // -// - "absent" — resolutions that gave up and lost the actor's name. Every one of these is a -// commit attributed to the committer instead of the human or service account responsible. +// - "absent" — resolutions that gave up without naming an actor. Each live commit is authored +// as `unknown (attribution unresolved)`, not as the configured committer. For an e2e mutation +// that should be attributable, this is an audit-attribution configuration or delivery problem. // - resolutions resolved ABOVE 3s — facts that arrived later than the 3s default grace. Each is // an attribution the default would have dropped, so this is the direct measure of how much // headroom --author-attribution-grace is buying (e2e sets 10s; see config/deployment.yaml). @@ -169,7 +168,9 @@ func reportAttributionStats() { _, _ = fmt.Fprintf(GinkgoWriter, "\n📊 author attribution — %.0f resolutions this run\n", total) var absent float64 - for _, result := range []string{"exact_user", "weak", "exact_deletecollection_item", "absent"} { + for _, result := range []string{ + "exact_user", "exact_serviceaccount", "weak", "exact_deletecollection_item", "absent", + } { n, qErr := queryPrometheus(fmt.Sprintf( `sum(max_over_time(gitopsreverser_attribution_resolutions_total{result=%q}[2h])) or vector(0)`, result)) if qErr != nil { @@ -203,12 +204,11 @@ func reportAttributionStats() { } if absent > 0 { _, _ = fmt.Fprintf(GinkgoWriter, - " ⚠ %.0f resolution(s) lost the actor (%.1f%%): committed as the configured committer.\n"+ - " If their waits sit at the grace ceiling above, the fact never arrived at all and a\n"+ - " LONGER grace will not help — check delivery with hack/attribution-diagnostics.sh.\n"+ - " Note some absences are legitimate: a watch's initial replay re-delivers"+ - " pre-existing\n objects that no live actor touched, and those have no fact by"+ - " construction.\n", + " ⚠ %.0f resolution(s) produced unknown (attribution unresolved) (%.1f%%).\n"+ + " For an e2e mutation that should be attributable, check the audit policy, webhook\n"+ + " route, source identity, and Redis delivery with hack/attribution-diagnostics.sh.\n"+ + " An absent fact is not proof of a configuration defect for every live event: some\n"+ + " changes have no audit actor by construction.\n", absent, 100*absent/total) } } @@ -244,10 +244,9 @@ func printWaitBuckets(label, resultSelector string) { // which silently answers "attribution mode" whenever the controller has been up longer than // the log window, or when `kubectl logs` picks the wrong pod mid-rollout. That is a probe // that fails open on the exact question the author assertion turns on, and both author -// values are ambiguous on their own: "GitOps Reverser" is BOTH the configured-author -// identity and the fallback used when attribution finds no matching fact -// (git.DefaultCommitterName). A wrong answer here silently flips the assertion instead of -// failing, so this reads the spec and fails loudly when it cannot. +// values cannot be inferred safely from a single commit: the configured committer is expected in +// configured-author mode, while an attribution miss is explicitly unresolved. A wrong mode answer +// would still select the wrong assertion, so this reads the spec and fails loudly when it cannot. func configuredAuthorModeEnabled() bool { out, err := kubectlRunInNamespace(namespace, "get", "deployment", "gitops-reverser", "-o", "jsonpath={.spec.template.spec.containers[*].args}") From 1334d92d9df22aa88dd9e69e7f067b0d67c2e681 Mon Sep 17 00:00:00 2001 From: Simon Koudijs Date: Mon, 20 Jul 2026 15:46:18 +0000 Subject: [PATCH 17/17] fix(attribution): address review feedback --- AGENTS.md | 6 ++ api/v1alpha3/commitrequest_types.go | 19 +++--- .../bases/configbutler.ai_commitrequests.yaml | 12 ++-- docs/UPGRADING.md | 5 +- docs/configuration.md | 9 +-- .../commitrequest-admission-authorship.md | 9 +-- docs/spec/commitrequest-design.md | 3 +- docs/spec/status-conditions-guide.md | 9 +-- hack/attribution-diagnostics.sh | 3 +- ...ommitrequest_attribution_agreement_test.go | 40 +++++++++++-- .../commitrequest_controller_unit_test.go | 15 ++--- internal/controller/commitrequest_finalize.go | 24 +++++--- internal/controller/constants.go | 3 +- internal/git/commit_request_attach_test.go | 6 +- internal/giteaclient/pagination_test.go | 58 +++++++++++++++++++ test/e2e/author_mode_test.go | 5 +- 16 files changed, 171 insertions(+), 55 deletions(-) create mode 100644 internal/giteaclient/pagination_test.go diff --git a/AGENTS.md b/AGENTS.md index 9d0c87e6..7831d82a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -7,6 +7,12 @@ - **Skip conversational phrases** - No "Great!", "Certainly!", "Okay!" - get straight to the point - **One clear message per concept** - Don't repeat the same information in different ways +## PULL REQUESTS + +- Use a concise Conventional Commit-style PR title: `type(scope): imperative summary`, or omit the scope + when none adds useful context. +- Describe the final branch state, not the sequence of commits used to reach it. + ## MANDATORY PRE-COMPLETION VALIDATION **CRITICAL**: These commands MUST pass before any implementation is considered complete: diff --git a/api/v1alpha3/commitrequest_types.go b/api/v1alpha3/commitrequest_types.go index 074b4337..d69082cc 100644 --- a/api/v1alpha3/commitrequest_types.go +++ b/api/v1alpha3/commitrequest_types.go @@ -32,12 +32,12 @@ type CommitRequestSpec struct { Message string `json:"message,omitempty"` // CloseDelaySeconds optionally delays closing the open commit window for this - // many seconds after the CommitRequest is attributed, acting as an extra collect - // window: changes the author makes in the meantime still join the open commit - // window and are included in the resulting commit. Omitted or 0 closes the window - // as soon as the CommitRequest is attributed to its author. The window can still - // be closed earlier by another author's change or by the provider's commit window - // timer, exactly as without a CommitRequest. + // many seconds after the CommitRequest attaches to a matching open window, acting as + // an extra collect window: matching changes that arrive in the meantime still join + // that window and are included in the resulting commit. Omitted or 0 closes the + // window as soon as the CommitRequest attaches. The window can still be closed + // earlier by another author's change or by the provider's commit window timer, + // exactly as without a CommitRequest. // +optional // +kubebuilder:validation:Minimum=0 // +kubebuilder:validation:Maximum=300 @@ -55,9 +55,10 @@ type CommitRequestSpec struct { // while finalizing; Stalled=True when the finalize failed and needs attention. // - AuthorAttributed (domain): binary and settled immediately. True // (AttributedFromAdmission) when the submitter captured at admission named the -// commit author; False (CommitterFallback) when no admission record exists and the -// request claims no actor. False is not a failure and does not affect Ready; the -// final Git author remains the matching watch window's author. +// commit author; False (CommitterFallback) when capture ran but no admission record +// exists, or False (AuthorCaptureDisabled) when capture is disabled. In either False +// case the request claims no actor. False is not a failure and does not affect Ready; +// the final Git author remains the matching watch window's author. // - Pushed (domain): True once the commit is in the remote repository. type CommitRequestStatus struct { // ObservedGeneration is the most recent generation observed by the controller. diff --git a/config/crd/bases/configbutler.ai_commitrequests.yaml b/config/crd/bases/configbutler.ai_commitrequests.yaml index 60f2e22a..acd2f841 100644 --- a/config/crd/bases/configbutler.ai_commitrequests.yaml +++ b/config/crd/bases/configbutler.ai_commitrequests.yaml @@ -73,12 +73,12 @@ spec: closeDelaySeconds: description: |- CloseDelaySeconds optionally delays closing the open commit window for this - many seconds after the CommitRequest is attributed, acting as an extra collect - window: changes the author makes in the meantime still join the open commit - window and are included in the resulting commit. Omitted or 0 closes the window - as soon as the CommitRequest is attributed to its author. The window can still - be closed earlier by another author's change or by the provider's commit window - timer, exactly as without a CommitRequest. + many seconds after the CommitRequest attaches to a matching open window, acting as + an extra collect window: matching changes that arrive in the meantime still join + that window and are included in the resulting commit. Omitted or 0 closes the + window as soon as the CommitRequest attaches. The window can still be closed + earlier by another author's change or by the provider's commit window timer, + exactly as without a CommitRequest. format: int32 maximum: 300 minimum: 0 diff --git a/docs/UPGRADING.md b/docs/UPGRADING.md index 6ca538a2..9a2a2a4e 100644 --- a/docs/UPGRADING.md +++ b/docs/UPGRADING.md @@ -483,8 +483,9 @@ kubectl get commitrequest/ -n -o jsonpath='{.status.sha}' ``` `AuthorAttributed=True` with reason `AttributedFromAdmission` means the internal commands admission -webhook captured the submitter. `AuthorAttributed=False` with reason `CommitterFallback` is a valid -fallback, not a failed request. +webhook captured the submitter. `AuthorAttributed=False` with reason `CommitterFallback` means capture +ran but found no record; `AuthorCaptureDisabled` means capture is not configured. Neither is a failed +request. ### 4. `GitTarget.status.phase` and materialization rollups moved to stream conditions diff --git a/docs/configuration.md b/docs/configuration.md index 314f36a1..8b1052a9 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -806,10 +806,11 @@ Progress and outcome are reported through kstatus-compatible **conditions** (no request is finalizing or waiting through `closeDelaySeconds`; `Stalled=True` when the finalize failed and needs attention (kstatus reports the object Failed). - **AuthorAttributed**: `True` with reason `AttributedFromAdmission` when the internal commands - admission webhook captured the request submitter. `False` with reason `CommitterFallback` when no - admission record exists; that is not a failure. The request then claims no actor and can attach only to - an unnamed watch window, whose Git author remains either the configured committer or the explicit - unresolved author according to the watch attribution outcome. + admission webhook captured the request submitter. `False` with reason `CommitterFallback` means capture + ran but no admission record exists; `False` with reason `AuthorCaptureDisabled` means capture is not + configured. Neither is a failure. The request then claims no actor and can attach only to an unnamed + watch window, whose Git author remains either the configured committer or the explicit unresolved author + according to the watch attribution outcome. - **Pushed**: `True` once the commit is in the remote repository. ## Audit ingestion settings diff --git a/docs/spec/commitrequest-admission-authorship.md b/docs/spec/commitrequest-admission-authorship.md index fc869a3b..64ed1039 100644 --- a/docs/spec/commitrequest-admission-authorship.md +++ b/docs/spec/commitrequest-admission-authorship.md @@ -27,9 +27,10 @@ reconciled. The result is **present-or-never**; waiting cannot create a record t `AuthorAttributed=True` means admission captured the command submitter. The request can attach only to an open window with that same named actor and GitTarget. -`AuthorAttributed=False` (`CommitterFallback`) means no admission record was available. It is not a -failure and does not mean the eventual Git author is necessarily the configured committer. The request -claims no actor and can attach only to an unnamed window. That window determines the actual Git author: +`AuthorAttributed=False` (`CommitterFallback`) means capture ran but no admission record was available; +`AuthorCaptureDisabled` means capture was not configured. Neither is a failure and neither means the +eventual Git author is necessarily the configured committer. The request claims no actor and can attach only +to an unnamed window. That window determines the actual Git author: - configured-author mode or a replay/resync write: the configured committer; - live attribution that ran but found no usable audit fact: @@ -45,7 +46,7 @@ expires, the request ends successfully with `Ready=True`, `Pushed=False`, and `N | Condition | True | False | |---|---|---| -| `AuthorAttributed` | `AttributedFromAdmission`: the command submitter was captured | `CommitterFallback`: no command-author record; request claims no actor | +| `AuthorAttributed` | `AttributedFromAdmission`: the command submitter was captured | `CommitterFallback`: capture ran but no command-author record; `AuthorCaptureDisabled`: capture is off; request claims no actor | | `Pushed` | the attached window was committed and pushed | a benign no-commit or finalize failure | | `Ready` | a pushed commit or benign no-commit | progress or a finalize failure | diff --git a/docs/spec/commitrequest-design.md b/docs/spec/commitrequest-design.md index cc83aecc..8cec1d28 100644 --- a/docs/spec/commitrequest-design.md +++ b/docs/spec/commitrequest-design.md @@ -36,7 +36,8 @@ controller performs one best-effort, present-or-never lookup: | Admission result | `AuthorAttributed` | Request claim | |---|---|---| | submitter record found | `True` / `AttributedFromAdmission` | that named actor | -| no record, webhook disabled, or Redis unavailable | `False` / `CommitterFallback` | no actor | +| capture ran but no record | `False` / `CommitterFallback` | no actor | +| webhook disabled or Redis unavailable | `False` / `AuthorCaptureDisabled` | no actor | `AuthorAttributed=False` is not a statement about the eventual Git author. The matched live window decides that: diff --git a/docs/spec/status-conditions-guide.md b/docs/spec/status-conditions-guide.md index 84c8d4b0..6586e787 100644 --- a/docs/spec/status-conditions-guide.md +++ b/docs/spec/status-conditions-guide.md @@ -100,7 +100,8 @@ Canonical reads: * failed finalize: `Ready=False`, `Stalled=True`, reason `FinalizeFailed` → kstatus Failed `AuthorAttributed=True` (`AttributedFromAdmission`) means the command submitter was captured. `False` -(`CommitterFallback`) means no admission author record exists, so the request claims no actor and can attach -only to an unnamed window. That condition does not itself determine the Git author: the attached watch -window is configured-author (the committer) when watch attribution is disabled, or explicitly unresolved -when watch attribution ran but could not name an actor. +(`CommitterFallback`) means capture ran but no admission author record exists; `False` +(`AuthorCaptureDisabled`) means capture is disabled. Both claim no actor and can attach only to an unnamed +window. That condition does not itself determine the Git author: the attached watch window is +configured-author (the committer) when watch attribution is disabled, or explicitly unresolved when watch +attribution ran but could not name an actor. diff --git a/hack/attribution-diagnostics.sh b/hack/attribution-diagnostics.sh index 850efc59..15f20303 100755 --- a/hack/attribution-diagnostics.sh +++ b/hack/attribution-diagnostics.sh @@ -44,7 +44,8 @@ rows=[(r["metric"].get("result","?"), float(r["value"][1])) for r in d.get("data total=sum(v for _,v in rows) or 1 for k,v in sorted(rows, key=lambda x:-x[1]): print(f" {k:32s} {v:8.0f} ({100*v/total:5.1f}%)") -print(f" {'TOTAL':32s} {total:8.0f}") +total_label="TOTAL" +print(f" {total_label:32s} {total:8.0f}") absent=dict(rows).get("absent",0) if absent: print(f"\n >> {absent:.0f} resolution(s) produced the explicit unresolved Git author.") diff --git a/internal/controller/commitrequest_attribution_agreement_test.go b/internal/controller/commitrequest_attribution_agreement_test.go index 997297a8..59f6bdfa 100644 --- a/internal/controller/commitrequest_attribution_agreement_test.go +++ b/internal/controller/commitrequest_attribution_agreement_test.go @@ -8,9 +8,11 @@ import ( "github.com/go-logr/logr" "github.com/stretchr/testify/assert" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" k8stypes "k8s.io/apimachinery/pkg/types" + configv1alpha3 "github.com/ConfigButler/gitops-reverser/api/v1alpha3" "github.com/ConfigButler/gitops-reverser/internal/git" "github.com/ConfigButler/gitops-reverser/internal/queue" "github.com/ConfigButler/gitops-reverser/internal/watch" @@ -54,7 +56,7 @@ func windowOutcomeAttributionMissed(t *testing.T) git.AttributionOutcome { return outcome } -// TestCommitRequestAndWindowOutcomesAgree is the cross-subsystem half of the P0 regression test. +// TestCommitRequest_OutcomesAgree is the cross-subsystem half of the P0 regression test. // // The gap that let the blocker through was that every existing test set both sides of the // comparison to the same literal, so nothing ever checked that the two INDEPENDENT producers @@ -66,7 +68,7 @@ func windowOutcomeAttributionMissed(t *testing.T) git.AttributionOutcome { // controller emits attributionNotAttempted, while --author-attribution=false leaves the window's // outcome at the zero value. When those two were different strings, no CommitRequest could // attach to any window and the user's commit message was silently dropped. -func TestCommitRequestAndWindowOutcomesAgree(t *testing.T) { +func TestCommitRequest_OutcomesAgree(t *testing.T) { tests := []struct { name string requestSide commitRequestAttribution @@ -108,9 +110,9 @@ func TestCommitRequestAndWindowOutcomesAgree(t *testing.T) { } } -// TestAttributionOffLeavesZeroOutcome states the invariant the default deployment rests on, +// TestAttributionOutcome_OffLeavesZeroOutcome states the invariant the default deployment rests on, // separately from the agreement table so a regression names itself precisely. -func TestAttributionOffLeavesZeroOutcome(t *testing.T) { +func TestAttributionOutcome_OffLeavesZeroOutcome(t *testing.T) { assert.Equal(t, git.AttributionNotAttempted, windowOutcomeAttributionOff(), "configured-author mode must leave the event's outcome at AttributionNotAttempted") assert.Equal(t, git.AttributionNotAttempted, attributionNotAttempted.gitOutcome(), @@ -118,3 +120,33 @@ func TestAttributionOffLeavesZeroOutcome(t *testing.T) { assert.Equal(t, git.AttributionUnresolved, windowOutcomeAttributionMissed(t), "attribution that ran and found nothing must be AttributionUnresolved, not the zero value") } + +// TestCommitRequestAttribution_GitOutcome keeps every controller outcome's projection explicit, +// including the defensive zero value. +func TestCommitRequestAttribution_GitOutcome(t *testing.T) { + tests := []struct { + name string + input commitRequestAttribution + want git.AttributionOutcome + }{ + {name: "unset is not attempted", input: attributionUnset, want: git.AttributionNotAttempted}, + {name: "admission capture is resolved", input: attributionFromAdmission, want: git.AttributionResolved}, + {name: "admission miss is unresolved", input: attributionCommitter, want: git.AttributionUnresolved}, + {name: "capture disabled is not attempted", input: attributionNotAttempted, want: git.AttributionNotAttempted}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.want, tc.input.gitOutcome()) + }) + } +} + +// TestCommitRequestAttribution_UnsetStatusIsCaptureDisabled keeps an unset decision visible and safe. +func TestCommitRequestAttribution_UnsetStatusIsCaptureDisabled(t *testing.T) { + var request configv1alpha3.CommitRequest + + setCommitRequestAttributed(&request, attributionUnset) + + requireCondition(t, request, ConditionTypeAuthorAttributed, metav1.ConditionFalse, crReasonAuthorCaptureDisabled) +} diff --git a/internal/controller/commitrequest_controller_unit_test.go b/internal/controller/commitrequest_controller_unit_test.go index 5688c843..9b21fb58 100644 --- a/internal/controller/commitrequest_controller_unit_test.go +++ b/internal/controller/commitrequest_controller_unit_test.go @@ -258,9 +258,10 @@ func TestCommitRequestReconcile_FinalizeErrorBecomesFailed(t *testing.T) { requireCondition(t, got, ConditionTypePushed, metav1.ConditionFalse, "") } -// A lookup miss means the request claims no actor immediately — present-or-never, no wait. Even a -// freshly created CommitRequest attaches at once with a blank author and records -// AuthorAttributed=False (CommitterFallback). The matching window determines the Git author. +// TestCommitRequestReconcile_LookupMissClaimsNoActor verifies that a lookup miss claims no actor +// immediately — present-or-never, no wait. A freshly created CommitRequest attaches at once with a +// blank author and records AuthorAttributed=False (CommitterFallback). The matching window determines +// the Git author. func TestCommitRequestReconcile_LookupMissClaimsNoActor(t *testing.T) { cr := newCommitRequest("save-fresh") cr.CreationTimestamp = metav1.Now() // fresh: the old path would have requeued here @@ -380,7 +381,7 @@ func TestCommitRequestReconcile_StaleCacheEchoDoesNotRefinalize(t *testing.T) { // Webhook-disabled mode (no AuthorLookup) never waits: a freshly created CommitRequest // attaches immediately with a blank author and records AuthorAttributed=False -// (CommitterFallback). The matching window determines the Git author. +// (AuthorCaptureDisabled). The matching window determines the Git author. func TestCommitRequestReconcile_ConfiguredAuthorCommitsWithoutWaiting(t *testing.T) { cr := newCommitRequest("save-configured-author") cr.CreationTimestamp = metav1.Now() // fresh: a waiting path would requeue instead of commit @@ -397,13 +398,13 @@ func TestCommitRequestReconcile_ConfiguredAuthorCommitsWithoutWaiting(t *testing got := fetchCommitRequest(t, c, "save-configured-author") requireCondition(t, got, ConditionTypeReady, metav1.ConditionTrue, crReasonCommitted) requireCondition(t, got, ConditionTypePushed, metav1.ConditionTrue, crReasonPushed) - requireCondition(t, got, ConditionTypeAuthorAttributed, metav1.ConditionFalse, crReasonCommitterFallback) + requireCondition(t, got, ConditionTypeAuthorAttributed, metav1.ConditionFalse, crReasonAuthorCaptureDisabled) assert.Equal(t, "c0ffee", got.Status.SHA) require.Len(t, f.calls, 1, "the attach is sent immediately, with no attribution wait") assert.Empty(t, f.calls[0].Author, "webhook-disabled mode attaches with a blank author") } -// Webhook-disabled mode settles AuthorAttributed=False (CommitterFallback) immediately +// Webhook-disabled mode settles AuthorAttributed=False (AuthorCaptureDisabled) immediately // and never parks the request in a "waiting for the author" state: even while the // attach is still being polled, the request is in the WaitingForCloseDelay wait. func TestCommitRequestReconcile_ConfiguredAuthorAttributedImmediately(t *testing.T) { @@ -420,7 +421,7 @@ func TestCommitRequestReconcile_ConfiguredAuthorAttributedImmediately(t *testing assert.Empty(t, f.calls[0].Author) got := fetchCommitRequest(t, c, "save-committer-poll") - requireCondition(t, got, ConditionTypeAuthorAttributed, metav1.ConditionFalse, crReasonCommitterFallback) + requireCondition(t, got, ConditionTypeAuthorAttributed, metav1.ConditionFalse, crReasonAuthorCaptureDisabled) requireCondition(t, got, ConditionTypeReconciling, metav1.ConditionTrue, crReasonWaitingForCloseDelay) assert.False(t, commitRequestIsTerminal(&got), "in the close-delay wait, not terminal") } diff --git a/internal/controller/commitrequest_finalize.go b/internal/controller/commitrequest_finalize.go index f934b10a..ac161d31 100644 --- a/internal/controller/commitrequest_finalize.go +++ b/internal/controller/commitrequest_finalize.go @@ -28,6 +28,7 @@ const ( crReasonUnexpectedOutcome = "UnexpectedOutcome" crReasonAttributedFromAdmission = "AttributedFromAdmission" crReasonCommitterFallback = "CommitterFallback" + crReasonAuthorCaptureDisabled = "AuthorCaptureDisabled" crReasonPushed = "Pushed" ) @@ -50,12 +51,14 @@ const notStalledMessage = "the CommitRequest is not stalled" type commitRequestAttribution int const ( + // attributionUnset is the defensive zero value. It must never report a resolved + // actor when a caller neglected to initialize the attribution decision. + attributionUnset commitRequestAttribution = iota // attributionFromAdmission means the submitter was captured at admission by the // validate-operator-types webhook and named as the commit author. - attributionFromAdmission commitRequestAttribution = iota - // attributionCommitter means no admission author record exists — the - // validate-operator-types webhook is not configured (or did not record one) — so the - // request does not claim an actor. + attributionFromAdmission + // attributionCommitter means command-author capture ran but no admission author record + // exists, so the request does not claim an actor. attributionCommitter // attributionNotAttempted means command-author capture is switched off entirely (the // validate-operator-types webhook is not running), so no submitter was ever sought. It is @@ -68,6 +71,8 @@ const ( // controller's own enum down — means the git package never learns about admission records. func (a commitRequestAttribution) gitOutcome() git.AttributionOutcome { switch a { + case attributionUnset: + return git.AttributionNotAttempted case attributionFromAdmission: return git.AttributionResolved case attributionCommitter: @@ -122,10 +127,15 @@ func markCommitRequestWaitingForCloseDelay(cr *configv1alpha3.CommitRequest, att } // setCommitRequestAttributed records the settled, binary author decision on the -// AuthorAttributed condition. False (CommitterFallback) is not a failure and does not -// affect Ready — it is the honest signal that no admission author record was found. +// AuthorAttributed condition. A False reason distinguishes an absent record +// (CommitterFallback) from capture disabled (AuthorCaptureDisabled). Neither is a failure or +// affects Ready; the request claims no actor. func setCommitRequestAttributed(cr *configv1alpha3.CommitRequest, attribution commitRequestAttribution) { switch attribution { + case attributionUnset: + setCommitRequestCondition(cr, ConditionTypeAuthorAttributed, metav1.ConditionFalse, + crReasonAuthorCaptureDisabled, + "command-author capture state was unset; the request can attach only to a window with no named actor") case attributionFromAdmission: setCommitRequestCondition(cr, ConditionTypeAuthorAttributed, metav1.ConditionTrue, crReasonAttributedFromAdmission, @@ -140,7 +150,7 @@ func setCommitRequestAttributed(cr *configv1alpha3.CommitRequest, attribution co "window with no named actor") case attributionNotAttempted: setCommitRequestCondition(cr, ConditionTypeAuthorAttributed, metav1.ConditionFalse, - crReasonCommitterFallback, + crReasonAuthorCaptureDisabled, "command-author capture is disabled (the validate-operator-types webhook is not "+ "configured); the request can attach only to a window with no named actor") } diff --git a/internal/controller/constants.go b/internal/controller/constants.go index 770c77b2..a36b92fe 100644 --- a/internal/controller/constants.go +++ b/internal/controller/constants.go @@ -46,7 +46,8 @@ const ( // was named from the submitter captured at admission. It is binary and immediately // settled (no Unknown, no timeout): True (AttributedFromAdmission) when the // validate-operator-types webhook recorded the submitter, False (CommitterFallback) when - // no admission record exists. False is not a failure and does not affect Ready: the + // capture ran but found no record, or False (AuthorCaptureDisabled) when capture is off. + // False is not a failure and does not affect Ready: the // request claims no actor and can attach only to an unnamed watch window // (docs/architecture.md#commitrequest-finalize). ConditionTypeAuthorAttributed = "AuthorAttributed" diff --git a/internal/git/commit_request_attach_test.go b/internal/git/commit_request_attach_test.go index 39ae795a..f967fe27 100644 --- a/internal/git/commit_request_attach_test.go +++ b/internal/git/commit_request_attach_test.go @@ -488,7 +488,7 @@ func TestAttach_ConflictReplayResolvesToPostReplaySHA(t *testing.T) { assert.Equal(t, message, commit.Message, "the replayed commit keeps the user's message") } -// TestAttributionZeroValueIsNotAttempted pins the coupling the whole matching rule rests on: +// TestAttributionOutcome_ZeroValueIsNotAttempted pins the coupling the whole matching rule rests on: // the zero value of AttributionOutcome must BE AttributionNotAttempted. // // Most producers of an Event never assign Attribution — reconcile, resync, bootstrap, and @@ -496,7 +496,7 @@ func TestAttach_ConflictReplayResolvesToPostReplaySHA(t *testing.T) { // of them mean "no actor was sought". When this constant was "not_attempted" the zero value was a // silent fourth state that compared equal to no named outcome, and no test caught it because // every case set both sides to the zero value and compared "" to "". -func TestAttributionZeroValueIsNotAttempted(t *testing.T) { +func TestAttributionOutcome_ZeroValueIsNotAttempted(t *testing.T) { var zero AttributionOutcome assert.Equal(t, AttributionNotAttempted, zero, "the zero value must be AttributionNotAttempted; three doc comments and every non-live "+ @@ -514,7 +514,7 @@ func TestAttributionZeroValueIsNotAttempted(t *testing.T) { // The window's outcome and the request's outcome come from two subsystems configured by two // different flags (--author-attribution vs --admission-webhook, cmd/main.go:311-316), so this // walks every pair either side can actually produce and names the deployment that produces it. -// TestCommitRequestAndWindowOutcomesAgree in internal/controller pins that these are the real +// TestCommitRequest_OutcomesAgree in internal/controller pins that these are the real // values the two producers emit; here we pin what the matching rule does with them. func TestMatchesWindow_AcrossIndependentlyConfiguredSubsystems(t *testing.T) { const otherAuthor = "bob" diff --git a/internal/giteaclient/pagination_test.go b/internal/giteaclient/pagination_test.go new file mode 100644 index 00000000..dab4980d --- /dev/null +++ b/internal/giteaclient/pagination_test.go @@ -0,0 +1,58 @@ +// SPDX-License-Identifier: Apache-2.0 + +package giteaclient + +import ( + "context" + "errors" + "io" + "net/http" + "strings" + "testing" +) + +type roundTripperFunc func(*http.Request) (*http.Response, error) + +func (f roundTripperFunc) RoundTrip(request *http.Request) (*http.Response, error) { + return f(request) +} + +// TestListPaginated_PropagatesRequestFailures keeps transport and non-200 errors observable to callers. +func TestListPaginated_PropagatesRequestFailures(t *testing.T) { + tests := []struct { + name string + transport http.RoundTripper + want string + }{ + { + name: "transport error", + transport: roundTripperFunc(func(*http.Request) (*http.Response, error) { + return nil, errors.New("connection refused") + }), + want: "connection refused", + }, + { + name: "unexpected status", + transport: roundTripperFunc(func(*http.Request) (*http.Response, error) { + return &http.Response{ + StatusCode: http.StatusTooManyRequests, + Body: io.NopCloser(strings.NewReader("rate limited")), + Header: make(http.Header), + }, nil + }), + want: "HTTP 429: rate limited", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + client := New("https://gitea.example.test/api/v1", "", "") + client.HTTPClient = &http.Client{Transport: tc.transport} + + _, err := listPaginated[string](context.Background(), client, "/items", 50, 1) + if err == nil || !strings.Contains(err.Error(), tc.want) { + t.Fatalf("listPaginated() error = %v, want substring %q", err, tc.want) + } + }) + } +} diff --git a/test/e2e/author_mode_test.go b/test/e2e/author_mode_test.go index 2abab94d..39e091a5 100644 --- a/test/e2e/author_mode_test.go +++ b/test/e2e/author_mode_test.go @@ -4,7 +4,8 @@ package e2e import "testing" -// The author-mode decision must be read from the Deployment's args, and both defaults are +// TestConfiguredAuthorModeFromArgs_AllScenarios verifies that the author-mode decision is read from +// the Deployment's args. Both defaults are // ENABLED in cmd/main.go — so an absent flag means attribution mode, and only an explicit // opt-out selects configured-author mode. Getting this backwards silently flips the commit // author assertion instead of failing it. @@ -13,7 +14,7 @@ import "testing" // default of true is FATAL at startup ("redis-addr is required when author-attribution is // enabled", cmd/main.go), so it is not a mode this probe can ever be asked to classify. Every // configured-author row below therefore turns attribution off explicitly. -func TestConfiguredAuthorModeFromArgs(t *testing.T) { +func TestConfiguredAuthorModeFromArgs_AllScenarios(t *testing.T) { // The real jsonpath output shape: a bracketed, quoted, comma-separated list. const live = `["--metrics-bind-address=:8443","--admission-webhook",` + `"--redis-addr=valkey.valkey-e2e.svc.cluster.local:6379","--redis-insecure"]`