From 470d6aa32c12faa5ac4ced274cd63533edd57a1a Mon Sep 17 00:00:00 2001 From: GenWave Radio Date: Wed, 16 Sep 2026 16:51:46 -0600 Subject: [PATCH 1/2] ci(release): T502 gate promotion on the stack gate (gh-#777) SPEC F179: merge-manifests tags home- only; new stack-gate job (needs merge-manifests, 40 min, fetch-depth 0 for the upgrade worktree, install-ffmpeg, GH_TOKEN) runs stack_gate.sh --fresh --upgrade --capture and uploads gate-report/ (md, json, capture.wav, compose-*.log) for 7 days; new promote job retags the five images home-latest from home- and re-asserts both platforms; create-release needs promote, downloads the report and creates the Release with --generate-notes --notes-file (proof section first, gh prepends the body). A red gate leaves home- and nothing else; rerun --failed re-enters at the gate. Story448 pins un-skipped, 21 facts. --- .github/workflows/release.yml | 174 +++++++++++++++++- .../Specs/Story448_ReleaseWorkflowGate.cs | 44 +++-- 2 files changed, 188 insertions(+), 30 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index bab5a116..cdd83eed 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -17,7 +17,17 @@ name: Release # stitches the digests into one manifest list per image with `docker buildx imagetools create`. # The release FAILS if either arch fails: merge-manifests `needs` the whole build matrix, the # merge script demands exactly two digests, and the inspect step asserts both platforms are in -# the pushed index. `home-latest` and the Release object only move after every merge succeeds. +# the pushed index. merge-manifests tags `home-` only (SPEC F179.1) -- nothing is called +# "latest" yet. +# +# Stack gate before promotion (SPEC F179, STORY-448, gh-#777): +# `home-latest` and the Release move only after `stack-gate` goes green (SPEC F179.2-F179.5). +# `stack-gate` boots the just-published tag fresh AND upgraded-from-the-previous-release, records +# the live stream, and proves the appliance actually holds up before anything downstream moves. A +# red gate leaves `home-` published and nothing else -- the promote and Release steps never +# run, and no pins PR follows. `gh run rerun --failed` re-enters at the gate (merge-manifests's +# images are already pushed, so nothing upstream needs to re-run). This workflow never deletes a +# tag. # # Pushing the tag is the ONLY manual act: this workflow never commits or pushes back to the repo. @@ -171,8 +181,9 @@ jobs: if-no-files-found: error retention-days: 1 - # One manifest list per image: stitch the two per-arch digests into - # `home-` + `home-latest`. Runs only when EVERY build-images leg succeeded. + # One manifest list per image: stitch the two per-arch digests into `home-` (SPEC F179.1 + # -- `home-latest` moves later, in `promote`, only once `stack-gate` proves the tag boots). + # Runs only when EVERY build-images leg succeeded. merge-manifests: needs: build-images runs-on: ubuntu-latest @@ -241,7 +252,6 @@ jobs: --annotation "index:org.opencontainers.image.licenses=AGPL-3.0-only" \ --annotation "index:org.opencontainers.image.version=$GW_TAG" \ -t "$REGISTRY_IMAGE:home-$GW_TAG" \ - -t "$REGISTRY_IMAGE:home-latest" \ "${digests[@]}" - name: Verify both architectures shipped env: @@ -253,9 +263,144 @@ jobs: echo "$inspect_output" | grep -q 'linux/amd64' echo "$inspect_output" | grep -q 'linux/arm64' - create-release: + # New job (SPEC F179.2, F183.2, STORY-448, gh-#777): proves the tag merge-manifests just + # published actually boots -- a fresh install AND an upgrade from the previous release -- before + # anything downstream promotes it. `promote` and `create-release` both need this (transitively). + stack-gate: needs: merge-manifests runs-on: ubuntu-latest + # gh-#777 STORY-448: --fresh + --upgrade + --capture run four legs end to end (setup, the + # upgrade-worktree checkout, a live boot, a recorded stream) -- 40 min budgets real headroom + # over the ~10-15 min observed on tools/gate/stack_gate.sh's own dev-box runs. SPEC F179.5: a + # red leg here leaves the tag published and nothing else -- `gh run rerun --failed` re-enters + # exactly here, since merge-manifests's images are already pushed. + timeout-minutes: 40 + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} # --upgrade's `gh release list` needs it + GW_TAG: ${{ github.ref_name }} + steps: + - uses: actions/checkout@v7 + with: + # --upgrade does `git worktree add` of the previous v* tag: needs the full tag + # history, not the default single-commit shallow clone. + fetch-depth: 0 + - name: Install ffmpeg (stack_gate.sh's own capture leg needs the real binary) + uses: ./.github/actions/install-ffmpeg + - name: Fresh + upgrade + capture gate against the just-published tag + run: tools/gate/stack_gate.sh --tag "$GW_TAG" --fresh --upgrade --capture --report gate-report + # gate-report/ carries gate-report.md + gate-report.json (the pass/fail verdict per leg), + # capture.wav (the recorded stream), and, on any failed leg, the compose-fresh.log / + # compose-upgrade.log dumps stack_gate.sh attaches for debugging. + - name: Upload gate report + if: always() + uses: actions/upload-artifact@v7 + with: + name: gate-report + path: gate-report/ + retention-days: 7 + + # New job (SPEC F179.3): now that stack-gate has proven $GW_TAG boots fresh and upgraded, move + # the floating `home-latest` tag onto the images that just proved themselves. Five explicit + # retag+verify pairs, not a matrix: STORY-448's ItRetagsFiveImagesToHomeLatest / + # EachRetagSourcesTheTaggedImage pins count literal `-t …:home-latest` / source lines in this + # file's own YAML text, which a matrix's single parameterized step body would only satisfy once. + promote: + needs: stack-gate + runs-on: ubuntu-latest + # gh-#581 N1 ballpark: five retag+inspect pairs against ghcr.io, each well under a minute in + # merge-manifests's own timing; 10 min is large headroom below the 360-min default. + timeout-minutes: 10 + permissions: + contents: read + packages: write # ghcr.io retag + env: + GW_TAG: ${{ github.ref_name }} + steps: + - uses: docker/login-action@v4.6.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - uses: docker/setup-buildx-action@v4 + - name: Retag api -> home-latest + env: + REGISTRY_IMAGE: ghcr.io/genwave-org/genwave + run: | + docker buildx imagetools create \ + -t "$REGISTRY_IMAGE:home-latest" \ + "$REGISTRY_IMAGE:home-$GW_TAG" + - name: Verify api home-latest ships both architectures + env: + REGISTRY_IMAGE: ghcr.io/genwave-org/genwave + run: | + inspect_output="$(docker buildx imagetools inspect "$REGISTRY_IMAGE:home-latest")" + echo "$inspect_output" + echo "$inspect_output" | grep -q 'linux/amd64' + echo "$inspect_output" | grep -q 'linux/arm64' + - name: Retag icecast -> home-latest + env: + REGISTRY_IMAGE: ghcr.io/genwave-org/genwave-icecast + run: | + docker buildx imagetools create \ + -t "$REGISTRY_IMAGE:home-latest" \ + "$REGISTRY_IMAGE:home-$GW_TAG" + - name: Verify icecast home-latest ships both architectures + env: + REGISTRY_IMAGE: ghcr.io/genwave-org/genwave-icecast + run: | + inspect_output="$(docker buildx imagetools inspect "$REGISTRY_IMAGE:home-latest")" + echo "$inspect_output" + echo "$inspect_output" | grep -q 'linux/amd64' + echo "$inspect_output" | grep -q 'linux/arm64' + - name: Retag admin-ui -> home-latest + env: + REGISTRY_IMAGE: ghcr.io/genwave-org/genwave-admin-ui + run: | + docker buildx imagetools create \ + -t "$REGISTRY_IMAGE:home-latest" \ + "$REGISTRY_IMAGE:home-$GW_TAG" + - name: Verify admin-ui home-latest ships both architectures + env: + REGISTRY_IMAGE: ghcr.io/genwave-org/genwave-admin-ui + run: | + inspect_output="$(docker buildx imagetools inspect "$REGISTRY_IMAGE:home-latest")" + echo "$inspect_output" + echo "$inspect_output" | grep -q 'linux/amd64' + echo "$inspect_output" | grep -q 'linux/arm64' + - name: Retag engine -> home-latest + env: + REGISTRY_IMAGE: ghcr.io/genwave-org/genwave-engine + run: | + docker buildx imagetools create \ + -t "$REGISTRY_IMAGE:home-latest" \ + "$REGISTRY_IMAGE:home-$GW_TAG" + - name: Verify engine home-latest ships both architectures + env: + REGISTRY_IMAGE: ghcr.io/genwave-org/genwave-engine + run: | + inspect_output="$(docker buildx imagetools inspect "$REGISTRY_IMAGE:home-latest")" + echo "$inspect_output" + echo "$inspect_output" | grep -q 'linux/amd64' + echo "$inspect_output" | grep -q 'linux/arm64' + - name: Retag piper -> home-latest + env: + REGISTRY_IMAGE: ghcr.io/genwave-org/genwave-piper + run: | + docker buildx imagetools create \ + -t "$REGISTRY_IMAGE:home-latest" \ + "$REGISTRY_IMAGE:home-$GW_TAG" + - name: Verify piper home-latest ships both architectures + env: + REGISTRY_IMAGE: ghcr.io/genwave-org/genwave-piper + run: | + inspect_output="$(docker buildx imagetools inspect "$REGISTRY_IMAGE:home-latest")" + echo "$inspect_output" + echo "$inspect_output" | grep -q 'linux/amd64' + echo "$inspect_output" | grep -q 'linux/arm64' + + create-release: + needs: promote + runs-on: ubuntu-latest # gh-#581 N1: observed 7-40s (a single `gh release` call); 5 min is large headroom below # the 360-min default. timeout-minutes: 5 @@ -263,12 +408,27 @@ jobs: contents: write # gh release create steps: - uses: actions/checkout@v7 + # SPEC F179.4: pull the stack-gate's own report so the Release notes can carry its proof + # verbatim, instead of asserting a second time what stack-gate already asserted. + - name: Download the stack-gate report + uses: actions/download-artifact@v8 + with: + name: gate-report + path: gate-report + - name: Build release notes (generated notes + the stack-gate's own proof) + run: | + { + echo "## ✅ What this release proved" + cat gate-report/gate-report.md + } > notes.md # gh CLI over a third-party action -- it's preinstalled on the runner and needs only the # default token. No repo write beyond the Release object itself (not a commit, not a push). # Idempotent: a release created through the GitHub UI ("Draft a new release") creates the # tag AND the Release together, so this workflow still fires on the tag push and must not # 422 on the already-existing Release (observed on v2.0.0). Bare-tag pushes still get their - # Release created here. + # Release created here. `--generate-notes` composes with `--notes-file`: gh sends the file + # as the body with generate_release_notes on, and GitHub prepends a supplied body to its + # generated notes -- so the proof section above comes first, the generated notes after. - name: Create GitHub Release env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -277,5 +437,5 @@ jobs: if gh release view "$GW_TAG" --json name >/dev/null 2>&1; then echo "Release $GW_TAG already exists (UI-created) -- skipping creation." else - gh release create "$GW_TAG" --generate-notes --title "$GW_TAG" + gh release create "$GW_TAG" --generate-notes --notes-file notes.md --title "$GW_TAG" fi diff --git a/tests/GenWave.Host.Tests/Specs/Story448_ReleaseWorkflowGate.cs b/tests/GenWave.Host.Tests/Specs/Story448_ReleaseWorkflowGate.cs index 64de0998..b59d3aef 100644 --- a/tests/GenWave.Host.Tests/Specs/Story448_ReleaseWorkflowGate.cs +++ b/tests/GenWave.Host.Tests/Specs/Story448_ReleaseWorkflowGate.cs @@ -14,8 +14,6 @@ namespace GenWave.Host.Tests.Specs; public static class FeatureReleasePromotionWaitsForTheStackGate { - const string Pending = "pending: T502 — release.yml: stack-gate + promote, home-latest after the gate, notes from the report (STORY-448)"; - static string Workflow => File.ReadAllText(Path.Combine( RepoRootLocator.Find(AppContext.BaseDirectory), ".github", "workflows", "release.yml")); @@ -34,7 +32,7 @@ static string Job(string name) public sealed class ScenarioMergeManifestsNoLongerTagsHomeLatest { - [Fact(Skip = Pending)] + [Fact] public void NoHomeLatestInTheMergeJob() => Assert.DoesNotContain("home-latest", Job("merge-manifests"), StringComparison.Ordinal); } @@ -42,32 +40,32 @@ public sealed class ScenarioStackGateFollowsMergeManifests { readonly string job = Job("stack-gate"); - [Fact(Skip = Pending)] + [Fact] public void TheJobExists() => Assert.NotEqual("", job); - [Fact(Skip = Pending)] + [Fact] public void ItNeedsMergeManifests() => Assert.Matches(@"needs:\s*merge-manifests", job); - [Fact(Skip = Pending)] + [Fact] public void ItHasAFortyMinuteBudget() => Assert.Matches(@"timeout-minutes:\s*40", job); - [Fact(Skip = Pending)] + [Fact] public void ItRunsTheFourLegs() => Assert.Matches(@"tools/gate/stack_gate\.sh .*--fresh .*--upgrade .*--capture .*--report", job); - [Fact(Skip = Pending)] + [Fact] public void ItUploadsTheReportMarkdown() => Assert.Contains("gate-report.md", job, StringComparison.Ordinal); - [Fact(Skip = Pending)] + [Fact] public void ItUploadsTheReportJson() => Assert.Contains("gate-report.json", job, StringComparison.Ordinal); - [Fact(Skip = Pending)] + [Fact] public void ItUploadsTheCapture() => Assert.Contains("capture.wav", job, StringComparison.Ordinal); - [Fact(Skip = Pending)] + [Fact] public void ItUploadsTheComposeLogs() => Assert.Contains("compose-", job, StringComparison.Ordinal); - [Fact(Skip = Pending)] + [Fact] public void TheArtifactsKeepForSevenDays() => Assert.Matches(@"retention-days:\s*7", job); } @@ -75,16 +73,16 @@ public sealed class ScenarioPromoteRetagsAfterTheGate { readonly string job = Job("promote"); - [Fact(Skip = Pending)] + [Fact] public void ItNeedsStackGate() => Assert.Matches(@"needs:\s*stack-gate", job); - [Fact(Skip = Pending)] + [Fact] public void ItUsesImagetoolsCreate() => Assert.Contains("imagetools create", job, StringComparison.Ordinal); - [Fact(Skip = Pending)] + [Fact] public void ItRetagsFiveImagesToHomeLatest() => Assert.Equal(5, Regex.Matches(job, @"-t\s+""?\S*:home-latest").Count); - [Fact(Skip = Pending)] + [Fact] public void EachRetagSourcesTheTaggedImage() => Assert.Equal(5, Regex.Matches(job, @"\S*:home-\$\{?GW_TAG\}?""?\s*$", RegexOptions.Multiline).Count); } @@ -92,28 +90,28 @@ public sealed class ScenarioCreateReleaseNeedsPromote { readonly string job = Job("create-release"); - [Fact(Skip = Pending)] + [Fact] public void ItNeedsPromote() => Assert.Matches(@"needs:\s*promote", job); - [Fact(Skip = Pending)] + [Fact] public void ItGeneratesNotes() => Assert.Contains("--generate-notes", job, StringComparison.Ordinal); - [Fact(Skip = Pending)] + [Fact] public void ItAppendsTheNotesFile() => Assert.Contains("--notes-file", job, StringComparison.Ordinal); - [Fact(Skip = Pending)] + [Fact] public void TheNotesAreBuiltFromTheReport() => Assert.Contains("gate-report.md", job, StringComparison.Ordinal); - [Fact(Skip = Pending)] + [Fact] public void TheNotesCarryTheProofHeading() => Assert.Contains("## ✅ What this release proved", job, StringComparison.Ordinal); } public sealed class ScenarioTheHeaderTellsTheNewTruth { - [Fact(Skip = Pending)] + [Fact] public void HomeLatestMovesAfterStackGate() => Assert.Matches(@"home-latest[^\n]*(after|behind)[^\n]*stack-gate", Header); - [Fact(Skip = Pending)] + [Fact] public void TheOldPromiseIsGone() => Assert.DoesNotContain("only move after every merge succeeds", Header, StringComparison.Ordinal); } } From be8fe82acf1e92cadf58ae7cc3ea4dbb3a928a22 Mon Sep 17 00:00:00 2001 From: GenWave Radio Date: Wed, 16 Sep 2026 17:32:53 -0600 Subject: [PATCH 2/2] ops(gate): T503 run the upgrade leg last, after the fresh stack comes down (gh-#777) `--fresh --upgrade --capture` in one run (release.yml's combination) failed the upgrade leg at setup: the previous release's setup.sh preflight found port 8080 bound by the still-running fresh stack, which the capture leg records. compose.yaml publishes fixed host ports, so the upgrade leg now runs after capture/chaos and teardown_project brings the fresh project down first, dropping its PROJECTS entry so the EXIT trap does not repeat the `down -v`. Runs without --upgrade are unchanged. Replayed on the dev box 2026-09-16: fresh passed, upgrade passed (previous v5.8.2), capture red only on loudness (gh-#797). --- tools/gate/stack_gate.sh | 60 ++++++++++++++++++++++++++++++---------- 1 file changed, 45 insertions(+), 15 deletions(-) diff --git a/tools/gate/stack_gate.sh b/tools/gate/stack_gate.sh index c102e198..8ea5349d 100755 --- a/tools/gate/stack_gate.sh +++ b/tools/gate/stack_gate.sh @@ -8,13 +8,6 @@ # --fresh a clean install from nothing (setup.sh --yes, bare launch.sh — the # wizard's own .env picks the pinned piper-only topology — health/on-air # waits) -# --upgrade an existing station on the previous release, upgraded onto the tag: a -# `git worktree` of the previous release stands up its own pinned stack, -# waits on-air, then `compose stop api`, swaps in the CURRENT tag's db/ + -# migrate.sh, runs the migration, checks the role boundary (station_svc/ -# library_svc each locked out of the other's schema, SPEC F178.7), brings api -# back on the CURRENT tag, and waits health/on-air again (SPEC F178.6/F178.7; -# see run_upgrade_leg) # --capture records CAPTURE_SECS of the fresh leg's own live stream and measures it: no # silent gaps, loudness within TOL_LU of the station's own configured target, # speech aired through the booth log (requires --fresh) @@ -32,6 +25,15 @@ # "engine-reconnect on-air"; then a FRESH # 60-second capture measured for silence only, else "engine-reconnect silence" # (see run_engine_reconnect_scenario). +# --upgrade an existing station on the previous release, upgraded onto the tag: a +# `git worktree` of the previous release stands up its own pinned stack, +# waits on-air, then `compose stop api`, swaps in the CURRENT tag's db/ + +# migrate.sh, runs the migration, checks the role boundary (station_svc/ +# library_svc each locked out of the other's schema, SPEC F178.7), brings api +# back on the CURRENT tag, and waits health/on-air again (SPEC F178.6/F178.7; +# see run_upgrade_leg). Runs LAST, after capture/chaos, because the fresh +# stack holds the fixed host ports until it is measured — see the upgrade +# call site at the bottom of this file (T503). # --from the previous release the upgrade leg starts from — overrides the newest # other `v*` GitHub release otherwise resolved via `gh release list` # --report write gate-report.md + gate-report.json there (default: .) @@ -51,9 +53,10 @@ # seconds, no cap on the event count (a real nightly run against v5.8.3 saw two events, 7.1s + # 24.4s, during one outage): measure_audio.sh's own SILENCE_MAX_SECS knob, passed # SILENCE_MAX_SECS=GATE_OUTAGE_SILENCE_MAX_SECS only for the capture leg's own measure call when -# --chaos is given (see run_capture_leg). +# --chaos is given (see run_capture_leg). PLAN T503 (gh-#777) moved the upgrade leg's call site +# to run LAST and added teardown_project — see the call site at the bottom of this file. # -# Usage: tools/gate/stack_gate.sh --tag [--fresh] [--upgrade] [--capture] [--chaos] +# Usage: tools/gate/stack_gate.sh --tag [--fresh] [--capture] [--chaos] [--upgrade] # [--from ] [--report ] # Exit codes: 0 = every requested leg passed (or none were requested); 1 = a leg failed (see the # report); 2 = usage error or a missing prerequisite (docker, ffmpeg, jq, curl; gh @@ -233,6 +236,22 @@ cleanup() { } trap cleanup EXIT +# teardown_project — runs cleanup()'s per-entry `down -v` for ONE entry now +# (same $scratch/.env guard), then drops it from PROJECTS so the EXIT trap does not repeat it. The +# scratch DIRECTORY stays for cleanup() to remove at EXIT; the report has already copied capture.wav +# out. Used by the upgrade call site (T503) to free the fresh stack's host ports. +teardown_project() { + local scratch="$1" project="$2" kept=() e + local entry="$scratch|$project" + if [ -f "$scratch/.env" ]; then + ( cd "$scratch" && docker compose -p "$project" "${COMPOSE_FILES[@]}" down -v ) || true + fi + for e in "${PROJECTS[@]}"; do + [ "$e" = "$entry" ] || kept+=("$e") + done + PROJECTS=("${kept[@]}") +} + # --------------------------------------------------------------------------------------------- # Arg parsing — usage errors exit 2 with a one-line reason on stderr. # --------------------------------------------------------------------------------------------- @@ -838,12 +857,6 @@ run_upgrade_leg() { LEG_DETAIL[upgrade]="" } -if [ "$DO_UPGRADE" = 1 ]; then - run_upgrade_leg -else - LEG_STATUS[upgrade]="skipped"; LEG_DETAIL[upgrade]="--upgrade not given" -fi - # capture_booth_log_count — the same compose invocation shape (project, -f # list, cwd) fresh_onair_frame (above) uses for its own `exec -T engine` metadata poll, read # against the db service instead: how many station.booth_log rows carry a non-null segment_kind @@ -1188,6 +1201,23 @@ else LEG_STATUS[chaos]="skipped"; LEG_DETAIL[chaos]="--chaos not given" fi +# The upgrade leg runs LAST — after fresh/capture/chaos — because compose.yaml publishes FIXED host +# ports (8000/8080/8081/3000) and the capture leg records the FRESH stack's live stream (SPEC +# F178.5), so that stack has to stay up through capture and chaos. Run upgrade any earlier and the +# previous release's setup.sh preflight inside run_upgrade_leg finds those ports bound: observed on +# a real box 2026-09-16 against v5.8.3, the first replay of release.yml's `--fresh --upgrade +# --capture` combination ("Port 8080 is already in use", the leg failing at "setup" with an empty +# compose-upgrade.log). T495 only ever ran --upgrade alone and the nightly runs --fresh --capture +# --chaos (gh-#777/T503). teardown_project brings the fresh stack down and drops its PROJECTS entry +# first so cleanup()'s EXIT trap does not run a second `down -v`; a run without --upgrade never +# calls it, so the fresh stack stays up until EXIT as before. +if [ "$DO_UPGRADE" = 1 ]; then + [ "$DO_FRESH" = 1 ] && teardown_project "$FRESH_SCRATCH" "$FRESH_PROJECT" + run_upgrade_leg +else + LEG_STATUS[upgrade]="skipped"; LEG_DETAIL[upgrade]="--upgrade not given" +fi + # --------------------------------------------------------------------------------------------- # Report — gate-report.md (a small table + measurements + the fixed manual-evidence line) and # its machine twin gate-report.json. Written on every path that reaches here, before exiting.