From de1be60adde3f5bfde3eeb9a979c3aa84cb4f2eb Mon Sep 17 00:00:00 2001 From: piquel Date: Thu, 12 Nov 2026 22:14:54 +0100 Subject: [PATCH] ci: test each release on the OpenID Foundation's hosted conformance suite `just conformance release` promotes staging's current release into a new public `conformance` environment, runs the hosted suite at www.certification.openid.net against it with that commit's plans and expectations, stops the environment whatever happens, and, if every result matched, offers to promote the same deployment to production. piquelctl asks before each promotion. The driver takes the suite's address and API token, and runs the back-channel logout plan whenever auth can reach the suite. --- docs/dev/operations/deploying.md | 4 ++ docs/dev/operations/secrets.md | 2 +- infra/piqueld.toml | 26 +++++++++- justfile | 12 +++-- tests/conformance/README.md | 47 ++++++++++++++++- tests/conformance/driver.ts | 59 +++++++++++++++------ tests/conformance/run.sh | 89 +++++++++++++++++++++++++++----- 7 files changed, 200 insertions(+), 39 deletions(-) diff --git a/docs/dev/operations/deploying.md b/docs/dev/operations/deploying.md index 5980912f..8cd00e64 100644 --- a/docs/dev/operations/deploying.md +++ b/docs/dev/operations/deploying.md @@ -135,6 +135,10 @@ per-environment value is a runtime `environment` entry. piquelctl --timeout 30m env promote sourceorigin production --deployment --yes ``` +`just conformance release` runs the OpenID Foundation's conformance suite +against staging's current release first, then offers to promote that release: +see [test a release](https://github.com/sourceorigin-dev/sourceorigin/blob/main/tests/conformance/README.md#test-a-release). + piqueld refuses the promotion unless staging's current deployment succeeded and every service runs healthy. With `--deployment`, it also refuses if staging deployed again since, so it never promotes something you didn't test. diff --git a/docs/dev/operations/secrets.md b/docs/dev/operations/secrets.md index eb32e557..eb5cf5cb 100644 --- a/docs/dev/operations/secrets.md +++ b/docs/dev/operations/secrets.md @@ -128,7 +128,7 @@ Each is 32 random bytes, as hex or unpadded base64url text. | Secret | Mountable by | What it is | | --- | --- | --- | | `resend-production` | production | the Resend key production sends with: see [email](email.md) | -| `resend-dev` | staging and previews | the Resend key for `dev.sourceorigin.dev` | +| `resend-dev` | staging, conformance and previews | the Resend key for `dev.sourceorigin.dev` | | `github-production`, `github-staging` | that environment | its GitHub OAuth app's client secret: see [sign-in providers](sign-in-providers.md) | | `google-production`, `google-staging` | that environment | its Google OAuth client's secret | | `audit-key-production-2`, `audit-key-staging-2` | that environment | the seed that signs its audit checkpoints, whose verifier key is committed: see [audit history](../security/audit-history.md#the-audit-key) | diff --git a/infra/piqueld.toml b/infra/piqueld.toml index e3ddb615..802c906c 100644 --- a/infra/piqueld.toml +++ b/infra/piqueld.toml @@ -1,7 +1,8 @@ #:schema https://raw.githubusercontent.com/sourceorigin-dev/piqueld/main/docs/application-manifest.schema.json # SourceOrigin on piqueld. The environments below are the application's only ones: # staging builds `main` on every push, production only receives releases -# promoted from staging, and previews deploy branches or local worktrees. See +# promoted from staging, conformance tests them on the OpenID Foundation's hosted +# suite first, and previews deploy branches or local worktrees. See # docs/dev/operations/deploying.md. api_version = "piqueld.dev/v1alpha1" kind = "Application" @@ -80,6 +81,29 @@ github_secret = "github-staging" google_client_id = "981383173703-rvipsfet9n9270sqp30jttjrsl8beo66.apps.googleusercontent.com" google_secret = "google-staging" +# Staging's releases, public while `just conformance release` runs the OpenID +# Foundation's hosted conformance suite against them, stopped otherwise. Flat +# public hostnames, which Cloudflare's certificate for `*.sourceorigin.dev` covers, +# and the admin dashboard on the tailnet, under `*.dev.sourceorigin.dev`. It has +# no sign-in providers, and its test account no address to mail. See +# tests/conformance/README.md. +[spec.environments.conformance] +promote_from = "staging" + +[spec.environments.conformance.variables] +web_host = "conformance.sourceorigin.dev" +auth_host = "auth-conformance.sourceorigin.dev" +www_host = "www-conformance.sourceorigin.dev" +admin_host = "admin-conformance.dev.sourceorigin.dev" +docs_host = "docs-conformance.sourceorigin.dev" +smtp_secret = "resend-dev" +email_from = "Origin Account " +github_client_id = "" +github_secret = "no-provider-secret" +google_client_id = "" +google_secret = "no-provider-secret" +otlp_endpoint = "" + # Flat hostnames, so the `*.dev.sourceorigin.dev` certificate covers every preview. # Preview databases start empty, so anyone on the tailnet may register. [spec.previews] diff --git a/justfile b/justfile index d8193035..7aea2bdd 100644 --- a/justfile +++ b/justfile @@ -103,12 +103,14 @@ coverage *ARGS: close("sort"); print row("workspace", LH, LF, FH, FF) }' target/lcov.info -# Without a URL, it builds and starts auth itself, on PostgreSQL and Caddy in -# Docker. With one, CONFORMANCE_CLI must run that deployment's CLI. Needs Docker -# and Playwright's Chromium; see tests/conformance/README.md. +# Without a target, it builds and starts auth itself, on PostgreSQL and Caddy in +# Docker. With a URL, such as a preview's, CONFORMANCE_CLI must run that +# deployment's CLI. `release` tests staging's release on the OpenID Foundation's +# hosted suite, then offers to promote it to production. Needs Docker and +# Playwright's Chromium; see tests/conformance/README.md. # Run the OpenID conformance suite against auth, and compare with expected.json. -conformance AUTH_URL='': - @tests/conformance/run.sh {{ quote(AUTH_URL) }} +conformance TARGET='': + @tests/conformance/run.sh {{ quote(TARGET) }} # Regenerate tests/sdk-vectors; `just test` fails while the committed files are stale. sdk-vectors: diff --git a/tests/conformance/README.md b/tests/conformance/README.md index fbb58835..9c4d1563 100644 --- a/tests/conformance/README.md +++ b/tests/conformance/README.md @@ -15,7 +15,8 @@ It runs these plans, from [`plans.json`](plans.json): | `oidcc-rp-initiated-logout-certification-test-plan` | RP-Initiated Logout | | `oidcc-backchannel-rp-initiated-logout-certification-test-plan` | Back-Channel Logout | -The suite runs in Docker, from its prebuilt image. Its own browser can't create +The suite runs in Docker, from its prebuilt image, or, to test a release, on the +Foundation's own hosted suite. Its own browser can't create passkeys, so [`driver.ts`](driver.ts) plays the browser's part in Chromium with a virtual passkey, as the [browser tests](../browser/README.md) do. It signs an account up, then, for every test, opens each page the suite asks for, signs in, @@ -49,6 +50,49 @@ CONFORMANCE_TESTS=oidcc-server,oidcc-prompt-login just conformance Only the tests you name are compared then. +### Test a release + +Before promoting staging to production, test its release on the OpenID +Foundation's own hosted suite at , the one +certification uses: + +```console +export CONFORMANCE_SUITE_TOKEN=file:///path/to/token +just conformance release +``` + +1. The recipe finds staging's current deployment, and reads `plans.json` and + `expected.json` from the commit it was built from. +2. It promotes that release into the `conformance` environment, which makes it + public at `https://auth-conformance.sourceorigin.dev`. piquelctl shows what it + deploys and asks first. +3. It registers the suite's clients and a test account through the environment's + CLI, runs every plan on the hosted suite, and compares the results. +4. It stops the environment, whatever happened, so it is public only while the + run lasts. +5. If every result matched, it offers to promote the same deployment to + production, and piquelctl asks again. Otherwise it stops there. + +The token is the hosted suite's API token: sign in at + with Google or GitLab, then +create a permanent token at +. Keep it in a file outside the +repository, and point `CONFORMANCE_SUITE_TOKEN` at it, as `file:///path` or +`env://NAME`. + +You need `piquelctl` with `apps:deploy` and `apps:exec` on `sourceorigin`, and the +`conformance` environment needs the `resend-dev` secret, granted once: + +```console +piquelctl app secret sourceorigin access resend-dev --environments staging,conformance --previews +``` + +The account has no email address, since the run can't read the environment's +email, so `oidcc-scope-email` isn't compared. The plans stay in your account on +the hosted suite, where +[certification](https://github.com/sourceorigin-dev/sourceorigin/issues/52) can +use them. + ### Against a preview The suite reaches the preview from your machine, which must be on the tailnet. @@ -124,6 +168,7 @@ The recipe sets them for a local run. Against a preview, set | Variable | What it is | | --- | --- | | `CONFORMANCE_CLI` | A command that runs `auth-service` with the deployment's configuration. The driver appends its arguments. | +| `CONFORMANCE_SUITE_TOKEN` | For `release`: where the hosted suite's API token is, as `file:///path` or `env://NAME`. | | `CONFORMANCE_TESTS` | Comma-separated test names to run instead of every test. | | `CONFORMANCE_OUTPUT` | Where the results go, instead of `target/conformance`. | | `CHROMIUM_EXECUTABLE` | A Chromium to use instead of Playwright's. | diff --git a/tests/conformance/driver.ts b/tests/conformance/driver.ts index 86300c9b..1bc8f487 100644 --- a/tests/conformance/driver.ts +++ b/tests/conformance/driver.ts @@ -18,8 +18,13 @@ import { mkdirSync, readFileSync, writeFileSync } from 'node:fs'; import { join } from 'node:path'; import { mailed } from '../browser/helpers.ts'; -/** The suite's address, which docker-compose.yml gives it. */ -const suiteUrl = 'https://localhost:8443'; +/** + * The suite's address: the OpenID Foundation's hosted one, or the one + * docker-compose.yml runs on this machine, which serves a self-signed certificate. + */ +const suiteUrl = process.env.CONFORMANCE_SUITE_URL ?? 'https://localhost:8443'; +/** Whether the suite runs on this machine. */ +const localSuite = new URL(suiteUrl).hostname === 'localhost'; /** The name the suite gives this deployment in its callback URLs. */ const alias = 'sourceorigin'; /** How long one test may take, its waits included, before it counts as `UNKNOWN`. */ @@ -34,8 +39,8 @@ type Expectation = { result: Exclude; reason: stri /** * A plan to run, with its variant, from plans.json. A `backchannel` plan's client * receives back-channel logout tokens, which no other plan expects. It runs only - * against a local auth: the tokens must reach the suite on this machine, which a - * preview can't. + * when auth can post to the suite: a local auth, or any auth to the hosted suite, + * but not a preview to the suite on this machine. */ type Plan = { plan: string; variant: Record; backchannel?: boolean }; @@ -58,8 +63,18 @@ function required(name: string): string { const authUrl = required('CONFORMANCE_AUTH_URL').replace(/\/$/, ''); const output = required('CONFORMANCE_OUTPUT'); const only = process.env.CONFORMANCE_TESTS?.split(',').filter(Boolean); -/** Whether `run.sh` started auth on this machine. */ -const local = Boolean(process.env.CONFORMANCE_LOCAL); +/** Whether auth can post back-channel logout tokens to the suite. */ +const backchannel = Boolean(process.env.CONFORMANCE_BACKCHANNEL); + +/** + * The value `reference` names, as `file:///path` or `env://NAME`, as auth reads its + * secrets, such as the hosted suite's API token. + */ +function resolve(reference: string): string { + if (reference.startsWith('env://')) return required(reference.slice('env://'.length)); + if (reference.startsWith('file://')) return readFileSync(new URL(reference), 'utf8').trim(); + throw new Error(`${reference} must be file:///path or env://NAME`); +} /** The suite's REST API. */ class Suite { @@ -71,8 +86,13 @@ class Suite { /** Connects once the suite answers, which takes it a minute after starting. */ static async connect(): Promise { - // The suite serves its own self-signed certificate. - const api = await request.newContext({ baseURL: suiteUrl, ignoreHTTPSErrors: true }); + // The hosted suite takes the API token of its user's account. + const token = process.env.CONFORMANCE_SUITE_TOKEN; + const api = await request.newContext({ + baseURL: suiteUrl, + ignoreHTTPSErrors: localSuite, + extraHTTPHeaders: token ? { authorization: `Bearer ${resolve(token)}` } : {} + }); for (let attempt = 0; attempt < 120; attempt++) { const ready = await api.get('/api/plan?length=1').catch(() => undefined); if (ready?.ok()) return new Suite(api); @@ -261,8 +281,8 @@ class Browser { /** Launches Chromium with a virtual authenticator, which keeps its passkey for the run. */ static async launch(): Promise { const browser = await chromium.launch({ executablePath: process.env.CHROMIUM_EXECUTABLE }); - // The suite's certificate is always self-signed, and so is a local auth's. - const context = await browser.newContext({ ignoreHTTPSErrors: true }); + // The local suite's certificate is self-signed, and so is a local auth's. + const context = await browser.newContext({ ignoreHTTPSErrors: localSuite }); const page = await context.newPage(); const cdp = await context.newCDPSession(page); await cdp.send('WebAuthn.enable'); @@ -449,17 +469,20 @@ function differences( ) { found.push(`${test}: expected.json must link the issue that fixes it`); } - if (!only && local && !results.has(test)) + if (!only && backchannel && !results.has(test)) found.push(`${test}: in expected.json, but no plan ran it`); } return found; } const here = import.meta.dirname; -const plans = JSON.parse(readFileSync(join(here, 'plans.json'), 'utf8')) as Plan[]; -const expected = JSON.parse(readFileSync(join(here, 'expected.json'), 'utf8')) as Partial< - Record ->; +// A release is tested against its own commit's plans and expectations. +const plans = JSON.parse( + readFileSync(process.env.CONFORMANCE_PLANS ?? join(here, 'plans.json'), 'utf8') +) as Plan[]; +const expected = JSON.parse( + readFileSync(process.env.CONFORMANCE_EXPECTED ?? join(here, 'expected.json'), 'utf8') +) as Partial>; mkdirSync(join(output, 'screenshots'), { recursive: true }); const deployment = new Deployment(required('CONFORMANCE_CLI')); @@ -488,8 +511,10 @@ const results = new Map(); /** Each test's ID in the suite. */ const ids = new Map(); for (const plan of plans) { - if (plan.backchannel && !local) { - console.log(`${plan.plan}: skipped, as auth is not on this machine`); + if (plan.backchannel && !backchannel) { + console.log( + `${plan.plan}: skipped, as auth can't post back-channel logout tokens to the suite` + ); continue; } const created = await suite.createPlan( diff --git a/tests/conformance/run.sh b/tests/conformance/run.sh index 656a2806..92797d93 100755 --- a/tests/conformance/run.sh +++ b/tests/conformance/run.sh @@ -1,7 +1,14 @@ #!/usr/bin/env bash -# `just conformance [AUTH_URL]`: runs the OpenID conformance suite against auth -# at AUTH_URL, whose CLI CONFORMANCE_CLI runs, or, without a URL, against a -# local auth it builds and starts with its own PostgreSQL. See README.md. +# `just conformance [TARGET]`: runs the OpenID conformance suite and compares every +# result with expected.json. See README.md. +# +# (none) the suite on this machine, against a local auth this script builds +# and starts with its own PostgreSQL and Mailpit +# the suite on this machine, against auth at , such as a preview, +# whose CLI CONFORMANCE_CLI runs +# release the OpenID Foundation's hosted suite, against staging's current +# release promoted into the public `conformance` environment, stopped +# afterwards; then offers to promote that release to production set -euo pipefail here=$(cd "$(dirname "$0")" && pwd) @@ -17,18 +24,20 @@ mkdir -p "$CONFORMANCE_OUTPUT" cleanup=() finish() { for ((i = ${#cleanup[@]} - 1; i >= 0; i--)); do eval "${cleanup[i]}" || true; done + cleanup=() } trap finish EXIT -compose=(docker compose --file "$here/docker-compose.yml") -[[ -z "${1:-}" ]] && compose+=(--profile local) -cleanup+=("${compose[*]} logs --no-color > '$CONFORMANCE_OUTPUT/compose.log' 2>&1; ${compose[*]} down --volumes > /dev/null 2>&1") -"${compose[@]}" up --detach --wait --quiet-pull +# Starts the suite on this machine, with a local auth's services under `local`. +start_suite() { + compose=(docker compose --file "$here/docker-compose.yml" "$@") + cleanup+=("${compose[*]} logs --no-color > '$CONFORMANCE_OUTPUT/compose.log' 2>&1; ${compose[*]} down --volumes > /dev/null 2>&1") + "${compose[@]}" up --detach --wait --quiet-pull +} -if [[ -n "${1:-}" ]]; then - export CONFORMANCE_AUTH_URL=$1 - : "${CONFORMANCE_CLI:?set CONFORMANCE_CLI to a command that runs auth-service on the deployment}" -else +# Builds and starts auth on this machine, at https://localhost:7443. +start_auth() { + start_suite --profile local cargo build --locked -p auth-service secrets=$(mktemp -d) cleanup+=("rm -r '$secrets'") @@ -55,7 +64,7 @@ else export AUTH_REQUESTS_PER_MINUTE=1000 export CONFORMANCE_AUTH_URL=$AUTH_PUBLIC_BASE_URL export CONFORMANCE_CLI="$root/target/debug/auth-service --operator conformance" - export CONFORMANCE_LOCAL=1 + export CONFORMANCE_BACKCHANNEL=1 export AUTH_SMTP_URL=smtp://127.0.0.1:7025 export CONFORMANCE_MAIL_URL=http://127.0.0.1:7026 @@ -63,6 +72,58 @@ else target/debug/auth-service serve > "$CONFORMANCE_OUTPUT/auth.log" 2>&1 & cleanup+=("kill $!") curl -kfs --retry 30 --retry-delay 1 --retry-all-errors "$CONFORMANCE_AUTH_URL/health/ready" >/dev/null -fi +} + +# Promotes staging's current release into `conformance`, runs the hosted suite +# against it with the expectations of the commit it was built from, stops it, and +# offers to promote the same release to production. piquelctl asks before each +# promotion. +release() { + : "${CONFORMANCE_SUITE_TOKEN:?set CONFORMANCE_SUITE_TOKEN to where your certification.openid.net token is, such as file:///path}" + export CONFORMANCE_SUITE_URL=https://www.certification.openid.net + local current deployment commit + current=$(piquelctl --json env history sourceorigin staging --limit 20 | node -e ' + const { items } = JSON.parse(require("node:fs").readFileSync(0, "utf8")); + const current = items.find((item) => item.current_target && item.last_successful); + if (!current) throw new Error("staging has no current, successful deployment"); + console.log(current.operation.id, current.template.spec.manifest.repository.commit);') + read -r deployment commit <<< "$current" + echo "Testing staging's deployment $deployment, built from $commit." + + # The expectations, and the plans, of the commit under test. + git fetch --quiet origin "$commit" + local tested + tested=$(mktemp -d) + cleanup+=("rm -r '$tested'") + git show "$commit:tests/conformance/expected.json" > "$tested/expected.json" + git show "$commit:tests/conformance/plans.json" > "$tested/plans.json" + export CONFORMANCE_EXPECTED="$tested/expected.json" CONFORMANCE_PLANS="$tested/plans.json" + + cleanup+=("piquelctl env stop sourceorigin conformance --yes") + piquelctl --timeout 30m env promote sourceorigin conformance --deployment "$deployment" + export CONFORMANCE_AUTH_URL=https://auth-conformance.sourceorigin.dev + export CONFORMANCE_CLI="piquelctl app exec sourceorigin auth --env conformance -- auth-service --operator conformance" + # The hosted suite is public, so auth posts back-channel logout tokens to it. + export CONFORMANCE_BACKCHANNEL=1 + curl -fs --retry 30 --retry-delay 2 --retry-all-errors "$CONFORMANCE_AUTH_URL/health/ready" >/dev/null + + node "$here/driver.ts" + finish + piquelctl --timeout 30m env promote sourceorigin production --deployment "$deployment" +} -node "$here/driver.ts" +case "${1:-}" in + "") + start_auth + node "$here/driver.ts" + ;; + release) + release + ;; + *) + export CONFORMANCE_AUTH_URL=$1 + : "${CONFORMANCE_CLI:?set CONFORMANCE_CLI to a command that runs auth-service on the deployment}" + start_suite + node "$here/driver.ts" + ;; +esac