diff --git a/docs/dev/operations/deploying.md b/docs/dev/operations/deploying.md index 5980912f..8cd00e64 100644 --- a/docs/dev/operations/deploying.md +++ b/docs/dev/operations/deploying.md @@ -135,6 +135,10 @@ per-environment value is a runtime `environment` entry. piquelctl --timeout 30m env promote sourceorigin production --deployment --yes ``` +`just conformance release` runs the OpenID Foundation's conformance suite +against staging's current release first, then offers to promote that release: +see [test a release](https://github.com/sourceorigin-dev/sourceorigin/blob/main/tests/conformance/README.md#test-a-release). + piqueld refuses the promotion unless staging's current deployment succeeded and every service runs healthy. With `--deployment`, it also refuses if staging deployed again since, so it never promotes something you didn't test. diff --git a/docs/dev/operations/secrets.md b/docs/dev/operations/secrets.md index eb32e557..eb5cf5cb 100644 --- a/docs/dev/operations/secrets.md +++ b/docs/dev/operations/secrets.md @@ -128,7 +128,7 @@ Each is 32 random bytes, as hex or unpadded base64url text. | Secret | Mountable by | What it is | | --- | --- | --- | | `resend-production` | production | the Resend key production sends with: see [email](email.md) | -| `resend-dev` | staging and previews | the Resend key for `dev.sourceorigin.dev` | +| `resend-dev` | staging, conformance and previews | the Resend key for `dev.sourceorigin.dev` | | `github-production`, `github-staging` | that environment | its GitHub OAuth app's client secret: see [sign-in providers](sign-in-providers.md) | | `google-production`, `google-staging` | that environment | its Google OAuth client's secret | | `audit-key-production-2`, `audit-key-staging-2` | that environment | the seed that signs its audit checkpoints, whose verifier key is committed: see [audit history](../security/audit-history.md#the-audit-key) | diff --git a/infra/piqueld.toml b/infra/piqueld.toml index e3ddb615..802c906c 100644 --- a/infra/piqueld.toml +++ b/infra/piqueld.toml @@ -1,7 +1,8 @@ #:schema https://raw.githubusercontent.com/sourceorigin-dev/piqueld/main/docs/application-manifest.schema.json # SourceOrigin on piqueld. The environments below are the application's only ones: # staging builds `main` on every push, production only receives releases -# promoted from staging, and previews deploy branches or local worktrees. See +# promoted from staging, conformance tests them on the OpenID Foundation's hosted +# suite first, and previews deploy branches or local worktrees. See # docs/dev/operations/deploying.md. api_version = "piqueld.dev/v1alpha1" kind = "Application" @@ -80,6 +81,29 @@ github_secret = "github-staging" google_client_id = "981383173703-rvipsfet9n9270sqp30jttjrsl8beo66.apps.googleusercontent.com" google_secret = "google-staging" +# Staging's releases, public while `just conformance release` runs the OpenID +# Foundation's hosted conformance suite against them, stopped otherwise. Flat +# public hostnames, which Cloudflare's certificate for `*.sourceorigin.dev` covers, +# and the admin dashboard on the tailnet, under `*.dev.sourceorigin.dev`. It has +# no sign-in providers, and its test account no address to mail. See +# tests/conformance/README.md. +[spec.environments.conformance] +promote_from = "staging" + +[spec.environments.conformance.variables] +web_host = "conformance.sourceorigin.dev" +auth_host = "auth-conformance.sourceorigin.dev" +www_host = "www-conformance.sourceorigin.dev" +admin_host = "admin-conformance.dev.sourceorigin.dev" +docs_host = "docs-conformance.sourceorigin.dev" +smtp_secret = "resend-dev" +email_from = "Origin Account " +github_client_id = "" +github_secret = "no-provider-secret" +google_client_id = "" +google_secret = "no-provider-secret" +otlp_endpoint = "" + # Flat hostnames, so the `*.dev.sourceorigin.dev` certificate covers every preview. # Preview databases start empty, so anyone on the tailnet may register. [spec.previews] diff --git a/justfile b/justfile index d8193035..7aea2bdd 100644 --- a/justfile +++ b/justfile @@ -103,12 +103,14 @@ coverage *ARGS: close("sort"); print row("workspace", LH, LF, FH, FF) }' target/lcov.info -# Without a URL, it builds and starts auth itself, on PostgreSQL and Caddy in -# Docker. With one, CONFORMANCE_CLI must run that deployment's CLI. Needs Docker -# and Playwright's Chromium; see tests/conformance/README.md. +# Without a target, it builds and starts auth itself, on PostgreSQL and Caddy in +# Docker. With a URL, such as a preview's, CONFORMANCE_CLI must run that +# deployment's CLI. `release` tests staging's release on the OpenID Foundation's +# hosted suite, then offers to promote it to production. Needs Docker and +# Playwright's Chromium; see tests/conformance/README.md. # Run the OpenID conformance suite against auth, and compare with expected.json. -conformance AUTH_URL='': - @tests/conformance/run.sh {{ quote(AUTH_URL) }} +conformance TARGET='': + @tests/conformance/run.sh {{ quote(TARGET) }} # Regenerate tests/sdk-vectors; `just test` fails while the committed files are stale. sdk-vectors: diff --git a/tests/conformance/README.md b/tests/conformance/README.md index fbb58835..9c4d1563 100644 --- a/tests/conformance/README.md +++ b/tests/conformance/README.md @@ -15,7 +15,8 @@ It runs these plans, from [`plans.json`](plans.json): | `oidcc-rp-initiated-logout-certification-test-plan` | RP-Initiated Logout | | `oidcc-backchannel-rp-initiated-logout-certification-test-plan` | Back-Channel Logout | -The suite runs in Docker, from its prebuilt image. Its own browser can't create +The suite runs in Docker, from its prebuilt image, or, to test a release, on the +Foundation's own hosted suite. Its own browser can't create passkeys, so [`driver.ts`](driver.ts) plays the browser's part in Chromium with a virtual passkey, as the [browser tests](../browser/README.md) do. It signs an account up, then, for every test, opens each page the suite asks for, signs in, @@ -49,6 +50,49 @@ CONFORMANCE_TESTS=oidcc-server,oidcc-prompt-login just conformance Only the tests you name are compared then. +### Test a release + +Before promoting staging to production, test its release on the OpenID +Foundation's own hosted suite at , the one +certification uses: + +```console +export CONFORMANCE_SUITE_TOKEN=file:///path/to/token +just conformance release +``` + +1. The recipe finds staging's current deployment, and reads `plans.json` and + `expected.json` from the commit it was built from. +2. It promotes that release into the `conformance` environment, which makes it + public at `https://auth-conformance.sourceorigin.dev`. piquelctl shows what it + deploys and asks first. +3. It registers the suite's clients and a test account through the environment's + CLI, runs every plan on the hosted suite, and compares the results. +4. It stops the environment, whatever happened, so it is public only while the + run lasts. +5. If every result matched, it offers to promote the same deployment to + production, and piquelctl asks again. Otherwise it stops there. + +The token is the hosted suite's API token: sign in at + with Google or GitLab, then +create a permanent token at +. Keep it in a file outside the +repository, and point `CONFORMANCE_SUITE_TOKEN` at it, as `file:///path` or +`env://NAME`. + +You need `piquelctl` with `apps:deploy` and `apps:exec` on `sourceorigin`, and the +`conformance` environment needs the `resend-dev` secret, granted once: + +```console +piquelctl app secret sourceorigin access resend-dev --environments staging,conformance --previews +``` + +The account has no email address, since the run can't read the environment's +email, so `oidcc-scope-email` isn't compared. The plans stay in your account on +the hosted suite, where +[certification](https://github.com/sourceorigin-dev/sourceorigin/issues/52) can +use them. + ### Against a preview The suite reaches the preview from your machine, which must be on the tailnet. @@ -124,6 +168,7 @@ The recipe sets them for a local run. Against a preview, set | Variable | What it is | | --- | --- | | `CONFORMANCE_CLI` | A command that runs `auth-service` with the deployment's configuration. The driver appends its arguments. | +| `CONFORMANCE_SUITE_TOKEN` | For `release`: where the hosted suite's API token is, as `file:///path` or `env://NAME`. | | `CONFORMANCE_TESTS` | Comma-separated test names to run instead of every test. | | `CONFORMANCE_OUTPUT` | Where the results go, instead of `target/conformance`. | | `CHROMIUM_EXECUTABLE` | A Chromium to use instead of Playwright's. | diff --git a/tests/conformance/driver.ts b/tests/conformance/driver.ts index 86300c9b..1bc8f487 100644 --- a/tests/conformance/driver.ts +++ b/tests/conformance/driver.ts @@ -18,8 +18,13 @@ import { mkdirSync, readFileSync, writeFileSync } from 'node:fs'; import { join } from 'node:path'; import { mailed } from '../browser/helpers.ts'; -/** The suite's address, which docker-compose.yml gives it. */ -const suiteUrl = 'https://localhost:8443'; +/** + * The suite's address: the OpenID Foundation's hosted one, or the one + * docker-compose.yml runs on this machine, which serves a self-signed certificate. + */ +const suiteUrl = process.env.CONFORMANCE_SUITE_URL ?? 'https://localhost:8443'; +/** Whether the suite runs on this machine. */ +const localSuite = new URL(suiteUrl).hostname === 'localhost'; /** The name the suite gives this deployment in its callback URLs. */ const alias = 'sourceorigin'; /** How long one test may take, its waits included, before it counts as `UNKNOWN`. */ @@ -34,8 +39,8 @@ type Expectation = { result: Exclude; reason: stri /** * A plan to run, with its variant, from plans.json. A `backchannel` plan's client * receives back-channel logout tokens, which no other plan expects. It runs only - * against a local auth: the tokens must reach the suite on this machine, which a - * preview can't. + * when auth can post to the suite: a local auth, or any auth to the hosted suite, + * but not a preview to the suite on this machine. */ type Plan = { plan: string; variant: Record; backchannel?: boolean }; @@ -58,8 +63,18 @@ function required(name: string): string { const authUrl = required('CONFORMANCE_AUTH_URL').replace(/\/$/, ''); const output = required('CONFORMANCE_OUTPUT'); const only = process.env.CONFORMANCE_TESTS?.split(',').filter(Boolean); -/** Whether `run.sh` started auth on this machine. */ -const local = Boolean(process.env.CONFORMANCE_LOCAL); +/** Whether auth can post back-channel logout tokens to the suite. */ +const backchannel = Boolean(process.env.CONFORMANCE_BACKCHANNEL); + +/** + * The value `reference` names, as `file:///path` or `env://NAME`, as auth reads its + * secrets, such as the hosted suite's API token. + */ +function resolve(reference: string): string { + if (reference.startsWith('env://')) return required(reference.slice('env://'.length)); + if (reference.startsWith('file://')) return readFileSync(new URL(reference), 'utf8').trim(); + throw new Error(`${reference} must be file:///path or env://NAME`); +} /** The suite's REST API. */ class Suite { @@ -71,8 +86,13 @@ class Suite { /** Connects once the suite answers, which takes it a minute after starting. */ static async connect(): Promise { - // The suite serves its own self-signed certificate. - const api = await request.newContext({ baseURL: suiteUrl, ignoreHTTPSErrors: true }); + // The hosted suite takes the API token of its user's account. + const token = process.env.CONFORMANCE_SUITE_TOKEN; + const api = await request.newContext({ + baseURL: suiteUrl, + ignoreHTTPSErrors: localSuite, + extraHTTPHeaders: token ? { authorization: `Bearer ${resolve(token)}` } : {} + }); for (let attempt = 0; attempt < 120; attempt++) { const ready = await api.get('/api/plan?length=1').catch(() => undefined); if (ready?.ok()) return new Suite(api); @@ -261,8 +281,8 @@ class Browser { /** Launches Chromium with a virtual authenticator, which keeps its passkey for the run. */ static async launch(): Promise { const browser = await chromium.launch({ executablePath: process.env.CHROMIUM_EXECUTABLE }); - // The suite's certificate is always self-signed, and so is a local auth's. - const context = await browser.newContext({ ignoreHTTPSErrors: true }); + // The local suite's certificate is self-signed, and so is a local auth's. + const context = await browser.newContext({ ignoreHTTPSErrors: localSuite }); const page = await context.newPage(); const cdp = await context.newCDPSession(page); await cdp.send('WebAuthn.enable'); @@ -449,17 +469,20 @@ function differences( ) { found.push(`${test}: expected.json must link the issue that fixes it`); } - if (!only && local && !results.has(test)) + if (!only && backchannel && !results.has(test)) found.push(`${test}: in expected.json, but no plan ran it`); } return found; } const here = import.meta.dirname; -const plans = JSON.parse(readFileSync(join(here, 'plans.json'), 'utf8')) as Plan[]; -const expected = JSON.parse(readFileSync(join(here, 'expected.json'), 'utf8')) as Partial< - Record ->; +// A release is tested against its own commit's plans and expectations. +const plans = JSON.parse( + readFileSync(process.env.CONFORMANCE_PLANS ?? join(here, 'plans.json'), 'utf8') +) as Plan[]; +const expected = JSON.parse( + readFileSync(process.env.CONFORMANCE_EXPECTED ?? join(here, 'expected.json'), 'utf8') +) as Partial>; mkdirSync(join(output, 'screenshots'), { recursive: true }); const deployment = new Deployment(required('CONFORMANCE_CLI')); @@ -488,8 +511,10 @@ const results = new Map(); /** Each test's ID in the suite. */ const ids = new Map(); for (const plan of plans) { - if (plan.backchannel && !local) { - console.log(`${plan.plan}: skipped, as auth is not on this machine`); + if (plan.backchannel && !backchannel) { + console.log( + `${plan.plan}: skipped, as auth can't post back-channel logout tokens to the suite` + ); continue; } const created = await suite.createPlan( diff --git a/tests/conformance/run.sh b/tests/conformance/run.sh index 656a2806..92797d93 100755 --- a/tests/conformance/run.sh +++ b/tests/conformance/run.sh @@ -1,7 +1,14 @@ #!/usr/bin/env bash -# `just conformance [AUTH_URL]`: runs the OpenID conformance suite against auth -# at AUTH_URL, whose CLI CONFORMANCE_CLI runs, or, without a URL, against a -# local auth it builds and starts with its own PostgreSQL. See README.md. +# `just conformance [TARGET]`: runs the OpenID conformance suite and compares every +# result with expected.json. See README.md. +# +# (none) the suite on this machine, against a local auth this script builds +# and starts with its own PostgreSQL and Mailpit +# the suite on this machine, against auth at , such as a preview, +# whose CLI CONFORMANCE_CLI runs +# release the OpenID Foundation's hosted suite, against staging's current +# release promoted into the public `conformance` environment, stopped +# afterwards; then offers to promote that release to production set -euo pipefail here=$(cd "$(dirname "$0")" && pwd) @@ -17,18 +24,20 @@ mkdir -p "$CONFORMANCE_OUTPUT" cleanup=() finish() { for ((i = ${#cleanup[@]} - 1; i >= 0; i--)); do eval "${cleanup[i]}" || true; done + cleanup=() } trap finish EXIT -compose=(docker compose --file "$here/docker-compose.yml") -[[ -z "${1:-}" ]] && compose+=(--profile local) -cleanup+=("${compose[*]} logs --no-color > '$CONFORMANCE_OUTPUT/compose.log' 2>&1; ${compose[*]} down --volumes > /dev/null 2>&1") -"${compose[@]}" up --detach --wait --quiet-pull +# Starts the suite on this machine, with a local auth's services under `local`. +start_suite() { + compose=(docker compose --file "$here/docker-compose.yml" "$@") + cleanup+=("${compose[*]} logs --no-color > '$CONFORMANCE_OUTPUT/compose.log' 2>&1; ${compose[*]} down --volumes > /dev/null 2>&1") + "${compose[@]}" up --detach --wait --quiet-pull +} -if [[ -n "${1:-}" ]]; then - export CONFORMANCE_AUTH_URL=$1 - : "${CONFORMANCE_CLI:?set CONFORMANCE_CLI to a command that runs auth-service on the deployment}" -else +# Builds and starts auth on this machine, at https://localhost:7443. +start_auth() { + start_suite --profile local cargo build --locked -p auth-service secrets=$(mktemp -d) cleanup+=("rm -r '$secrets'") @@ -55,7 +64,7 @@ else export AUTH_REQUESTS_PER_MINUTE=1000 export CONFORMANCE_AUTH_URL=$AUTH_PUBLIC_BASE_URL export CONFORMANCE_CLI="$root/target/debug/auth-service --operator conformance" - export CONFORMANCE_LOCAL=1 + export CONFORMANCE_BACKCHANNEL=1 export AUTH_SMTP_URL=smtp://127.0.0.1:7025 export CONFORMANCE_MAIL_URL=http://127.0.0.1:7026 @@ -63,6 +72,58 @@ else target/debug/auth-service serve > "$CONFORMANCE_OUTPUT/auth.log" 2>&1 & cleanup+=("kill $!") curl -kfs --retry 30 --retry-delay 1 --retry-all-errors "$CONFORMANCE_AUTH_URL/health/ready" >/dev/null -fi +} + +# Promotes staging's current release into `conformance`, runs the hosted suite +# against it with the expectations of the commit it was built from, stops it, and +# offers to promote the same release to production. piquelctl asks before each +# promotion. +release() { + : "${CONFORMANCE_SUITE_TOKEN:?set CONFORMANCE_SUITE_TOKEN to where your certification.openid.net token is, such as file:///path}" + export CONFORMANCE_SUITE_URL=https://www.certification.openid.net + local current deployment commit + current=$(piquelctl --json env history sourceorigin staging --limit 20 | node -e ' + const { items } = JSON.parse(require("node:fs").readFileSync(0, "utf8")); + const current = items.find((item) => item.current_target && item.last_successful); + if (!current) throw new Error("staging has no current, successful deployment"); + console.log(current.operation.id, current.template.spec.manifest.repository.commit);') + read -r deployment commit <<< "$current" + echo "Testing staging's deployment $deployment, built from $commit." + + # The expectations, and the plans, of the commit under test. + git fetch --quiet origin "$commit" + local tested + tested=$(mktemp -d) + cleanup+=("rm -r '$tested'") + git show "$commit:tests/conformance/expected.json" > "$tested/expected.json" + git show "$commit:tests/conformance/plans.json" > "$tested/plans.json" + export CONFORMANCE_EXPECTED="$tested/expected.json" CONFORMANCE_PLANS="$tested/plans.json" + + cleanup+=("piquelctl env stop sourceorigin conformance --yes") + piquelctl --timeout 30m env promote sourceorigin conformance --deployment "$deployment" + export CONFORMANCE_AUTH_URL=https://auth-conformance.sourceorigin.dev + export CONFORMANCE_CLI="piquelctl app exec sourceorigin auth --env conformance -- auth-service --operator conformance" + # The hosted suite is public, so auth posts back-channel logout tokens to it. + export CONFORMANCE_BACKCHANNEL=1 + curl -fs --retry 30 --retry-delay 2 --retry-all-errors "$CONFORMANCE_AUTH_URL/health/ready" >/dev/null + + node "$here/driver.ts" + finish + piquelctl --timeout 30m env promote sourceorigin production --deployment "$deployment" +} -node "$here/driver.ts" +case "${1:-}" in + "") + start_auth + node "$here/driver.ts" + ;; + release) + release + ;; + *) + export CONFORMANCE_AUTH_URL=$1 + : "${CONFORMANCE_CLI:?set CONFORMANCE_CLI to a command that runs auth-service on the deployment}" + start_suite + node "$here/driver.ts" + ;; +esac