diff --git a/docs/operations/relay-observability.md b/docs/operations/relay-observability.md index 6a1d7579b45ca..3f174e6882de1 100644 --- a/docs/operations/relay-observability.md +++ b/docs/operations/relay-observability.md @@ -121,6 +121,12 @@ Monitors count HTTP status codes, not span errors. A request the relay cuts off ends as an interrupted span without an error flag, but still answers 504. `/connect` 502s are normal: offline hosts produce most of them, retried by their clients every few seconds. +The dashboard's "Connecting" charts show connections as users see them. Web, desktop, and mobile +export a root `relay.connection.attempt` span per attempt, which lasts until initial sync finishes, in +the same trace as the relay's and the host's spans. Most reconnects reuse a cached relay token and go +straight to the host, so relay latency only affects cold connects. A slow total with fast phases +usually means the client waited out an unreachable direct (LAN) route before falling back to the relay. + **Links failing / tunnel recovery failing.** Open the dashboard's "Link failures by reason" table. `provisioning: ensure-tunnel` or `configure-tunnel` with Cloudflare 429s on the "Cloudflare API calls by status" chart is the #16810 failure: the account's Cloudflare API rate limit. Ask Cloudflare to diff --git a/infra/relay/src/monitoring.ts b/infra/relay/src/monitoring.ts index 5bdc99af1ad67..f791dfc450265 100644 --- a/infra/relay/src/monitoring.ts +++ b/infra/relay/src/monitoring.ts @@ -193,6 +193,7 @@ const relayDashboardCharts = (dataset: string): ReadonlyArray => { const source = `['${dataset}']`; const linkRequests = `${source}\n| where kind == "server" and ['attributes.http.request.method'] == "POST" and ['attributes.http.route'] == ${LINK_ROUTE}`; const tunnelRequests = `${source}\n| where kind == "server" and ['attributes.http.request.method'] == "POST" and ['attributes.http.route'] in (${TUNNEL_ROUTES})`; + const clientConnections = `${source}\n| where name == "relay.connection.attempt" and error != true\n| extend surface = tostring(['resource.custom']['t3.client.surface'])\n| where isnotempty(surface)`; return [ timeSeries( "links", @@ -242,6 +243,52 @@ const relayDashboardCharts = (dataset: string): ReadonlyArray => { apl: `${source}\n| where name == "relay.environment_linker.link" and error == true\n| extend message = tostring(['status.message'])\n| summarize failures = count() by reason = case(message contains "failed during '", strcat("provisioning: ", extract("failed during '([^']+)'", 1, message)), message startswith "Managed tunnel limit reached", "managed tunnel limit reached", substring(message, 0, 80))\n| top 10 by failures`, }, }, + // Connecting, as users see it: web, desktop and mobile export their + // connection spans into this dataset, in the same traces as the relay's + // and the host's spans. An attempt lasts until initial sync finishes. + timeSeries( + "connect-time", + "Connecting: time to connected by surface (ms)", + `${clientConnections}\n| summarize p50 = percentile(duration, 50) / 1000000, p90 = percentile(duration, 90) / 1000000 by bin_auto(_time), surface`, + ), + timeSeries( + "connect-outcomes", + "Connecting: attempts by outcome", + `${source}\n| where name == "relay.connection.attempt" and isnotempty(['resource.custom']['t3.client.surface'])\n| summarize connected = countif(error != true), failed = countif(error == true) by bin_auto(_time)`, + ), + { + id: "connect-phases", + name: "Connecting: where the time goes (ms)", + type: "Table", + query: { + // Failed direct route checks stay in: waiting out an unreachable LAN + // address before falling back to the relay is the slowest step. + apl: [ + source, + `| where isnotempty(['resource.custom']['t3.client.surface'])`, + `| extend url = tostring(['attributes.url.full'])`, + `| extend phase = case(`, + ` name == "ConnectionDriver.checkRoute", "0. Direct route check, before any relay fallback",`, + ` name == "relay.connection.cloudSessionToken.resolve", "1. Clerk session token",`, + ` name == "clientRuntime.managedRelay.exchangeAccessToken", "2. Relay token exchange",`, + ` kind == "client" and url startswith "https://relay.t3.codes/v1/environments/" and url endswith "/connect", "3. Relay connect, minting a host credential",`, + ` name == "environment.authorization.descriptor", "4. Host descriptor",`, + ` name == "environment.authorization.accessToken.exchange", "5. Host token exchange",`, + ` name == "clientRuntime.connection.remote.createDpopSocketUrl", "6. Socket ticket",`, + ` name == "environment.initialSync", "7. Initial sync",`, + ` name == "relay.connection.attempt", "Total: time to connected",`, + ` "")`, + `| where phase != "" and (error != true or phase startswith "0.")`, + `| summarize calls = count(), p50_ms = round(percentile(duration, 50) / 1000000, 0), p90_ms = round(percentile(duration, 90) / 1000000, 0) by phase`, + `| order by phase asc`, + ].join("\n"), + }, + }, + timeSeries( + "relay-latency-by-continent", + "Relay token and connect latency by user continent (p50 ms)", + `${source}\n| where kind == "server" and ['attributes.http.request.method'] == "POST" and (['attributes.http.route'] == "/v1/client/dpop-token" or ['attributes.http.route'] == "/v1/environments/:environmentId/connect")\n| extend continent = tostring(['attributes.custom']['relay.edge.continent'])\n| where isnotempty(continent)\n| summarize p50 = percentile(duration, 50) / 1000000 by bin_auto(_time), continent`, + ), ]; };