From ad496606531b9d35bdd5b00885ee16c6e211d684 Mon Sep 17 00:00:00 2001 From: Julius Marminge <51714798+juliusmarminge@users.noreply.github.com> Date: Tue, 27 Oct 2026 14:11:20 -0700 Subject: [PATCH 1/2] fix(relay): stop delivery attempts from growing the database forever relay_delivery_attempts gained 1.35M rows a day and was never pruned; its insert was 28.5% of database time. Prune rows older than 7 days from the five-minute cron in bounded batches, index created_at for that, and drop three indexes Insights shows are never read. Co-Authored-By: Claude Opus 5.5 --- .../migration.sql | 6 + .../snapshot.json | 1571 +++++++++++++++++ .../src/agentActivity/ApnsDeliveries.test.ts | 1 + .../agentActivity/DeliveryAttempts.test.ts | 44 + .../src/agentActivity/DeliveryAttempts.ts | 56 +- .../agentActivity/MobileRegistrations.test.ts | 1 + infra/relay/src/persistence/schema.ts | 8 +- infra/relay/src/worker.ts | 5 + 8 files changed, 1683 insertions(+), 9 deletions(-) create mode 100644 infra/relay/migrations/postgres/20261027210820_delivery_attempts_retention/migration.sql create mode 100644 infra/relay/migrations/postgres/20261027210820_delivery_attempts_retention/snapshot.json diff --git a/infra/relay/migrations/postgres/20261027210820_delivery_attempts_retention/migration.sql b/infra/relay/migrations/postgres/20261027210820_delivery_attempts_retention/migration.sql new file mode 100644 index 000000000000..4138c71522a7 --- /dev/null +++ b/infra/relay/migrations/postgres/20261027210820_delivery_attempts_retention/migration.sql @@ -0,0 +1,6 @@ +-- Alchemy runs migrations in a transaction, so CONCURRENTLY is unavailable here. Build +-- idx_relay_delivery_attempts_created concurrently before deploying so this is a no-op. +DROP INDEX IF EXISTS "idx_relay_delivery_attempts_environment";--> statement-breakpoint +DROP INDEX IF EXISTS "idx_relay_agent_activity_rows_updated";--> statement-breakpoint +DROP INDEX IF EXISTS "idx_relay_environment_credentials_environment";--> statement-breakpoint +CREATE INDEX IF NOT EXISTS "idx_relay_delivery_attempts_created" ON "relay_delivery_attempts" ("created_at"); diff --git a/infra/relay/migrations/postgres/20261027210820_delivery_attempts_retention/snapshot.json b/infra/relay/migrations/postgres/20261027210820_delivery_attempts_retention/snapshot.json new file mode 100644 index 000000000000..05c1ee2d059e --- /dev/null +++ b/infra/relay/migrations/postgres/20261027210820_delivery_attempts_retention/snapshot.json @@ -0,0 +1,1571 @@ +{ + "version": "8", + "dialect": "postgres", + "id": "7d8c1702-6f47-4453-a8ee-66ce5e3bfb41", + "prevIds": ["f8594baa-6f4a-4a0d-a7f0-0aa717dfe785"], + "ddl": [ + { + "isRlsEnabled": false, + "name": "relay_agent_activity_rows", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "relay_delivery_attempts", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "relay_dpop_proofs", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "relay_environment_credentials", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "relay_environment_links", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "relay_live_activities", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "relay_managed_endpoint_allocations", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "relay_managed_tunnel_limits", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "relay_mobile_devices", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "relay_upstream_api_pauses", + "entityType": "tables", + "schema": "public" + }, + { + "type": "varchar(191)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "environment_id", + "entityType": "columns", + "schema": "public", + "table": "relay_agent_activity_rows" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "environment_public_key", + "entityType": "columns", + "schema": "public", + "table": "relay_agent_activity_rows" + }, + { + "type": "varchar(512)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "thread_id", + "entityType": "columns", + "schema": "public", + "table": "relay_agent_activity_rows" + }, + { + "type": "jsonb", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "state_json", + "entityType": "columns", + "schema": "public", + "table": "relay_agent_activity_rows" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "relay_agent_activity_rows" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "relay_agent_activity_rows" + }, + { + "type": "varchar(36)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "type": "varchar(255)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "user_id", + "entityType": "columns", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "type": "varchar(191)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "environment_id", + "entityType": "columns", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "type": "varchar(512)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "thread_id", + "entityType": "columns", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "type": "varchar(255)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "device_id", + "entityType": "columns", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "kind", + "entityType": "columns", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "source_job_id", + "entityType": "columns", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "type": "varchar(16)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "token_suffix", + "entityType": "columns", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "apns_status", + "entityType": "columns", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "apns_reason", + "entityType": "columns", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "type": "varchar(128)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "apns_id", + "entityType": "columns", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "transport_error", + "entityType": "columns", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "type": "varchar(128)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "thumbprint", + "entityType": "columns", + "schema": "public", + "table": "relay_dpop_proofs" + }, + { + "type": "varchar(255)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "jti", + "entityType": "columns", + "schema": "public", + "table": "relay_dpop_proofs" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "iat", + "entityType": "columns", + "schema": "public", + "table": "relay_dpop_proofs" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "expires_at", + "entityType": "columns", + "schema": "public", + "table": "relay_dpop_proofs" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "relay_dpop_proofs" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "credential_id", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_credentials" + }, + { + "type": "varchar(191)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "environment_id", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_credentials" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "environment_public_key", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_credentials" + }, + { + "type": "varchar(191)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "credential_hash", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_credentials" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "revoked_at", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_credentials" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_credentials" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_credentials" + }, + { + "type": "varchar(191)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "user_id", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "varchar(191)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "environment_id", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'T3 Environment'", + "generated": null, + "identity": null, + "name": "environment_label", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "environment_public_key", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "endpoint_http_base_url", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "endpoint_ws_base_url", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "varchar(32)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "endpoint_provider_kind", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "boolean", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "true", + "generated": null, + "identity": null, + "name": "notifications_enabled", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "boolean", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "true", + "generated": null, + "identity": null, + "name": "live_activities_enabled", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "boolean", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "false", + "generated": null, + "identity": null, + "name": "managed_tunnels_enabled", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "boolean", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "false", + "generated": null, + "identity": null, + "name": "hold_webhooks_while_offline", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "varchar(191)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_by_device_id", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "revoked_at", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "varchar(255)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "user_id", + "entityType": "columns", + "schema": "public", + "table": "relay_live_activities" + }, + { + "type": "varchar(255)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "device_id", + "entityType": "columns", + "schema": "public", + "table": "relay_live_activities" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "activity_push_token", + "entityType": "columns", + "schema": "public", + "table": "relay_live_activities" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "remote_start_queued_at", + "entityType": "columns", + "schema": "public", + "table": "relay_live_activities" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "remote_started_at", + "entityType": "columns", + "schema": "public", + "table": "relay_live_activities" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "ended_at", + "entityType": "columns", + "schema": "public", + "table": "relay_live_activities" + }, + { + "type": "jsonb", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "last_aggregate_json", + "entityType": "columns", + "schema": "public", + "table": "relay_live_activities" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "last_live_activity_delivery_at", + "entityType": "columns", + "schema": "public", + "table": "relay_live_activities" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "relay_live_activities" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "relay_live_activities" + }, + { + "type": "varchar(191)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "user_id", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "type": "varchar(191)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "environment_id", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "hostname", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "type": "varchar(191)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "tunnel_id", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "tunnel_name", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "type": "varchar(191)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "dns_record_id", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "ready_at", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "recovery_enabled_at", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "recovery_environment_public_key", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "tunnel_released_at", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "type": "jsonb", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "origin", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "0", + "generated": null, + "identity": null, + "name": "generation", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "type": "varchar(191)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "user_id", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_tunnel_limits" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "max_tunnels", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_tunnel_limits" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_tunnel_limits" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_tunnel_limits" + }, + { + "type": "varchar(255)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "user_id", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "type": "varchar(255)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "device_id", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'iOS device'", + "generated": null, + "identity": null, + "name": "label", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "type": "varchar(16)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "platform", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "ios_major_version", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "android_api_level", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "app_version", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "type": "varchar(255)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "bundle_id", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "type": "varchar(16)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "aps_environment", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "push_token", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "push_to_start_token", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "type": "jsonb", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "preferences_json", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "api", + "entityType": "columns", + "schema": "public", + "table": "relay_upstream_api_pauses" + }, + { + "type": "bigint", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "paused_until_ms", + "entityType": "columns", + "schema": "public", + "table": "relay_upstream_api_pauses" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "created_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_relay_delivery_attempts_created", + "entityType": "indexes", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "source_job_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_relay_delivery_attempts_source_job", + "entityType": "indexes", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "expires_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_relay_dpop_proofs_expires_at", + "entityType": "indexes", + "schema": "public", + "table": "relay_dpop_proofs" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "credential_hash", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_relay_environment_credentials_hash", + "entityType": "indexes", + "schema": "public", + "table": "relay_environment_credentials" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "environment_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "environment_public_key", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "revoked_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_relay_environment_credentials_environment_key", + "entityType": "indexes", + "schema": "public", + "table": "relay_environment_credentials" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "environment_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "revoked_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_relay_environment_links_environment", + "entityType": "indexes", + "schema": "public", + "table": "relay_environment_links" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "activity_push_token", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_relay_live_activities_activity_push_token", + "entityType": "indexes", + "schema": "public", + "table": "relay_live_activities" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "hostname", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_relay_managed_endpoint_allocations_hostname", + "entityType": "indexes", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "tunnel_name", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_relay_managed_endpoint_allocations_tunnel_name", + "entityType": "indexes", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "push_token", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_relay_mobile_devices_push_token", + "entityType": "indexes", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "push_to_start_token", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_relay_mobile_devices_push_to_start_token", + "entityType": "indexes", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "columns": ["environment_id", "environment_public_key", "thread_id"], + "nameExplicit": false, + "name": "relay_agent_activity_rows_pkey", + "entityType": "pks", + "schema": "public", + "table": "relay_agent_activity_rows" + }, + { + "columns": ["thumbprint", "jti"], + "nameExplicit": false, + "name": "relay_dpop_proofs_pkey", + "entityType": "pks", + "schema": "public", + "table": "relay_dpop_proofs" + }, + { + "columns": ["user_id", "environment_id"], + "nameExplicit": false, + "name": "relay_environment_links_pkey", + "entityType": "pks", + "schema": "public", + "table": "relay_environment_links" + }, + { + "columns": ["user_id", "device_id"], + "nameExplicit": false, + "name": "relay_live_activities_pkey", + "entityType": "pks", + "schema": "public", + "table": "relay_live_activities" + }, + { + "columns": ["user_id", "environment_id"], + "nameExplicit": false, + "name": "relay_managed_endpoint_allocations_pkey", + "entityType": "pks", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "columns": ["user_id", "device_id"], + "nameExplicit": false, + "name": "relay_mobile_devices_pkey", + "entityType": "pks", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "columns": ["id"], + "nameExplicit": false, + "name": "relay_delivery_attempts_pkey", + "schema": "public", + "table": "relay_delivery_attempts", + "entityType": "pks" + }, + { + "columns": ["credential_id"], + "nameExplicit": false, + "name": "relay_environment_credentials_pkey", + "schema": "public", + "table": "relay_environment_credentials", + "entityType": "pks" + }, + { + "columns": ["user_id"], + "nameExplicit": false, + "name": "relay_managed_tunnel_limits_pkey", + "schema": "public", + "table": "relay_managed_tunnel_limits", + "entityType": "pks" + }, + { + "columns": ["api"], + "nameExplicit": false, + "name": "relay_upstream_api_pauses_pkey", + "schema": "public", + "table": "relay_upstream_api_pauses", + "entityType": "pks" + } + ], + "renames": [] +} diff --git a/infra/relay/src/agentActivity/ApnsDeliveries.test.ts b/infra/relay/src/agentActivity/ApnsDeliveries.test.ts index ddabfe557fa5..4614b757532d 100644 --- a/infra/relay/src/agentActivity/ApnsDeliveries.test.ts +++ b/infra/relay/src/agentActivity/ApnsDeliveries.test.ts @@ -190,6 +190,7 @@ function layerFor(input: { }), }), Layer.succeed(DeliveryAttempts.DeliveryAttempts, { + pruneExpired: Effect.succeed(0), record: (attempt) => Effect.sync(() => { input.attempts.push(attempt); diff --git a/infra/relay/src/agentActivity/DeliveryAttempts.test.ts b/infra/relay/src/agentActivity/DeliveryAttempts.test.ts index 056b906bdb43..7e8f1b1a8c1c 100644 --- a/infra/relay/src/agentActivity/DeliveryAttempts.test.ts +++ b/infra/relay/src/agentActivity/DeliveryAttempts.test.ts @@ -168,4 +168,48 @@ describe("DeliveryAttempts", () => { ), ); }); + + const pruneWith = (batches: ReadonlyArray) => { + const deleteCalls: number[] = []; + const fakeDb = { + select: () => ({ from: () => ({ where: () => ({ limit: () => ({}) }) }) }), + delete: () => ({ + where: () => ({ + returning: () => { + const size = batches[deleteCalls.length] ?? 0; + deleteCalls.push(size); + return Effect.succeed(Array.from({ length: size }, (_, index) => ({ id: `${index}` }))); + }, + }), + }), + } as unknown as RelayDb.RelayDb["Service"]; + const run = DeliveryAttempts.DeliveryAttempts.pipe( + Effect.flatMap((attempts) => attempts.pruneExpired), + Effect.provide( + DeliveryAttempts.layer.pipe( + Layer.provide(NodeCryptoLayer.layer), + Layer.provide(Layer.succeed(RelayDb.RelayDb, fakeDb)), + ), + ), + ); + return { run, deleteCalls }; + }; + + it.effect("stops pruning once a batch comes back short", () => { + const full = DeliveryAttempts.DELIVERY_ATTEMPT_PRUNE_BATCH_SIZE; + const { run, deleteCalls } = pruneWith([full, 12]); + return Effect.gen(function* () { + expect(yield* run).toBe(full + 12); + expect(deleteCalls).toEqual([full, 12]); + }); + }); + + it.effect("caps each run at the batch limit while a backlog remains", () => { + const full = DeliveryAttempts.DELIVERY_ATTEMPT_PRUNE_BATCH_SIZE; + const { run, deleteCalls } = pruneWith(Array.from({ length: 10 }, () => full)); + return Effect.gen(function* () { + expect(yield* run).toBe(full * DeliveryAttempts.DELIVERY_ATTEMPT_PRUNE_MAX_BATCHES); + expect(deleteCalls).toHaveLength(DeliveryAttempts.DELIVERY_ATTEMPT_PRUNE_MAX_BATCHES); + }); + }); }); diff --git a/infra/relay/src/agentActivity/DeliveryAttempts.ts b/infra/relay/src/agentActivity/DeliveryAttempts.ts index 3e938617a5be..b64bf974206d 100644 --- a/infra/relay/src/agentActivity/DeliveryAttempts.ts +++ b/infra/relay/src/agentActivity/DeliveryAttempts.ts @@ -3,7 +3,7 @@ import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; -import { and, eq, isNull } from "drizzle-orm"; +import { and, eq, inArray, isNull, lt } from "drizzle-orm"; import * as Crypto from "effect/Crypto"; import * as Schema from "effect/Schema"; @@ -13,7 +13,12 @@ import { relayDeliveryAttempts } from "../persistence/schema.ts"; export class DeliveryAttemptRecordPersistenceError extends Schema.TaggedError()( "DeliveryAttemptRecordPersistenceError", { - operation: Schema.Literals(["record", "claim-source-job", "complete-source-job"]), + operation: Schema.Literals([ + "record", + "claim-source-job", + "complete-source-job", + "prune-expired", + ]), sourceJobId: Schema.NullOr(Schema.String), userId: Schema.NullOr(Schema.String), environmentId: Schema.NullOr(Schema.String), @@ -64,10 +69,16 @@ export class DeliveryAttempts extends Context.Service< readonly completeSourceJob: ( input: DeliveryAttemptCompletionInput, ) => Effect.Effect; + /** Delete attempts untouched for the retention period; returns how many were removed. */ + readonly pruneExpired: Effect.Effect; } >()("t3code-relay/agentActivity/DeliveryAttempts") {} const SOURCE_JOB_CLAIM_LEASE_MINUTES = 10; +export const DELIVERY_ATTEMPT_RETENTION_DAYS = 7; +export const DELIVERY_ATTEMPT_PRUNE_BATCH_SIZE = 5_000; +// Caps each cron run so draining the pre-retention backlog cannot monopolize the database. +export const DELIVERY_ATTEMPT_PRUNE_MAX_BATCHES = 4; function insertValues( input: DeliveryAttemptInput, @@ -246,6 +257,47 @@ export const make = Effect.gen(function* () { ), ); }), + pruneExpired: Effect.gen(function* () { + const before = DateTime.formatIso( + DateTime.subtract(yield* DateTime.now, { days: DELIVERY_ATTEMPT_RETENTION_DAYS }), + ); + yield* Effect.annotateCurrentSpan({ "relay.delivery_attempts_prune.before": before }); + let deleted = 0; + for (let batch = 0; batch < DELIVERY_ATTEMPT_PRUNE_MAX_BATCHES; batch += 1) { + const rows = yield* db + .delete(relayDeliveryAttempts) + .where( + inArray( + relayDeliveryAttempts.id, + db + .select({ id: relayDeliveryAttempts.id }) + .from(relayDeliveryAttempts) + .where(lt(relayDeliveryAttempts.createdAt, before)) + .limit(DELIVERY_ATTEMPT_PRUNE_BATCH_SIZE), + ), + ) + .returning({ id: relayDeliveryAttempts.id }); + deleted += rows.length; + if (rows.length < DELIVERY_ATTEMPT_PRUNE_BATCH_SIZE) break; + } + yield* Effect.annotateCurrentSpan({ "relay.delivery_attempts_prune.deleted": deleted }); + return deleted; + }).pipe( + Effect.mapError( + (cause) => + new DeliveryAttemptRecordPersistenceError({ + operation: "prune-expired", + sourceJobId: null, + userId: null, + environmentId: null, + threadId: null, + deviceId: null, + kind: null, + cause, + }), + ), + Effect.withSpan("relay.delivery_attempts.prune_expired"), + ), }); }); diff --git a/infra/relay/src/agentActivity/MobileRegistrations.test.ts b/infra/relay/src/agentActivity/MobileRegistrations.test.ts index 38840cf02b5b..83b7e30a7473 100644 --- a/infra/relay/src/agentActivity/MobileRegistrations.test.ts +++ b/infra/relay/src/agentActivity/MobileRegistrations.test.ts @@ -131,6 +131,7 @@ function makeDeliveryAttempts( record: () => Effect.void, claimSourceJob: () => Effect.succeed("claimed"), completeSourceJob: () => Effect.void, + pruneExpired: Effect.succeed(0), ...overrides, }; } diff --git a/infra/relay/src/persistence/schema.ts b/infra/relay/src/persistence/schema.ts index 132ceba16a90..0a15473776df 100644 --- a/infra/relay/src/persistence/schema.ts +++ b/infra/relay/src/persistence/schema.ts @@ -139,7 +139,6 @@ export const relayEnvironmentCredentials = pgTable( }, (table) => [ uniqueIndex("idx_relay_environment_credentials_hash").on(table.credentialHash), - index("idx_relay_environment_credentials_environment").on(table.environmentId, table.revokedAt), index("idx_relay_environment_credentials_environment_key").on( table.environmentId, table.environmentPublicKey, @@ -160,7 +159,6 @@ export const relayAgentActivityRows = pgTable( }, (table) => [ primaryKey({ columns: [table.environmentId, table.environmentPublicKey, table.threadId] }), - index("idx_relay_agent_activity_rows_updated").on(table.updatedAt), ], ); @@ -182,11 +180,7 @@ export const relayDeliveryAttempts = pgTable( transportError: text("transport_error"), }, (table) => [ - index("idx_relay_delivery_attempts_environment").on( - table.environmentId, - table.threadId, - table.createdAt, - ), + index("idx_relay_delivery_attempts_created").on(table.createdAt), uniqueIndex("idx_relay_delivery_attempts_source_job").on(table.sourceJobId), ], ); diff --git a/infra/relay/src/worker.ts b/infra/relay/src/worker.ts index e8f1dae91bde..45414b2c68f0 100644 --- a/infra/relay/src/worker.ts +++ b/infra/relay/src/worker.ts @@ -481,6 +481,11 @@ export const layer = Api.make( ), ), ), + Effect.andThen( + DeliveryAttempts.DeliveryAttempts.pipe( + Effect.flatMap((attempts) => attempts.pruneExpired), + ), + ), Effect.catchCause((cause) => Cause.hasInterrupts(cause) ? Effect.interrupt From 7145da4f5553d4e476870cdf929a0d4fd0ea8c4f Mon Sep 17 00:00:00 2001 From: Julius Marminge <51714798+juliusmarminge@users.noreply.github.com> Date: Tue, 27 Oct 2026 15:08:00 -0700 Subject: [PATCH 2/2] fix(relay): build the delivery attempt prune subquery outside the proxied db The Worker's db is Alchemy's proxyChain, so db.select() inside inArray() rendered as nothing and every prune failed with `id in returning id`. Build the subquery with QueryBuilder, as EnvironmentCredentials does, and test the prune against the same proxied db the Worker uses. Co-Authored-By: Claude Opus 5.5 --- .../agentActivity/DeliveryAttempts.test.ts | 27 ++++++++++++++++++- .../src/agentActivity/DeliveryAttempts.ts | 4 ++- 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/infra/relay/src/agentActivity/DeliveryAttempts.test.ts b/infra/relay/src/agentActivity/DeliveryAttempts.test.ts index 7e8f1b1a8c1c..a7b95e30600d 100644 --- a/infra/relay/src/agentActivity/DeliveryAttempts.test.ts +++ b/infra/relay/src/agentActivity/DeliveryAttempts.test.ts @@ -1,7 +1,9 @@ import * as NodeCryptoLayer from "@effect/platform-node/NodeCrypto"; import { describe, expect, it } from "@effect/vitest"; +import * as Drizzle from "@alchemy.run/drizzle/Postgres"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; +import * as Redacted from "effect/Redacted"; import * as RelayDb from "../db.ts"; import * as DeliveryAttempts from "./DeliveryAttempts.ts"; @@ -172,7 +174,6 @@ describe("DeliveryAttempts", () => { const pruneWith = (batches: ReadonlyArray) => { const deleteCalls: number[] = []; const fakeDb = { - select: () => ({ from: () => ({ where: () => ({ limit: () => ({}) }) }) }), delete: () => ({ where: () => ({ returning: () => { @@ -212,4 +213,28 @@ describe("DeliveryAttempts", () => { expect(deleteCalls).toHaveLength(DeliveryAttempts.DELIVERY_ATTEMPT_PRUNE_MAX_BATCHES); }); }); + + // The Worker's db is Alchemy's proxyChain, whose builders render as nothing inside another + // query. Nothing listens on the port; the failed query still reports the SQL it sent. + it.live("prunes with a subquery the Worker's proxied database can render", () => + Effect.gen(function* () { + const db = yield* Drizzle.Postgres( + Effect.succeed(Redacted.make("postgres://relay:relay@127.0.0.1:1/relay")), + { client: { prepare: false } }, + ); + const error = yield* DeliveryAttempts.DeliveryAttempts.pipe( + Effect.flatMap((attempts) => attempts.pruneExpired), + Effect.provide( + DeliveryAttempts.layer.pipe( + Layer.provide(NodeCryptoLayer.layer), + Layer.provide(Layer.succeed(RelayDb.RelayDb, db)), + ), + ), + Effect.flip, + ); + expect((error.cause as { readonly query: string }).query).toMatch( + /"relay_delivery_attempts"\."id" in \(select "id" from "relay_delivery_attempts" where "relay_delivery_attempts"\."created_at" < \$1 limit \$2\)/, + ); + }).pipe(Effect.scoped), + ); }); diff --git a/infra/relay/src/agentActivity/DeliveryAttempts.ts b/infra/relay/src/agentActivity/DeliveryAttempts.ts index b64bf974206d..f326fe9f7a32 100644 --- a/infra/relay/src/agentActivity/DeliveryAttempts.ts +++ b/infra/relay/src/agentActivity/DeliveryAttempts.ts @@ -4,6 +4,7 @@ import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; import { and, eq, inArray, isNull, lt } from "drizzle-orm"; +import { QueryBuilder } from "drizzle-orm/pg-core"; import * as Crypto from "effect/Crypto"; import * as Schema from "effect/Schema"; @@ -269,7 +270,8 @@ export const make = Effect.gen(function* () { .where( inArray( relayDeliveryAttempts.id, - db + // The Worker's db is a proxy whose builders render as nothing inside another query. + new QueryBuilder() .select({ id: relayDeliveryAttempts.id }) .from(relayDeliveryAttempts) .where(lt(relayDeliveryAttempts.createdAt, before))