diff --git a/.testcoverage.yml b/.testcoverage.yml index fe2c658f4..744849d48 100644 --- a/.testcoverage.yml +++ b/.testcoverage.yml @@ -54,6 +54,9 @@ exclude: # internal/mq/mqtest/ is the Broker conformance suite: test code that # lives outside *_test.go only so each backend's tests can import it. - ^internal/mq/mqtest/ + # internal/mq/natstest/ stands up NATS as an operator deploys it, for + # tests outside internal/mq; test code, like mqtest. + - ^internal/mq/natstest/ - ^tests/ # scripts/ holds Go helpers (cov, orchestrator) that drive the build but # aren't part of the shipped binary; they show up in `-coverpkg=./...` @@ -101,3 +104,36 @@ exclude: # outage. The unit and integration suites cover both. - ^internal/config/cache_redis\.go$ - ^internal/cache/pending\.go$ + # The external-NATS topology spec, verifier and manifest generator + # (and the `mq manifests` CLI) run against an operator's NATS, which + # the e2e stack (embedded broker) never has: unit territory, covered + # there by a fixture server. Excluding them keeps e2e at ~61%. The + # external broker is the same, covered by the integration suite. + - ^internal/mq/nats_topology\.go$ + - ^internal/mq/nats_manifests\.go$ + - ^internal/mq/nats_permprobe\.go$ + - ^internal/mq/subject_nats\.go$ + - ^internal/mq/external(_consumer)?\.go$ + - ^internal/mq/lease\.go$ + - ^cmd/wavehouse/mq\.go$ + # Their wiring, apart from wire.go for this: the NATS queue and lease + # wiring and the retention warning under it. + - ^internal/app/wire_nats\.go$ + # The ops-only listener of a process without the api role, moved + # out of wire.go the same way: the e2e binary runs every role. + - ^internal/app/wire_ops\.go$ + # The mq.nats block's checks, as cache_redis.go above: boot adopts the + # embedded fixture, so e2e never reads it. + - ^internal/config/mq_nats\.go$ + # The ingest processes' shard claims, which run only over the external + # broker's shards: the unit suite covers them with a fake queue, and + # the integration suite against NATS. + - ^internal/ingest/(claims|assign)\.go$ + - ^internal/mq/settle\.go$ + unit: + # The external NATS broker's tests start a server per case, which the + # unit suite's 15s per package cannot hold: they are integration-tagged + # (make test-integration), and the merged total counts them. + - ^internal/mq/external(_consumer)?\.go$ + # The NATS KV leases, the same way. + - ^internal/mq/lease\.go$ diff --git a/AGENTS.md b/AGENTS.md index 4573713a9..03bafa627 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -29,18 +29,18 @@ One binary: Twenty internal packages under `internal/` (plus `internal/testutil/` for shared test helpers): - **`api/`** — Chi HTTP router, JWT/JWKS middleware (from `auth/`), ingest/query/structured-query/SSE/schema/DLQ/pipes handlers; `ch_errors.go` (`writeCHError`) is the one mapping from a failed ClickHouse query to status, `code` and `retryable` (`writeCHWriteError` for a write pipe: never retryable, no `Retry-After`, since the write may have run) -- **`app/`** — the process wiring: `New` builds every component from the boot config and the settings directory (each one wired in one place — what it opens, what it loops, what it releases — with the settings registry handed to its wiring function whole, the injection point of the per-tenant registry of #583: store-keyed getters for the handlers, `perTenant` for the async paths (with the tenant each message's `mq.Topic` names for the stream hub and the ingest worker), the `chconn.Pools` and the per-tenant `discoveries` reconciled from `AfterAdopt`, `shortestKeepalive` for the one setting folded over every tenant served, `gapWindows` handing the sweeper each tenant's own gap window (a rejected tenant's as its folder last had it, unbounded for one rejected since boot) and the `mq.max_bytes_gb` reconcile each served tenant's byte budget, and `defaultPolicy` for the one setting that still follows tenant `0`, a flat directory's ops-gate admin role; the auth verifiers are per tenant, reconfigured (rebuilt only on changed wiring) and pruned from `AfterAdopt`, the same hook's `Hub.Prune` ends the open streams of a tenant no longer served, and `wireCache`'s hook drops, through `LocalCache.Prune`, the cache version index of a tenant no longer served ([#262](https://github.com/Wave-RF/WaveHouse/issues/262))), `Run` drives the long-lived ones under one `errgroup` until the context is cancelled or one fails, `Close` releases them in reverse order. `New` wires only what the process's `roles` need (discovery, dedupe, auth verifiers, the hub bridge and keepalive per API process; the ingest worker per ingest process; the sweeper under its lease through `elected`); a process without `api` serves `api.NewOpsRouter` — probes, `/version`, metrics, and the settings reload behind the operator key alone. `cmd/wavehouse` and `tests/integration` both boot through it +- **`app/`** — the process wiring: `New` builds every component from the boot config and the settings directory (each one wired in one place — what it opens, what it loops, what it releases — with the settings registry handed to its wiring function whole, the injection point of the per-tenant registry of #583: store-keyed getters for the handlers, `perTenant` for the async paths (with the tenant each message's `mq.Topic` names for the stream hub and the ingest worker), the `chconn.Pools` and the per-tenant `discoveries` reconciled from `AfterAdopt`, `shortestKeepalive` for the one setting folded over every tenant served, `gapWindows` handing the sweeper each tenant's own gap window (a rejected tenant's as its folder last had it, unbounded for one rejected since boot) and the `mq.max_bytes_gb` reconcile each served tenant's byte budget, and `defaultPolicy` for the one setting that still follows tenant `0`, a flat directory's ops-gate admin role; the auth verifiers are per tenant, reconfigured (rebuilt only on changed wiring) and pruned from `AfterAdopt`, the same hook's `Hub.Prune` ends the open streams of a tenant no longer served, and `wireCache`'s hook drops, through `LocalCache.Prune`, the cache version index of a tenant no longer served ([#262](https://github.com/Wave-RF/WaveHouse/issues/262))), `Run` drives the long-lived ones under one `errgroup` until the context is cancelled or one fails, `Close` releases them in reverse order. `New` wires only what the process's `roles` need (discovery, dedupe, auth verifiers, the hub bridge and keepalive per API process; the ingest worker per ingest process; the sweeper under its lease through `elected`, on the embedded MQ only); a process without `api` serves `api.NewOpsRouter` — probes, `/version`, metrics, and the settings reload behind the operator key alone. `cmd/wavehouse` and `tests/integration` both boot through it - **`auth/`** — JWT auth middleware: HMAC **or** JWKS verification with `alg` pinned to the active verifier, role extraction from a configurable claim path; always runs, never rejects (bad token → empty role + stashed reason). One verifier per tenant ([#583](https://github.com/Wave-RF/WaveHouse/issues/583) story 9): `Authenticator` keys them by `tenant.ID` — the request store's `settings.Store.Tenant()`, through an injected `TenantSource`; `tenant.Default` on the tenant-exempt routes — built from each tenant's `auth` block by `Reconfigure`, dropped by `Prune` once the tenant stops being served (rejected or removed), released by `Close`; the secrets (`Config`) are boot-level and shared. A JWKS key set is fetched off the boot and reload paths: until one has been stored the verifier is pending and a token-bearing request gets `503` + `Retry-After` from `api.refuseUnverifiable` (`auth.ErrVerifierPending`), never a `default_role` evaluation; refresh is library-managed (Eric, 2026-09-22), response capped at 1 MiB; the operator key's admin role is the request tenant's - **`cache/`** — `Cache` interface → `LocalCache` (Ristretto: one pool for every tenant) + `VersionManager` (the invalidation index), and `RedisCache`, the Redis-compatible shared backend (random version tokens under the tenant's hash tag, one-round-trip lookups, bypass on failure behind a circuit breaker, deferred invalidations retried; selected by `cache.backend: redis`, configured by the boot config's `cache.redis` block — [#613](https://github.com/Wave-RF/WaveHouse/issues/613)). Every key carries the tenant (in `RedisCache`, after the key prefix: `:{}:…` for a version token, `:q::…` for a value); in `LocalCache` and the version index it leads ([#583](https://github.com/Wave-RF/WaveHouse/issues/583) story 8) — `:query:` for the caller's query key and its singleflight, escaped whole as the lead field of the stored key `|.||…`, where each raw table and scope name is escaped by `keyenc` (a `Namespace` carries them raw, so no caller escapes); the index holds a version per tenant, per (tenant, table) and per (tenant, table, scope), keyed by raw name and bumped in place (one entry per live namespace however often it is bumped, [#262](https://github.com/Wave-RF/WaveHouse/issues/262)) — so no cached read or coalesced flight crosses tenants, a bump through `Invalidate` names one tenant's namespaces and no other's, and `InvalidateTenant` drops the tenant's index so its next key gets a process-unique generation, orphaning its every cached result in one step, pipe results included (no insert reaches a pipe result until [#343](https://github.com/Wave-RF/WaveHouse/pull/343)); `Lookup` returns a `Snapshot` of the versions it read, taken before the handler chooses any input a bump invalidates — the tenant's connection included — and `Set` files the fill under it, so a write landing mid-query, or a reload moving the tenant to another address or database after the request took its connection, orphans the fill ([#382](https://github.com/Wave-RF/WaveHouse/issues/382)), and every backend runs the conformance suite `internal/testutil/cachetest`; the one crossing is the wiring's, above the package: `internal/app` hands the ingest worker the cache through `sharedTables`, which repeats each of the worker's bumps under every tenant on the same ClickHouse address and database (`chconn.Pools.SharingTables`, whatever their user or tls block — they read the same tables), and orphans the whole cache of a tenant back on a pool after an absence, since it was out of that fan-out while away, or moved to another address or database, since it now reads other tables (story 6) - **`chconn/`** — `Pools`, one `Manager` (a `driver.Conn`) per distinct `Identity{Addr, Database, Username, Password, TLS}` tuple among the served tenants, reconciled from the settings registry's `AfterAdopt` after every reload ([#583](https://github.com/Wave-RF/WaveHouse/issues/583) story 6): tenants naming one tuple share its pool, sized to their largest `max_open_conns`/`max_idle_conns`; a tenant whose tuple changed is repointed; a tuple no tenant names is released after the longest `query_timeout` among the tenants it had (never dials; a resize swaps the connection with the same grace). The boot config's `clickhouse.max_total_conns` bounds the open pools' `max_open_conns` together: boot refuses naming sum and ceiling; at a reload a resize above it keeps the pool's size, and a tuple that cannot be opened (the ceiling, an unreadable certificate, or options the driver refuses) leaves its tenants on the pool they had or on none — logged, retried by the next reload. Every consumer resolves its tenant's pool per call: `For` (nil for a tenant on no pool, a `503`), `Target` (the tenant's own HTTP wiring over its pool's TLS config), `SharingTables`, `Ping` (every pool at once, ready at the first answer). `HTTPClients` keeps one `http.Client` per TLS config. `Classify` (`errclass.go`) says what a failed ClickHouse request means for the request — `Unavailable`, `Denied`, `Rejected` (any unlisted exception code: the server read it and refused it), or `Unknown` (no code, no recognizable transport failure) — over the driver's error types and the HTTP interface's `HTTPError`; the ingest worker and the query handlers (`api/ch_errors.go` `writeCHError`, [#403](https://github.com/Wave-RF/WaveHouse/issues/403), [#271](https://github.com/Wave-RF/WaveHouse/issues/271)) both use it - **`chsql/`** — dependency-free ClickHouse SQL helpers shared by `query`/`policy` (avoids an import cycle): `QuoteIdent` (backtick-quote every identifier) + `BindUnsafe` (reject names with a literal `?`) -- **`config/`** — YAML + env var config loading (cleanenv); strict on both sides (undeclared YAML key, unbound `WH_*` variable) and probes `data_dir` writability when a selected backend keeps state there (`NeedsDataDir`); `backends.go` holds each layer's `.backend` (the in-process value by default; `cache.backend` also takes `redis`, whose sub-block is `cache_redis.go`, and `dedupe.backend` takes `dynamodb`, with its `dedupe.dynamodb` sub-block); `config.go` holds `roles` (`Has(Role)`) and `instance_id`, and `Validate` refuses a role split the backends cannot serve (any split over the embedded MQ; `api` without `ingest`, or the reverse, over a local cache) — boot is the validator, there is no dry run -- **`coord/`** — leases for work that must run in one process at a time: `Coordinator.TryAcquire(ctx, name)` → a `Term` (fencing `Token`, strictly increasing per name; `Done`/`Err`, `ErrLost` on loss; `Resign`), `ErrHeld` while another holder's — or this coordinator's own — term is live; `RunElected` runs a loop only while holding its lease, resigning when the loop returns and campaigning again every `RetryPeriod`. `Local` is the in-process implementation (first taker wins, never expires; `Peer` is a second handle over the same table for tests); every implementation runs `coordtest.Conformance`. Imports only the standard library, so a distributed backend lives beside its connection (NATS KV in `internal/mq`). `internal/app`'s `wireCoord` opens the one `coord.backend` selects and the sweeper runs through `RunElected` under the `sweeper` lease +- **`config/`** — YAML + env var config loading (cleanenv); strict on both sides (undeclared YAML key, unbound `WH_*` variable) and probes `data_dir` writability when a selected backend keeps state there (`NeedsDataDir`); `backends.go` holds each layer's `.backend` (the in-process value by default; `mq.backend` also takes `nats`, with its `mq.nats` sub-block of file-path-only credentials; `coord.backend` takes `nats`, whose `coord.nats` block names only the lease bucket and rides `mq.nats`'s connection (both blocks, their rules and warnings are `mq_nats.go`); `cache.backend` takes `redis`, whose sub-block is `cache_redis.go`; and `dedupe.backend` takes `dynamodb`, with its `dedupe.dynamodb` sub-block) and `Warnings`, the valid combinations boot logs at `WARN`; `config.go` holds `roles` (`Has(Role)`) and `instance_id`, and `Validate` refuses a role split the backends cannot serve (any split over the embedded MQ; `api` without `ingest`, or the reverse, over a local cache; `coord.backend=nats` without `mq.backend=nats`; `mq.backend=nats` with `coord.backend=local` in a process running `ingest`; a process running only `sweeper` under `mq.backend=nats`) — boot is the validator, there is no dry run +- **`coord/`** — leases for work that must run in one process at a time (`Observer.Held` reads whether one is held without campaigning): `Coordinator.TryAcquire(ctx, name)` → a `Term` (fencing `Token`, strictly increasing per name; `Done`/`Err`, `ErrLost` on loss; `Resign`), `ErrHeld` while another holder's — or this coordinator's own — term is live; `RunElected` runs a loop only while holding its lease, resigning when the loop returns and campaigning again every `RetryPeriod`. `Local` is the in-process implementation (first taker wins, never expires; `Peer` is a second handle over the same table for tests); every implementation runs `coordtest.Conformance`. Imports only the standard library, so a distributed backend lives beside its connection: `coord.backend: nats` is `internal/mq/lease.go` (`ExternalNATS.Leases`), a key per lease in the operator's KV bucket, the KV revision as the fencing token, and expiry judged on the candidate's own clock (the same revision seen unchanged for 15s), never by a server TTL. `internal/app`'s `wireCoord` opens the one `coord.backend` selects and the sweeper runs through `RunElected` under the `sweeper` lease - **`dedupe/`** — `Deduplicator` interface (two-phase `Reserve`/`Commit`/`Release` over `Key{Table, ID}`; every backend passes the `dedupetest` conformance suite) → `Embedded` (Pebble: every tenant's seen ids in one instance at `data_dir/pebble`, each key led by its tenant and table, pending claims in memory, committed ids stored with their expiry and deleted by an hourly background sweep along with the version-0 keys from before the table joined the key, open while any tenant's store is — the layout is the implementation's call, and the wiring hands it `data_dir` once; its `Stats` feed the system gauges) or `Dynamo` (one shared DynamoDB table, conditional `PutItem` claims; conformance-tested against dynamodb-local, selected by `dedupe.backend: dynamodb`; boot checks the table and never creates it outside dynamodb-local), wrapped by `Managed` whose open/closed state follows the hot-reloadable `dedupe.enabled` in the settings directory's `config.json`; `Stores` holds one `Managed` per tenant, built through a `Factory` (`func(tenant.ID) *Managed`, `Embedded.Tenant` or, gated on the table check (`Factory.Gated`), `Dynamo.Tenant` in production; `Managed` opens its store through a function, so every backend gets the same switch), and reconciled from the registry's `AfterAdopt` hook — open exactly when the tenant is served with its switch on, closed with its seen ids kept otherwise ([#583](https://github.com/Wave-RF/WaveHouse/issues/583) stories 7 and 3) - **`discovery/`** — `SchemaRegistry`, one per served tenant over a `Source` read once per refresh — the tenant's pool's connection and the database that pool was opened for, one snapshot, so a refused move keeps discovering the database the tenant's queries still use (`internal/app`'s `discoveries` builds, runs and stops them from `AfterAdopt` and `App.Close`: `RetryRefresh` until the first success, then `StartAutoRefresh` with a random first tick; `Lookup` answers `ErrNotLoaded` before the first success — the handlers' `503` with `Retry-After` — and `ErrUnknownTable` after; a failed loop attempt counts in `wavehouse_schema_refresh_failures_total{tenant}`), that introspects ClickHouse `system.columns` (name/type/nullability plus `default_expression` and 1-based `position`) and `system.tables` (each table's `create_table_query`, kept in-process and never serialized — an external-engine table renders its wiring there unconditionally — endpoint, bucket/host, database, username, S3 access key id; ClickHouse masks the password as `[HIDDEN]` from ~23.9, so the exposure is the topology, not the secret), records the server version, + `Validate()` for ingest payloads + `CanonicalizeTimestamps()` rewriting top-level `DateTime`/`DateTime64` column values to the canonical RFC 3339 UTC wire form pre-publish (Key Design Decision #19) -- **`ingest/`** — Ingest worker pipeline (`worker.go`: JetStream input → per-table batch INSERT with DLQ output). The pipeline is **insert-only**. The wire format `EventMessage` (`types.go`) carries `{table_name, scope, received_timestamp, format, columns, row}` and nothing else — `row` is one positional `JSONCompactEachRow` line and `columns` names its slots, the table's **insertable** columns (a `MATERIALIZED`/`ALIAS` column cannot be named in an `INSERT`); the worker batches per (tenant, table, column list), the tenant read off each message's `mq.Topic`, and inserts each batch into its tenant's own ClickHouse (`chconn.Pools.Target`); the worker accepts whatever table name the envelope carries (table existence was already checked by the HTTP ingest handler, which `404`s an unknown table before publish; the worker doesn't re-validate), then bulk-INSERTs. In the embedded-NATS deployment (the default), the server runs with `DontListen: true` (`internal/mq/embedded.go`), so the only Publishers reachable on the `ingest.>` subjects are in-process Go code — today, only the HTTP `/v1/ingest?table={table}` handler. Non-insert mutations (`DELETE`/`UPDATE`/`TRUNCATE`/…) must go through `POST /v1/ops/query` under the admin role (the same `RequireAdmin` gate as the rest of `/v1/ops/*`, so non-admin callers never reach the proxy) or through an operator-authored pipe that writes, gated only by its `allowed_roles` (#386). A request with no token (or an invalid one) resolves to the `default_role`, which in a production config is not the admin role (setting them equal is a loudly-warned dev-only setting), so it can't reach this endpoint. Plus `Sweeper` (Active Sweeper for NATS message lifecycle) + `EventMessage`/`BufferConsumerName` types (`types.go`) +- **`ingest/`** — Ingest worker pipeline (`worker.go`: JetStream input → per-table batch INSERT with DLQ output; over a `Sharded` queue, `claims.go`'s `ClaimShards` narrows the worker to the units this process is assigned — membership leases, capped rendezvous, halt-drain-then-release handover and stop, reset at takeover from a dead owner, each unit's share of a 10,000-row budget of unsettled rows). The pipeline is **insert-only**. The wire format `EventMessage` (`types.go`) carries `{table_name, scope, received_timestamp, format, columns, row}` and nothing else — `row` is one positional `JSONCompactEachRow` line and `columns` names its slots, the table's **insertable** columns (a `MATERIALIZED`/`ALIAS` column cannot be named in an `INSERT`); the worker batches per (tenant, table, column list), the tenant read off each message's `mq.Topic`, and inserts each batch into its tenant's own ClickHouse (`chconn.Pools.Target`); the worker accepts whatever table name the envelope carries (table existence was already checked by the HTTP ingest handler, which `404`s an unknown table before publish; the worker doesn't re-validate), then bulk-INSERTs. In the embedded-NATS deployment (the default), the server runs with `DontListen: true` (`internal/mq/embedded.go`), so the only Publishers reachable on the `ingest.>` subjects are in-process Go code — today, only the HTTP `/v1/ingest?table={table}` handler. Non-insert mutations (`DELETE`/`UPDATE`/`TRUNCATE`/…) must go through `POST /v1/ops/query` under the admin role (the same `RequireAdmin` gate as the rest of `/v1/ops/*`, so non-admin callers never reach the proxy) or through an operator-authored pipe that writes, gated only by its `allowed_roles` (#386). A request with no token (or an invalid one) resolves to the `default_role`, which in a production config is not the admin role (setting them equal is a loudly-warned dev-only setting), so it can't reach this endpoint. Plus `Sweeper` (Active Sweeper for NATS message lifecycle) + `EventMessage`/`BufferConsumerName` types (`types.go`) - **`keyenc/`** — the one escaping composite keys are built from: `Escape` keeps `[A-Za-z0-9_-]` (exactly the tenant-id grammar, so a tenant id is its own escaped form) and writes every other byte as `%XX`, `Unescape` is `url.PathUnescape` (lenient: either hex case, and a byte left unescaped reads as itself, so a `%2D` an earlier build wrote still reads), `Join`/`AppendJoin` escape each field and put a separator between them (they panic on no fields, and on a separator the escaping could write or one outside ASCII) and `Split` reverses them. The package that builds a key takes raw names and escapes them itself, so no caller has to and no field reaches a key unescaped: NATS subjects (`Join`/`Split` after the verbatim tenant) and the cache's keys — the version index and the shared backend's Redis keys (`internal/cache`) — and the dedupe keys (`//`) use it; changing what it keeps orphans every stored key (an orphaned dedupe key lets a seen id through again), and on the shared backend, whose keys every process builds for itself, splits them between builds for the length of a rolling upgrade (a bump one build makes misses the entries the other filed, served until their TTL) -- **`mq/`** — the message-queue boundary: the **only** package that imports NATS/JetStream (Key Design Decision #20). and the only one that knows how the broker works. Everything else addresses events by `Topic{Tenant, Table, Scope}` (a validated tenant id and raw names — the tenant leads every subject, `ingest..
`, so one wildcard selects a tenant's traffic, and a topic without one is refused) and states intent through the interfaces — `Publisher` (`ErrQueueFull` is the backpressure signal, `ErrUnavailable` a broker that cannot be reached — both a `503`, with `Retry-After` `30` and `5`; `WithIdempotencyKey` makes a republish inside the queue's duplicate window a no-op), `Subscriber`, `ConsumerManager`/`Consumer`/`ConsumerConfig` (the ingest worker's durable consumer), `DeadLetterer` and `DeadLetterStats` (park a message, count what is parked), `Purger` (drop what is both acked and older than a cutoff — the sweeper), `Replayer` (SSE gap-fill) — composed into `Broker`, which adds each tenant's byte budget (`SetMaxBytes`/`MaxBytes`: the `mq.max_bytes_gb` reload, which opens a tenant's queue the first time) and `Stats` (the system gauges' source). Every interface speaks per tenant, never per stream: the embedded implementation gives each tenant a queue of its own (a stream pair, `INGEST_`/`DLQ_`), and nothing outside the package may assume that layout — an external implementation may keep one shared stream. Subjects, prefixes, wildcards, stream names, sequences, and ack floors are private to the one implementation, `EmbeddedNATS` (`embedded.go`, `subject.go`, `purge.go`, `deadletter.go`), whose subject tokens are escaped by the shared `internal/keyenc`; `internal/app` constructs it and hands everything else a `mq.Broker`. Every implementation passes the conformance suite in `internal/mq/mqtest` (`mqtest.Run`), which states the `Broker` contract as behavior; a new backend runs it from its own test, with `mqtest.Caps` only where its semantics legitimately differ +- **`mq/`** — the message-queue boundary: the **only** package that imports NATS/JetStream (Key Design Decision #20), and the only one that knows how the broker works. Everything else addresses events by `Topic{Tenant, Table, Scope}` (a validated tenant id and raw names — the tenant leads every subject, `ingest..
`, so one wildcard selects a tenant's traffic, and a topic without one is refused) and states intent through the interfaces — `Publisher` (`ErrQueueFull` is the backpressure signal, `ErrUnavailable` a broker that cannot be reached — both a `503`, with `Retry-After` `30` and `5`; `WithIdempotencyKey` makes a republish inside the queue's duplicate window a no-op), `Subscriber`, `ConsumerManager`/`Consumer`/`ConsumerConfig` (the ingest worker's durable consumer), `DeadLetterer` and `DeadLetterStats` (park a message, count what is parked), `Purger` (drop what is both acked and older than a cutoff — the sweeper), `Replayer` (SSE gap-fill) — composed into `Broker`, which adds each tenant's byte budget (`SetMaxBytes`/`MaxBytes`: the `mq.max_bytes_gb` reload, which opens a tenant's queue the first time) and `Stats` (the system gauges' source). A broker whose ingest queue is split into units one consumer at a time owns implements `Sharded` too (`IngestUnits`, `ResetOrphaned`, `Unowned`; `ConsumerConfig.Units` narrows a consumer to some of them, and its consumer is a `Releaser` and a `Halter`, capped per unit by `ConsumerConfig.MaxHeld`, with `ErrConsumerMismatch` when an operator durable no longer fits), and `Message.OnSettled` runs a hook once, at the first ack or nak attempt, confirmed or not. Every interface speaks per tenant, never per stream: the embedded implementation gives each tenant a queue of its own (a stream pair, `INGEST_`/`DLQ_`), and nothing outside the package may assume that layout — an external implementation may keep one shared stream. Subjects, prefixes, wildcards, stream names, sequences, and ack floors are private to the implementations, whose subject tokens are escaped by the shared `internal/keyenc`: `EmbeddedNATS` (`embedded.go`, `subject.go`, `purge.go`, `deadletter.go`), which `internal/app` constructs and hands everything else as a `mq.Broker`, and `ExternalNATS` (`external.go`, `subject_nats.go`, `nats_topology.go`: an operator-owned cluster whose streams, durables and lease bucket it never creates, changes, purges or deletes; `lease.go` holds `coord.backend: nats`'s leases in that bucket), which `internal/app` constructs from the `mq.nats` block when `mq.backend` is `nats` ([#613](https://github.com/Wave-RF/WaveHouse/issues/613)). Every implementation passes the conformance suite in `internal/mq/mqtest` (`mqtest.Run`), which states the `Broker` contract as behavior; a new backend runs it from its own test, with `mqtest.Caps` only where its semantics legitimately differ - **`observability/`** — OpenTelemetry pipeline: `InitProvider` wires trace/metric/log providers via OTLP gRPC (each signal independently gated). A top-level `Prometheus` config block drives an optional `/metrics` scrape endpoint that runs independently of OTLP push — standalone (Alloy/Mimir scrape, no collector), alongside OTLP, or off. `NewLogger` produces a slog handler that fans out to stdout AND OTLP (stdout always 100%, OTLP sample-rate-aware). `TraceHandler` injects trace_id/span_id from active spans. `tracer.go` provides W3C trace context propagation over message headers (`InjectHeaders`/`ExtractHeaders` on a plain header map; `internal/mq` injects on every publish and extracts on the `Subscribe` path, so this package never sees a NATS type). - **`pipes/`** — Named query pipes: `NamedQuery` type + `BindParams` + `Source` (read per request; `settings.Store` in production, `Static(q...)` in tests) - **`policy/`** — Hasura-style access control, **role-first**: `TablePolicy` is `map[string]RolePermissions`, and a role's grant splits by operation into `SelectPermissions` (columns, row `filter`, aggregations, the `max_*` limits) and `InsertPermissions` (columns, `check`) — so a field only one side honors does not exist on the other. `Evaluate()` resolves ONE operation and leaves the other side **nil** (`Select *ResolvedSelect` / `Insert *ResolvedInsert`), which every accessor fails closed on — nil is "not resolved", distinct from an empty side, which is "unrestricted" (what the admin return builds). Claim templating (`{{ jwt.claim.path }}`) resolves during that call. Policies come from `Source`, a `func() *Policy` read per call (`settings.Store.Policy` in production, `Static(p)` in tests) @@ -72,7 +72,7 @@ The invariant index — what must stay true. Full narrative and rationale live i 17. **Non-fatal boot** — schema-discovery failure on boot is non-fatal: `internal/app` records an `api.BootState`, binds `:8080`, serves 503 on `/livez`/`/readyz` with the diagnostic, and retries via `SchemaRegistry.RetryRefresh` (jittered backoff 2s → 60s), per tenant over a nested directory: `/livez` is 503 while no tenant has completed a first discovery, then sticky 200, and a tenant's outage after that is its log line and counter, never a probe failure. Until a tenant's first discovery its table lookups are a 503 with `Retry-After`, not a 404. Bounds supervisor restart loops. 18. **Health endpoints** — liveness `/livez`, readiness `/readyz` (k8s convention; `/readyz` pings every open ClickHouse pool at once and is ready at the first answer, 503 naming each when none answers); `/healthz` is a permanent alias of `/livez`; `/health` + `/ready` are deprecated (removal v0.2.0, CHANGELOG #144). `/v1/health` is the SDK's content-free public ping (no ClickHouse check), a `/v1` route so it survives reverse-proxy probe-path filtering. Point k8s at `/livez`/`/readyz`, SDK/online-checks at `/v1/health`, never the deprecated aliases. 19. **Canonical timestamp wire form (fail-open at ingest)** — the HTTP ingest handler rewrites every top-level `DateTime`/`DateTime64` column value it can parse to RFC 3339 UTC (`discovery.CanonicalizeTimestamps`; per-column precision + zone precomputed at schema refresh) after validation + policy checks and **before** the NATS publish, so the one payload every consumer shares — SSE subscribers, the ClickHouse insert, the DLQ — carries the same spelling `/v1/query` renders: live and query reads can't drift on the instant (#372). Zone-less inputs are read in the column's declared zone, else the discovered server default — ClickHouse's own rule, so the spelling changes but never the instant. Deliberately **fail-open**: an unparseable value or unresolvable zone (no tzdata embedded — never a failed refresh, never a silent UTC reinterpretation, which would move instants) publishes verbatim; ingest must not reject a record over its timestamp spelling — fail-closed enforcement belongs to the stream row-filter (#381). Don't re-spell timestamps downstream. Preserve when touching `internal/discovery`, the ingest handler, or the SSE fan-out. Detail: architecture.md § `discovery/` + §Ingest Path; the exact spelling spec (truncation, zero-trimming, `Z`-only) lives in api.md §Timestamp canonicalization — keep it in sync with `canonicalTimestamp`. -20. **Sealed MQ boundary** — only `internal/mq` imports NATS/JetStream (`github.com/nats-io/…`), enforced by the `depguard` rule in `.golangci.yml`, so `make lint` fails on a leak in every package it builds (the `integration`-tagged files under `tests/` are outside lint's build context — keep them clean by convention, through `mq.Broker`). The boundary is semantic as well: everything else addresses events by `mq.Topic` and states intent through mq-owned interfaces (`Publisher`, `Consumer`, `DeadLetterer`, `Purger`, `Replayer`, …), and never builds a subject, names a stream, or reasons in sequences — so a subject, stream, or broker change lands in one package ([#583](https://github.com/Wave-RF/WaveHouse/issues/583) story 4; story 5's tenant token landed there alone — `Topic.Tenant`, first in every subject). Don't add a raw accessor (`JetStream()`, `NatsConn()`, `GetServer()`) back, and don't hand-build `"ingest."`/`"dlq."` subjects outside `internal/mq` — widen the mq surface with an intent-level method instead. +20. **Sealed MQ boundary** — only `internal/mq` imports NATS/JetStream (`github.com/nats-io/…`), enforced by the `depguard` rule in `.golangci.yml`, so `make lint` fails on a leak in every package it builds (the `integration`-tagged files under `tests/` are outside lint's build context — keep them clean by convention, through `mq.Broker`). A test outside `internal/mq` that needs a real NATS server goes through `internal/mq/natstest`, which stands one up from the shipped `deployments/nats` files and hands back a URL and passwords, never a NATS type. The boundary is semantic as well: everything else addresses events by `mq.Topic` and states intent through mq-owned interfaces (`Publisher`, `Consumer`, `DeadLetterer`, `Purger`, `Replayer`, …), and never builds a subject, names a stream, or reasons in sequences — so a subject, stream, or broker change lands in one package ([#583](https://github.com/Wave-RF/WaveHouse/issues/583) story 4; story 5's tenant token landed there alone — `Topic.Tenant`, first in every subject). Don't add a raw accessor (`JetStream()`, `NatsConn()`, `GetServer()`) back, and don't hand-build `"ingest."`/`"dlq."` subjects outside `internal/mq` — widen the mq surface with an intent-level method instead. ## Code Conventions @@ -438,9 +438,9 @@ internal/config/ → Configuration structs + loader internal/coord/ → Leases with fencing tokens (interface, in-process Local, RunElected, coordtest conformance suite) internal/dedupe/ → Optional deduplication (Reserve/Commit/Release interface; Pebble, DynamoDB) internal/discovery/ → ClickHouse schema introspection + ingest validation -internal/ingest/ → Batch buffer with DLQ + Active Sweeper (NATS message lifecycle) +internal/ingest/ → Batch buffer with DLQ + Active Sweeper (NATS message lifecycle) + shard claims over a shared queue internal/keyenc/ → One escaping for composite keys (NATS subject tokens, cache keys, dedupe keys) -internal/mq/ → MQ boundary (the only NATS/JetStream importer: owned message/consumer/stream types + embedded server; mqtest/ is the Broker conformance suite) +internal/mq/ → MQ boundary (the only NATS/JetStream importer: owned message/consumer/stream types, the embedded server and the external-NATS broker; mqtest/ is the Broker conformance suite; natstest/ stands up NATS as an operator deploys it, for tests outside the package) internal/observability/ → OpenTelemetry pipeline (traces/metrics/logs providers, Prometheus exporter, slog fan-out, message-header trace propagation) internal/pipes/ → Named query pipes (types, parameter binding, Source) internal/policy/ → Access control policies (types, evaluation, Source) @@ -450,12 +450,13 @@ internal/stream/ → SSE fan-out (event Hub: project once per role, Subsc internal/tenant/ → Tenant id (type, grammar, reserved default, request header name) internal/testutil/ → Shared test helpers (mocks, JWT + schema helpers; logtest/ captures or silences the default logger; cachetest/ is the conformance suite every cache.Cache backend runs; mutationtest/ holds the shared write-classifier cases; storedir/ is the embedded broker's store directory in tests, removed once late consumer-state writes land) tests/ → Integration & E2E tests -tests/integration/ → Go integration tests (//go:build integration; ClickHouse testcontainer, and Redis for shared_cache_test.go). A package tested against its own external server keeps them beside it: internal/cache/redis_integration_test.go (Redis, Valkey, Dragonfly, Redis Cluster testcontainers) +tests/integration/ → Go integration tests (//go:build integration; ClickHouse testcontainer, Redis for shared_cache_test.go, and NATS for the `mq.backend: nats` end-to-end test). A package tested against its own external server keeps them beside it: internal/cache/redis_integration_test.go (Redis, Valkey, Dragonfly, Redis Cluster testcontainers). `make test-integration` also runs `internal/mq/natsspike` (nats-server semantics, under `internal/mq` for the NATS import boundary) and `internal/mq`'s integration-tagged external-NATS broker tests (`TestExternalNATS*`, `TestNewNATS*`, `TestNATSPermissions_Refuse*`, `TestLeases*`) tests/e2e/ → E2E test stack (scripts/orchestrator boots ClickHouse and Redis testcontainers + the wavehouse-cov binary) tests/e2e/fixtures/ → Idempotent ClickHouse DDL scripts for test tables tests/e2e/sdk/ → E2E integration tests via TypeScript SDK (Vitest) deployments/compose/ → Docker Compose files (standalone.yaml, dependencies.yaml) deployments/Dockerfile → Runtime image (+ Dockerfile.goreleaser for release builds) +deployments/nats/ → External NATS JetStream: nack CRs (`wavehouse mq manifests` output, golden-tested) + Helm values with WaveHouse's user permissions (test-pinned) docs/ → Project documentation .vscode/ → Workspace settings (gopls build flags, recommended extensions) ``` diff --git a/CHANGELOG.md b/CHANGELOG.md index b0636db2a..305b77cce 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,13 +10,18 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), ### Added +- **`mq.backend: nats` runs WaveHouse on an operator-owned NATS JetStream, so several processes share one queue** (`internal/mq/{external,settle,subject_nats}.go` (new; + integration-tagged tests, among them `external_conformance_test.go`), `internal/mq/{mq,embedded}.go` (+ tests), `internal/config/{backends,config,mq_nats}.go` (`mq_nats.go` new; + tests), `internal/app/{app,wire,wire_nats,wire_ops}.go` (`wire_nats.go` and `wire_ops.go` new; + `mq_nats_test.go`, `retention_warn_test.go`), `internal/settings/store.go` (+ test), `tests/integration/{setup,mq_nats}_test.go`, `Makefile`, `.testcoverage.yml`, `go.mod`, `config.yaml`, `CONTRIBUTING.md`, `AGENTS.md`, `docs/src/content/docs/{deployment,api,architecture,ingest-pipeline,durability,development}.md`, `docs/src/content/docs/{configuration,settings-directory}.mdx`): part of [#613](https://github.com/Wave-RF/WaveHouse/issues/613). `mq.backend` now takes `nats`, configured by a new `mq.nats` block (`WH_MQ_NATS_*`): the server URLs; one of a creds file, an nkey seed file, or a user with a password file (secrets are file paths only); TLS and mutual TLS; a JetStream domain; the subject prefix; the partition and shard counts (`partitions`, and `shards`, default 32, 1 to 256), which must match the operator's topology; the shard durables' base name and the history stream's name; and the connect, publish and topology-wait timeouts. WaveHouse never creates, changes, purges or deletes a stream, a durable or a bucket there: boot connects, waits up to `topology_wait` (60s) for the operator's topology (see the next entry), refuses a NATS server older than 2.14, and refuses to start with every finding at once while the topology is still wrong; it is checked again every five minutes. A publish retried after a lost answer reuses its `Nats-Msg-Id`, so the partition stores it once, and the ack's stream is checked, so a row another stream stored is a topology fault. A full partition or a table at its per-subject cap is `ErrQueueFull`; a broker that does not answer, a lost connection or a deleted partition is `mq.ErrUnavailable`, a `503` with `Retry-After: 5`. A deleted configured shard durable, or a connection closed for good, ends the worker and the process, so the next boot names what is missing. SSE replay and each API process's live events read the history stream through short-lived consumers of their own. The history is best effort: a row republished while it is unavailable reaches ClickHouse but not SSE, and a full or missing history never holds up ingest. Dead-letter counts are one subject-filtered read of the shared dead-letter stream, keyed by table (the fold the embedded broker now uses too), and zero for a tenant that never parked a row. A tenant's `mq.max_bytes_gb` is not applied, and boot says so. **An ingest `200` rests on the stream's replicas:** it follows the partition's ack, which at 3 replicas comes once a Raft quorum has stored the event, so WaveHouse does not require `sync_always`; the topology check recommends 3 replicas on every stream (so a one-server development cluster still boots) and refuses `persist_mode: async` on a partition. The embedded broker still `fsync`s every event. **There is no sweeper under `nats`:** the streams' retention does its work, so it is not wired and takes no lease, and a process whose only role is `sweeper` is refused. The API processes instead warn once per tenant and window about a gap window longer than the history's `max_age`, at boot, after every reload, and when the periodic check reads a changed `max_age`. Every partition's `duplicate_window` must cover every attempt of one publish, and `dedupe.lease` twice over plus a second, or boot refuses, and a tenant with dedupe on whose finite retention is under that window is logged at `WARN`. Lowering N loses no rows: the ingest worker also drains the shard durables left on any stream holding ingest subjects outside the N partitions (the generated `preventDelete` keeps them), and boot counts the rows each holds; a stream with no shard durable is only reported. Process `roles` can be split on it: an `ingest` process needs `coord.backend: nats`, and `api` and `ingest` in separate processes need `cache.backend: redis` too. Three gauges report on the broker: `wavehouse_mq_connected`, `wavehouse_mq_topology_ok` and `wavehouse_mq_history_behind_seconds`. The `mqtest` conformance suite passes against it, connected as the shipped restricted `wavehouse` user. `internal/mq/natstest` stands NATS up from the shipped Helm values and manifests for tests outside `internal/mq`, and `make test-integration` runs the external backend's tests, since each starts a NATS server. The deployment guide gains an External NATS section: the topology and how to create it, durability, permissions, the limits that differ from the embedded queue, choosing and changing N and V, scaling ingest processes, the manual checks before production, and monitoring. +- **The JetStream topology an external NATS must provide, `wavehouse mq manifests` and `wavehouse mq permissions`, and a check for it** (`internal/mq/{nats_topology,nats_manifests}.go` (new; + tests), `internal/mq/natstest/` (new), `internal/mq/natsspike/` (new, integration-tagged), `cmd/wavehouse/{main,mq}.go` (`mq.go` new; + test), `deployments/nats/{jetstream.yaml,values.yaml}` (new), `docs/src/content/docs/deployment.md`): part of [#613](https://github.com/Wave-RF/WaveHouse/issues/613). The operator owns every stream, durable and bucket: N ingest partitions of V shards each, a history stream, one dead-letter stream (`.dlq.>`) and the lease bucket (`_coord`). A row's subject is `.ingest.

...

`, its partition and shard chosen by independent jump consistent hashes of the tenant and table, so changing N or V moves only about 1/N or 1/V of the tables, and a tenant's tables spread over the partitions. Each partition has work-queue retention (a row stays until the ingest worker acks it, even while no consumer covers it, and is deleted then, so one tenant's unwritten rows never hold back another's) and republishes every row to `.hist..
`, which the history stream captures: it holds `.hist.>`, has no sources, and keeps `15m` in the generated manifests, the seed's `stream.gap_window_minutes`. Each shard has its own durable, `-`, with the `pinned_client` priority policy in the group `wavehouse` and a pinned TTL of at least twice the 5 seconds between the pulls that keep the pin of a shard at its cap of held rows, or stopping, so the server delivers a shard to one process at a time. `wavehouse mq manifests` (`--partitions`, `--shards`, `--prefix`, `--replicas`, `--ingest-consumer`, `--history-stream`, `--publish-timeout`, `--coord-bucket`, `--dedupe-lease`, `--file-store`, `--partition-max-bytes`) prints the whole topology as nack resources; `deployments/nats/jetstream.yaml` is its output for four partitions of eight shards, and `deployments/nats/values.yaml` is a NATS Helm chart snippet for a three-node cluster with a `wavehouse` user. `wavehouse mq permissions --shards V` prints that user's permissions: it may publish, read stream and consumer info, pull from, unpin and reset each shard durable, named one by one because a NATS wildcard is a whole token, and read and write the lease keys; it may not create, change, purge or delete a stream or a bucket, create a durable on a partition, or publish to the history. The verifier checks a live server against the same spec and reports every mismatch at once, `required` ones refusing boot and `recommended` ones logged. NATS 2.14 or later is required, the first release with the consumer reset that shard takeover uses. Tests pin the JetStream behavior this rests on against nats-server 2.14.6 (`internal/mq/natsspike`). Two things are left to a manual check on the operator's own cluster, listed in the deployment guide: that nack applies every generated field, and how a three-replica cluster behaves when a shard durable's or a partition's leader is killed, and under load with every shard's consumer. +- **`coord.backend: nats` holds leases in a KV bucket on the external NATS, which the ingest processes share** (`internal/mq/lease.go` (new; + integration-tagged `lease_test.go`), `internal/mq/{nats_topology,nats_manifests}.go` (+ tests), `internal/mq/natstest/natstest.go`, `internal/config/{backends,config}.go` (+ `coord_nats_test.go`, tests), `internal/app/{app,wire}.go`, `cmd/wavehouse/mq.go` (+ test), `deployments/nats/{jetstream.yaml,values.yaml}`, `tests/integration/{coord_nats,mq_nats}_test.go`, `Makefile`, `.testcoverage.yml`, `config.yaml`, `docs/src/content/docs/{deployment,architecture}.md`, `docs/src/content/docs/configuration.mdx`, `AGENTS.md`): part of [#613](https://github.com/Wave-RF/WaveHouse/issues/613), the first distributed `coord.Coordinator`. `ExternalNATS.Leases` keeps each lease as a key (`lease.`) in a KV bucket the operator creates, reached over the `mq.nats` connection and credentials; the KV revision a term was taken at is its fencing token. A candidate takes another holder's lease only after seeing the same revision unchanged for 15 seconds on its own clock, so no two servers' clocks are compared and the bucket needs no per-key TTL; the holder renews every 2 seconds and steps down after 10 without a renewal, before anyone can take over, and a clean stop deletes the key so the next holder takes over at once. Boot refuses a process running the `ingest` role with `mq.backend: nats` and `coord.backend: local`, and `coord.backend: nats` without `mq.backend: nats`. The bucket, `_coord` (`wh_coord`; `coord.nats.bucket` / `WH_COORD_NATS_BUCKET` names another), is part of the topology: `wavehouse mq manifests` prints it as a nack `KeyValue` (`--coord-bucket` renames it), boot waits for it with the streams and refuses while it is missing, the periodic check reports it on `wavehouse_mq_topology_ok`, and the shipped `wavehouse` user may read and write `lease.` keys in it and nothing else there. +- **Ingest processes share the external NATS shards out between them** (`internal/ingest/{claims,assign}.go` (new; + tests), `internal/coord/{coord,local}.go`, `internal/coord/coordtest/coordtest.go`, `internal/mq/{lease,external,mq}.go` (+ tests), `internal/app/wire.go`, `tests/integration/{shard_claims,coord_nats}_test.go`, `docs/src/content/docs/{deployment,architecture,ingest-pipeline}.md`, `docs/src/content/docs/configuration.mdx`, `AGENTS.md`): part of the external-NATS workstream of [#613](https://github.com/Wave-RF/WaveHouse/issues/613). Under `mq.backend: nats` each ingest process holds a membership lease (`ingest.m`) in the lease bucket, and every process gives every shard to one live member by rendezvous hashing capped at an even share, so no process owns more than an even share, rounded up, and a process joining or leaving moves roughly one to two times its share while that share holds (several times it where the share steps down, never more than half the shards). At most 64 processes hold a membership slot, read 16 at a time; leftover shard durables are assigned apart from the configured ones. While a rollout changes N or V, processes on the old and new counts share the shards out differently, so some may have no owner until it ends. A shard given up stops fetching while keeping its pin, waits until the rows it delivered are written, then releases the pin, so its next owner receives at once and only newer rows; a clean stop does the same for every shard around the worker's final flush. A shard whose owner died is reset to its ack floor before its new owner pulls it (waiting, unbound, until the dead owner's pin lapses and the shard has been idle for the pinned TTL), so the rows the dead process held come back at once instead of after `ack_wait`; a lone process restarting after a clean stop does the same on its first pass. Each shard may hold unsettled rows up to an even share of the worker's 10,000 (`maxAckPending`) over the shards its process holds, and never under 1,000, two batches (a row stops counting at its first ack or nak attempt, or after `ack_wait`), where a shard durable's own `max_ack_pending` of 10,000 would let a process owning 32 shards hold 320,000; a per-shard `max_ack_pending` of 10,000/V was not chosen, since at V=32 it is below the worker's batch of 500 and a busy table's batch could never fill. New metrics: `wavehouse_ingest_shards_owned`, `wavehouse_ingest_shard_members`, `wavehouse_ingest_shards_unowned` (the lowest-ranked member only), `wavehouse_ingest_rows_held`, `wavehouse_ingest_shard_events_total` and `wavehouse_ingest_shard_handover_seconds`. `coord.Observer` reads whether a lease is held without campaigning; the broker's side is `mq.Sharded` (`IngestUnits`, `ResetOrphaned`, which answers `ErrUnitHeld` while a consumer holds the unit or was active on it within its pinned TTL, and `Unowned`), `mq.Releaser` and `mq.Halter`. +- **A multi-process test of the split roles on shards** (`tests/integration/{roles,setup}_test.go`): part of the external-NATS workstream of [#613](https://github.com/Wave-RF/WaveHouse/issues/613). `TestRoles_SeparateProcesses` runs the real binary as separate processes, two with `roles=api` and two with `roles=ingest`, over an operator-provisioned NATS (the shipped four partitions of eight shards), a Redis cache and a dynamodb-local dedupe table. It checks that the two ingest processes split the 32 shards evenly with none unowned, that ingest lands in ClickHouse exactly once, that cache invalidation and SSE reach both API processes, and that a duplicate id is accepted once. A third ingest process then takes its share by handover, and one killed with `SIGKILL` has its shards taken over about one membership lease later, with every batch still written once. `TestRoles_BootRefusesWhatTheBackendsCannotServe` drives the boot rules through the binary, including an `ingest` process on nats with local leases and a sweeper-only process on nats. - **`dedupe.backend: dynamodb` selects the shared DynamoDB dedupe table** (`internal/config/{backends,config}.go` (+ tests), `internal/app/{app,wire,wire_dynamodb}.go` (+ `dedupe_dynamodb_test.go`), `internal/dedupe/{stores,dynamodb}.go` (+ tests), `tests/integration/dedupe_dynamodb_app_test.go` (new), `.testcoverage.yml`, `config.yaml`, `AGENTS.md`, `docs/src/content/docs/{configuration.mdx,settings-directory.mdx,deployment.md,architecture.md,api.md,sdk/reference.md}`): PR F5 of [#613](https://github.com/Wave-RF/WaveHouse/issues/613). Pods that set it share seen ids, so an id ingested through one is a duplicate through every other. New boot keys: `dedupe.lease` (`WH_DEDUPE_LEASE`, `30s`, how long a claimed id stays pending and the in-flight `503`'s `Retry-After`; at most `59s` with the embedded queue, so that the lease plus its own ceiling to the next second plus one more second fits its 2-minute duplicate window: a client obeying that `Retry-After` after an uncertain publish can republish as late as the lease plus that ceiling plus a second after the claim, since DynamoDB rounds a claim's expiry up to the second), `dedupe.reserve_concurrency` (`WH_DEDUPE_RESERVE_CONCURRENCY`, `64`, which also sizes the DynamoDB client's idle connections per host; ingest sends a window of up to 256 ids per call), and the `dedupe.dynamodb` block (`table` (required), `region`, `endpoint`, `timeout` `250ms`, `max_attempts` `3`, `retry_mode` `standard`/`adaptive`, `create_table`), each with its `WH_DEDUPE_DYNAMODB_*` variable. Their defaults are in `defaults()` like every boot key's, so an explicit `0` lease, concurrency, timeout or attempt count, or an empty `retry_mode`, refuses boot rather than becoming the default (the `dynamodb` block's only while `dynamodb` is selected). Credentials come from the AWS SDK's default chain, never from config. Boot checks the table (key schema `pk` String alone; TTL off on `ex` is a warning) in a process running the `api` role, the one that opens the dedupe stores, whether or not a tenant has dedupe on: a misconfigured table (missing, the wrong key schema, access denied) refuses boot over a flat settings directory whose tenant has dedupe on and is logged at `ERROR` otherwise; any other failure (a throttle, a timeout, the network), a nested directory, or no tenant deduping yet boots and fails every switched-on tenant's ingest closed until the check passes, retried in the background (1s backing off to 30s) and at once after every reload. A reload makes no table call and does not wait on a tenant whose dedupe setting is unchanged: it holds the lock that serializes reloads, so it applies each tenant's switch against the last check's result and only wakes the retry; it waits only for a tenant whose store it closes — dedupe switched off, or the tenant removed or rejected — and then only for that tenant's in-flight calls, before its store closes. No region from the config or the SDK chain refuses boot. `create_table` creates a missing table at boot (an endpoint not up yet is a transient failure, retried like the check) and is refused unless `endpoint` is set, so it only ever reaches dynamodb-local. - **A DynamoDB dedupe backend** (`internal/dedupe/dynamodb.go` (new, + tests), `tests/integration/{setup,dedupe_dynamodb}_test.go`, `go.mod`, `AGENTS.md`, `docs/src/content/docs/{architecture,deployment}.md`): PR F3 of [#613](https://github.com/Wave-RF/WaveHouse/issues/613). `dedupe.Dynamo` keeps every tenant's seen ids in one shared table, so pods that share the table also share seen ids, which the per-process Pebble store cannot do. The table's only key is a String `pk` holding the readable dedupe key (`acme/clicks/evt-123`), so an item reads as-is in the console. `Reserve` is a conditional `PutItem` that is atomic across pods; an SDK retry of a put DynamoDB applied but whose answer was lost (a `500`, a reset connection) finds its own item by token and keeps the claim, rather than answer `InFlight` and hold the id for the lease. `Commit` is `BatchWriteItem`, retrying for up to eight jittered rounds both the items DynamoDB leaves unprocessed and a batch it throttled whole, and `Release` is a `DeleteItem` conditional on the claim's token. A client that disconnects mid-`Reserve` does not strand its claim: puts already sent run to their answer on a context its cancellation does not reach, and are then released, so its retry is not answered `InFlight` for the lease. Only a put cut off by its own call timeout (which DynamoDB may apply after the release), or a release that fails, still holds its id until the lease ends. An expired item counts as absent without waiting for TTL. Each call has a 250 ms timeout covering the SDK's three attempts, whose retries back off with full jitter under a ceiling that keeps their waits within half the timeout, so a throttled call fails with the throttle as its cause rather than on the deadline. Throttling, timeouts and an unreachable table wrap `ErrUnavailable`, and five such failures in a row within a second short-circuit claims for a second. Credentials come from the AWS SDK's default chain. The HTTP client keeps one idle connection per host for each of the 64 calls a `Reserve`, `Commit` or `Release` runs at once (the SDK's default keeps 10), so a warm 64-key `Reserve` reuses every connection rather than open about 50. WaveHouse never creates the production table: `CreateTable` works only against dynamodb-local, and the Deployment page carries an example Terraform table and IAM policy. The backend passes the `dedupetest` conformance suite against a pinned `amazon/dynamodb-local` container, along with 32 clients racing one id, injected throttles and an unreachable endpoint. New metrics: `wavehouse_dedupe_dynamodb_requests_total`, `_request_duration_seconds`, `_unprocessed_items_total`, `_short_circuits_total`. `dedupe.backend: dynamodb` selects it (entry above). New dependencies: `aws-sdk-go-v2` (`service/dynamodb`, `config`) and what they require. - **`cache.backend: redis` shares the query cache across instances** (`internal/config/{cache_redis,backends,config}.go` (+ tests), `internal/app/wire.go` (+ tests), `internal/ingest/worker.go`, `tests/integration/shared_cache_test.go`, `scripts/orchestrator/main.go`, `tests/e2e/fixtures/config.yaml`, `.testcoverage.yml`, `deployments/compose/dependencies.yaml`, `config.yaml`, `.github/workflows/{ci.yml,README.md}`, `docs/src/content/docs/{configuration.mdx,deployment.md,architecture.md,settings-directory.mdx,getting-started.md,pipes.mdx,api.md,development.md,index.mdx,why-wavehouse.md,sdk/reference.md}`, `README.md`, `AGENTS.md`): PR E4 of the distributed-deployment epic ([#613](https://github.com/Wave-RF/WaveHouse/issues/613)). The boot config's `cache.backend` now takes `redis`, configured by a new `cache.redis` block (`WH_CACHE_REDIS_*`): `addrs` (required), `mode` (`standalone` or `cluster`; `sentinel` refuses boot until [#656](https://github.com/Wave-RF/WaveHouse/issues/656)), `username`, `password` (a secret — set it through the environment), `db`, `tls.{enabled,ca_file,cert_file,key_file,server_name,insecure_skip_verify}`, `key_prefix` (`wh`), `timeout` (`100ms`) and `dial_timeout` (`1s`), each at most `1s` since boot and shutdown each wait out a connection attempt they bound, `max_value_bytes` (1 MiB), `compress_min_bytes` (`1024`; `0` never compresses) and `version_ttl` (`168h`). Every instance pointed at one server shares its cached results, and an insert on any instance invalidates every instance's. It is also the shared cache a split of [`roles`](https://github.com/Wave-RF/WaveHouse/pull/622) needs: the refusal of `api` without `ingest`, or the reverse, over `cache.backend: local` now names `redis`, though every split is still refused while the queue is embedded. A malformed block — no address, an address without a valid port, more than one address in `standalone` mode, a URL-style address (refused without repeating it, since it may carry a password), an unknown mode, `db` other than `0` in cluster mode, an unreadable TLS file, a TLS key set while `tls.enabled` is off — refuses boot; an unreachable server, or one that rejects the password, does not: the process boots with the cache bypassed and keeps reconnecting, logging a rejected password at `ERROR` on every attempt, so a rotated secret cannot crash-loop every instance at once. `insecure_skip_verify`, and a `cache.redis.addrs` set while `cache.backend` is `local`, are logged at `WARN` at boot. The ingest worker's log of an invalidation that did not land drops from `ERROR` to `WARN`, since the shared backend defers and retries it: an outage would otherwise log an `ERROR` for every batch. The e2e suite now runs against a Redis testcontainer with `cache.backend: redis`, so the shared backend is exercised end to end; the e2e per-suite exclude now names what its run still can't reach instead — `internal/cache/pending.go` (the retry of an invalidation the server did not take, which needs an outage), `internal/config/cache_redis.go` (the block's own rejection paths) and `internal/cache/(local|version_manager).go` (the `local` backend, which e2e no longer runs) — and the unit and integration suites keep covering them. An integration test boots two instances over one Redis and one ClickHouse: a result one fills is a hit for the other, and a row ingested through one is served fresh by the other on its next query, well inside the stale entry's TTL; another pauses Redis and checks queries keep succeeding from ClickHouse, then turn back to hits; a third runs the first one's hit, insert and fresh-miss lifecycle on the suite's own `cache.backend: local` app, since e2e no longer exercises that backend. `deployments/compose/dependencies.yaml` gains an optional `redis` profile for local multi-instance work. The deployment guide gains a "Multiple instances and the shared cache" section: what each instance keeps to itself, what a reader on another instance can see and when, `maxmemory-policy`, and the metrics to alert on. - **A Redis-compatible shared cache backend** (`internal/cache/{redis,redis_codec,breaker,pending,metrics}.go` (+ tests), `internal/cache/cache.go`, `internal/cache/redis_integration_test.go`, `Makefile`, `go.mod`, `.testcoverage.yml`, `docs/src/content/docs/{architecture,development}.md`, `AGENTS.md`, `CONTRIBUTING.md`): PR E3 of the distributed-deployment epic ([#613](https://github.com/Wave-RF/WaveHouse/issues/613)). `cache.RedisCache` keeps query results and their versions in one Redis, Valkey, Dragonfly, ElastiCache or MemoryDB server shared by every process, so an insert one process makes invalidates what every other process has cached. Versions are random tokens, one per tenant, table and scope, under the tenant's hash tag, with the table and scope escaped into the key (`internal/keyenc`) so no two names share a token; a bump sets a fresh one, and a value carries the tokens it was computed under, so a lookup is one pipelined round trip (`MGET` of the tokens plus `GET` of the value, no scripts) and a token lost to eviction, expiry, `FLUSHALL` or a restart without persistence can only cause misses — `maxmemory-policy allkeys-lru` is safe. Restoring an RDB or AOF snapshot, or a backup, is a rollback instead (a restart after a crash that reloads the server's last save included, which stock Redis and Valkey make by default): the old tokens return with their values, so invalidations made since are undone until those entries' TTL. Values of 1 KiB or more are zstd-compressed when that makes them smaller, and a value over 1 MiB stored is not cached. A server that fails or takes longer than the per-operation timeout (100 ms) is a miss, a skipped fill and a deferred invalidation, never a failed query; five failures in a row, or one reply refusing writes (`READONLY` from a demoted primary, `OOM` when full under `noeviction`, and the like) or the credentials (`WRONGPASS` or `NOAUTH` after a password rotation), open a circuit breaker — logged once per opening, at `ERROR` for the credentials and `WARN` otherwise, while a failed probe reopening it every 5 s logs only at `DEBUG` unless its cause changed — that skips the server until a probe write succeeds within the per-operation timeout (connections are replaced every minute, so after a failover behind a stable address the process reaches the new primary, and delivers the bumps it owes, within about a minute; the probe also allows up to twice the dial timeout for a reconnect, but the client's other connections must reconnect within the per-operation timeout, so that timeout should still exceed a reconnect), and deferred invalidations are retried until they land — the first at once, and at the probe's cadence while the breaker is open — collapsing to one tenant-wide bump per tenant past 100,000 keys; until one lands, the process that owes it bypasses the lookups it would orphan. New metrics: `wavehouse_cache_lookups_total{backend,result}`, `wavehouse_cache_op_duration_seconds{backend,op}`, `wavehouse_cache_breaker_open{backend}`, `wavehouse_cache_invalidations_total{backend,result}`, `wavehouse_cache_invalidations_pending{backend}`, `wavehouse_cache_value_bytes{backend}`, `wavehouse_cache_oversize_total{backend}`, `wavehouse_cache_set_failures_total{backend,reason}`. `cache.backend: redis` selects it (the entry above). Tested against Redis 8.10, Valkey 8.1, Dragonfly 2.0 and a Redis Cluster node by the conformance suite, plus lost-token, snapshot-rollback, compression, stored-size, server-stops-answering, refused-writes (a demoted primary, a full `noeviction` server), rotated-credentials, slow-reconnect (over TLS), slow-server, failover-behind-a-stable-address and owed-invalidation cases; `make test-integration` now also runs `internal/cache`'s integration-tagged tests. Adds `github.com/redis/rueidis` (Redis org, Apache-2.0; its only runtime dependency is `golang.org/x/sys`) and makes `github.com/klauspost/compress` a direct dependency. -- **One conformance suite for every `mq.Broker`, and a transient broker failure is a `503`** (`internal/mq/mqtest/` (new: the suite and the embedded broker's run of it), `internal/mq/{mq,embedded}.go` (+ tests), `internal/api/ingest.go` (+ tests), `.testcoverage.yml`, `docs/src/content/docs/{api,architecture}.md`, `AGENTS.md`): the first piece of the external-NATS workstream of [#613](https://github.com/Wave-RF/WaveHouse/issues/613). `mqtest.Run` states the `Broker` contract as behavior — round trips with names that need encoding, per-tenant order, `Nak` and `AckWait` redelivery, the trace context reaching `Subscribe`, dead-lettering that keeps the topic and leaves the original unacked, per-tenant dead-letter counts, replay bounds and isolation, and exactly one `failed` report when delivery ends underneath a consumer — through the interfaces alone, so the external backend runs the same cases, with `mqtest.Caps` for the four places where its semantics legitimately differ. The embedded broker passes it; writing it turned up that a durable deleted on several tenants' queues could report on `failed` more than once, which is fixed and pinned by a test that deletes it on one queue after another. It also turned up a replay that lost its connection mid-pull passing for a caught-up one when the pull ended in a timeout; that is an error now, as the `Replayer` contract says. The interface comments now allow a delivery unit that is a partition holding several tenants, a `CreateConsumer` that finds a durable rather than creating one, a `PurgeAcked` that leaves retention to the operator, and zero dead-letter counts where there is no per-tenant queue. A new sentinel, `mq.ErrUnavailable`, is a broker that cannot be reached or does not answer in time: the ingest handler answers it with `503` + `Retry-After: 5` rather than the `500` "publish failed" it would have been. Nothing returns it yet; the external backend of #613 will. -- **Process roles: the API and the background workers can run in separate processes** (`internal/config/config.go` (+ `roles_test.go`, `defaults_test.go`), `internal/config/backends.go`, `internal/app/{app,wire}.go` (+ `roles_test.go`), `internal/api/router.go` (+ tests), `tests/integration/{setup,tenants}_test.go`, `config.yaml`, `docs/src/content/docs/{configuration.mdx,deployment.md,architecture.md}`, `AGENTS.md`), part of [#613](https://github.com/Wave-RF/WaveHouse/issues/613). The boot config gains `roles` (`WH_ROLES`, default `api,ingest,sweeper`, set in `defaults()` like every boot default, so an explicit `roles: []` refuses boot) and `instance_id` (`WH_INSTANCE_ID`, default `-<8 hex>`, fresh at every boot; logged at boot, and recorded as a lease holder once a shared `coord.backend` exists). A process wires only what its roles need: `api` runs the HTTP API with schema discovery, the token verifiers, the dedupe stores and the SSE hub (all per API process); `ingest` runs the ingest worker; `sweeper` runs the sweeper under its lease. A process without `api` serves an ops-only listener on `server.port` (the probes and their aliases, `/version`, the same-port metrics path, and `POST /v1/ops/settings/reload`, which takes the operator key alone); every other route answers 404, under `/v1/ops` once the operator key has passed. Boot refuses any split over the embedded MQ, which no other process can reach, and `api` without `ingest` (or the reverse) over a local cache, which the ingest worker's invalidations would never reach. Until a shared `mq.backend` exists, every process therefore runs every role, which is the default, so nothing changes for an existing deployment. `data_dir` is probed for Pebble only in a process running `api`. A `config.Config` built without `config.Load` must now name its roles (`config.AllRoles()` for all of them): `app.New` refuses an empty set. -- **Leases for work that must run in one process at a time, and the sweeper runs under one** (`internal/coord/` (new: `coord.go`, `local.go`, `elect.go`, `coordtest/`, + tests), `internal/app/{app,wire}.go` (+ tests), `internal/config/backends.go`, `internal/ingest/sweeper.go`, `.github/labeler.yml`, `.testcoverage.yml`, `docs/src/content/docs/{architecture,development,ingest-pipeline}.md`, `docs/src/content/docs/configuration.mdx`, `config.yaml`, `AGENTS.md`): part of [#613](https://github.com/Wave-RF/WaveHouse/issues/613). `coord.Coordinator` hands out named leases (`TryAcquire` → a `Term` with a strictly increasing fencing `Token`, a `Done` channel and `Resign`; `ErrHeld` while another holder's term is live), and `coord.RunElected` runs a loop only while its process holds the lease, resigning when the loop returns and campaigning again every 2s. `coord.Local` is the in-process implementation, and `coordtest.Conformance` is the suite every implementation runs — the NATS KV backend that lets several replicas share one queue comes next. The sweeper now runs through `RunElected` under the `sweeper` lease; with the in-process coordinator the one process always holds it, so nothing changes for a single-process deployment beyond one `coord: elected` log line at startup. `coord.backend` now selects the coordinator (`local`, the only value), so a `config.Config` built without `config.Load` must name it as well as the other three layers' backends. +- **One conformance suite for every `mq.Broker`, and a transient broker failure is a `503`** (`internal/mq/mqtest/` (new: the suite and the embedded broker's run of it), `internal/mq/{mq,embedded}.go` (+ tests), `internal/api/ingest.go` (+ tests), `.testcoverage.yml`, `docs/src/content/docs/{api,architecture}.md`, `AGENTS.md`): the first piece of the external-NATS workstream of [#613](https://github.com/Wave-RF/WaveHouse/issues/613). `mqtest.Run` states the `Broker` contract as behavior — round trips with names that need encoding, per-tenant order, `Nak` and `AckWait` redelivery, the trace context reaching `Subscribe`, dead-lettering that keeps the topic and leaves the original unacked, per-tenant dead-letter counts, replay bounds and isolation, and exactly one `failed` report when delivery ends underneath a consumer — through the interfaces alone, so the external backend runs the same cases, with `mqtest.Caps` for the four places where its semantics legitimately differ. The embedded broker passes it; writing it turned up that a durable deleted on several tenants' queues could report on `failed` more than once, which is fixed and pinned by a test that deletes it on one queue after another. It also turned up a replay that lost its connection mid-pull passing for a caught-up one when the pull ended in a timeout; that is an error now, as the `Replayer` contract says. The interface comments now allow a delivery unit that is a partition holding several tenants, a `CreateConsumer` that finds a durable rather than creating one, a `PurgeAcked` that leaves retention to the operator, and zero dead-letter counts where there is no per-tenant queue. A new sentinel, `mq.ErrUnavailable`, is a broker that cannot be reached or does not answer in time: the ingest handler answers it with `503` + `Retry-After: 5` rather than the `500` "publish failed" it would have been. The external NATS backend returns it. +- **Process roles: the API and the background workers can run in separate processes** (`internal/config/config.go` (+ `roles_test.go`, `defaults_test.go`), `internal/config/backends.go`, `internal/app/{app,wire}.go` (+ `roles_test.go`), `internal/api/router.go` (+ tests), `tests/integration/{setup,tenants}_test.go`, `config.yaml`, `docs/src/content/docs/{configuration.mdx,deployment.md,architecture.md}`, `AGENTS.md`), part of [#613](https://github.com/Wave-RF/WaveHouse/issues/613). The boot config gains `roles` (`WH_ROLES`, default `api,ingest,sweeper`, set in `defaults()` like every boot default, so an explicit `roles: []` refuses boot) and `instance_id` (`WH_INSTANCE_ID`, default `-<8 hex>`, fresh at every boot; logged at boot, and recorded as the holder of a NATS lease). A process wires only what its roles need: `api` runs the HTTP API with schema discovery, the token verifiers, the dedupe stores and the SSE hub (all per API process); `ingest` runs the ingest worker; `sweeper` runs the sweeper under its lease, on the embedded MQ only. A process without `api` serves an ops-only listener on `server.port` (the probes and their aliases, `/version`, the same-port metrics path, and `POST /v1/ops/settings/reload`, which takes the operator key alone); every other route answers 404, under `/v1/ops` once the operator key has passed. Boot refuses any split over the embedded MQ, which no other process can reach, and `api` without `ingest` (or the reverse) over a local cache, which the ingest worker's invalidations would never reach. Over the embedded MQ every process therefore runs every role, which is the default, so nothing changes for an existing deployment; a split needs `mq.backend: nats` (above). `data_dir` is probed for Pebble only in a process running `api`. A `config.Config` built without `config.Load` must now name its roles (`config.AllRoles()` for all of them): `app.New` refuses an empty set. +- **Leases for work that must run in one process at a time, and the sweeper runs under one** (`internal/coord/` (new: `coord.go`, `local.go`, `elect.go`, `coordtest/`, + tests), `internal/app/{app,wire}.go` (+ tests), `internal/config/backends.go`, `internal/ingest/sweeper.go`, `.github/labeler.yml`, `.testcoverage.yml`, `docs/src/content/docs/{architecture,development,ingest-pipeline}.md`, `docs/src/content/docs/configuration.mdx`, `config.yaml`, `AGENTS.md`): part of [#613](https://github.com/Wave-RF/WaveHouse/issues/613). `coord.Coordinator` hands out named leases (`TryAcquire` → a `Term` with a strictly increasing fencing `Token`, a `Done` channel and `Resign`; `ErrHeld` while another holder's term is live), and `coord.RunElected` runs a loop only while its process holds the lease, resigning when the loop returns and campaigning again every 2s. `coord.Local` is the in-process implementation, and `coordtest.Conformance` is the suite every implementation runs, the NATS KV backend above included. The sweeper now runs through `RunElected` under the `sweeper` lease; with the in-process coordinator the one process always holds it, so nothing changes for a single-process deployment beyond one `coord: elected` log line at startup. `coord.backend` now selects the coordinator (`local` by default, or `nats`), so a `config.Config` built without `config.Load` must name it as well as the other three layers' backends. - **Each layer's implementation is chosen at boot** (`internal/config/{backends,config}.go` (+ tests), `internal/config/defaults_test.go`, `internal/app/{app,wire}.go` (+ tests), `cmd/wavehouse/main.go`, `tests/integration/{setup,tenants}_test.go`, `config.yaml`, `docs/src/content/docs/{configuration.mdx,settings-directory.mdx,architecture.md}`): the first step of running WaveHouse as more than one process ([#613](https://github.com/Wave-RF/WaveHouse/issues/613)). The boot config gains `mq.backend` (`WH_MQ_BACKEND`, default `embedded`), `cache.backend` (`WH_CACHE_BACKEND`, `local`), `dedupe.backend` (`WH_DEDUPE_BACKEND`, `pebble`) and `coord.backend` (`WH_COORD_BACKEND`, `local`). Each layer's in-process backend is its default, so nothing changes for a config that sets none of them; a value with no backend refuses boot and names the valid ones. A backend's own settings go in a `.` sub-block (`dedupe.dynamodb` is the first); any other sub-block is an unknown key and refuses boot. `internal/app` picks each implementation in one `switch` per layer (`wireMQ`, `wireCache`, `wireDedupe`), `data_dir` is probed only when a selected backend keeps state there (`Config.NeedsDataDir`), and boot logs at `WARN` each line of `Config.Warnings`, the combinations that are correct for one replica only once a shared queue exists. A `config.Config` built without `config.Load` must now name the `mq`, `cache` and `dedupe` backends: the zero value is not the default, and `app.New` refuses it. The defaults live in `defaults()`, like every boot key's since [#631](https://github.com/Wave-RF/WaveHouse/issues/631), so an explicit `backend: ""` in `config.yaml` refuses boot rather than becoming the default. - **A tenant removed or rejected at runtime has its open streams ended** (`internal/stream/{hub,subscriber,bucket}.go` (+ tests), `internal/api/stream.go` (+ tests), `internal/api/router.go`, `internal/settings/{registry,tree}.go` (+ tests), `internal/ingest/worker.go` (+ tests), `internal/app/wire.go` (+ tests), `clients/ts/src/stream/sse.ts`, `docs/src/content/docs/{api,deployment,architecture,ingest-pipeline}.md`, `docs/src/content/docs/settings-directory.mdx`, `AGENTS.md`): story 3 of the multi-tenant epic ([#583](https://github.com/Wave-RF/WaveHouse/issues/583)). A `GET /v1/stream` used to outlive its tenant: the hub read no policy for it and withheld every row while the keepalive wheel held the connection open, so the client could not tell it from a quiet table. After every reload the hub now evicts the subscribers of each tenant no longer served, removed or rejected alike (`Hub.Prune`, through the close-once `Subscriber.Evict`), and the handler ends the stream: a gap-fill in progress included, and a stream `TenantMW` admitted just before the reload but registered just after it, which the handler checks for as it registers. The client's reconnect gets `404` (removed) or `503` (rejected); the SDK stops on the first and retries the second, resuming from `Last-Event-ID` once the folder is back — except in a browser going cross-origin where tenant `0` is not served or its CORS list does not admit the page, which cannot read either refusal (both are decorated from tenant `0`'s list) and re-dials as after a dropped connection. A flat directory never stops serving tenant `0`, so nothing changes there. Removing a tenant is deleting its folder, then reloading the whole directory, the last folder included: a server started with tenant folders reads the emptied directory as no folder left, where it read as a change of shape and the reload was rejected whole, leaving that tenant served; at boot an empty directory still reads as the four files, missing. Reloading the deleted folder by name leaves its tenant rejected. `GET /v1/health` keeps resolving a tenant, deliberately: its `404` tells a caller with no token no more than every tenant route's does, since they all answer before authenticating, and resolving answers a served tenant's ping from that tenant's own CORS list. A batch queued for a tenant with no ClickHouse connection — one no longer served, or one no pool could be opened for (such as by the connection ceiling) — skips the row-by-row retry, which no row of it could pass, and meets its DLQ switch once, whole, logged once per batch rather than twice per row; a tenant no longer served reads the switch as on, so its queued rows are parked under its own subject rather than dropped, inserted into another tenant's ClickHouse, or left unacked, redelivered for as long as the tenant is away and holding its queue's ack floor, which the purge waits on. Over a nested directory `/livez` no longer keeps naming a tenant that stopped being served before any tenant completed a first discovery: the diagnostic goes back to `no tenant has completed a first discovery yet`. @@ -90,6 +95,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), ### Fixed +- **A shard's owner keeps it while its ClickHouse is slow, one stuck shard no longer stalls the process's others, and a handover or clean stop keeps each table's rows in order** (`internal/mq/external_consumer.go` (new, from `external.go`), `internal/mq/{external,mq}.go`, `internal/ingest/{claims,worker}.go`, `.testcoverage.yml`, tests in `internal/mq/external_test.go`, `internal/ingest/{claims,worker}_test.go` and `tests/integration/shard_order_test.go` (new), `docs/src/content/docs/{deployment,ingest-pipeline,architecture}.md`, `AGENTS.md`): part of the external-NATS workstream of [#613](https://github.com/Wave-RF/WaveHouse/issues/613), fixing the shard ownership of the entries above before they ship. The pull consumer ran the worker's handler on its delivery goroutine and pulled again only when it returned, and the server renews a shard's pin only on a pull, so a process at its cap of held rows for longer than the pinned TTL (10 seconds) lost its pins: `wavehouse_ingest_shards_unowned` counted its shards, and a peer that judged it dead reset them, redelivering rows it still held — two writers per table. Each shard now has a puller that never runs the handler, fetches only what the shard's cap leaves room for, and at the cap renews the pin every 5 seconds with a one-row fetch, at most one `ack_wait` of such rows past the cap, 12 at the generated 1 minute (past that, and once halted, with a pull of `max_bytes` 1, which delivers nothing); measured, an owner blocked 13 seconds on a hung insert kept one pin throughout and took 2 rows past its share, and a unit drains as fast as before (160,000–192,000 rows a second, against 158,000–172,000). `ResetOrphaned` and `Unowned` also require that the shard delivered nothing and had no ack for its pinned TTL. The process-wide cap of 10,000 held rows let one stuck table take every slot and stall every shard the process owned; each shard now holds its own share, plus at most the 12 rows its renewals take (see the entry above), and with one shard's share full a healthy shard's rows kept flowing. A clean stop now halts every shard first (`mq.Halter`), keeping the pins, so the worker writes what it holds and what the shards still deliver, and releases them after its final flush; rows delivered during that flush were dropped before, and came back only once they had been waited out. A handover keeps the pin until the rows delivered are written, bounded by the worker's 60-second ack wait instead of 15 seconds, and a row the worker drops while stopping is NAKed before the release. Checked under continuous publishing, with the first process's ClickHouse slower than a batch window: every table's rows arrived exactly once and in order across a handover and a clean stop, and the next owner wrote its first rows about 2 seconds after the stop, against 9 seconds when the stop leaves rows behind. A shard fetches in 1-second pulls, so a shard stops fetching within a second of a halt, and an idle shard costs one pull a second. A shard durable deleted while no pull of it was waiting stalled silently until the five-minute topology check, since the next pull only got no responders: two such pulls in a row, or one renewal, now look the durable up, and one that is gone, or whose stream is, ends the worker (measured: 5 seconds after the delete, with the handler busy). A bind whose durable or stream is gone, or whose durable no longer fits (`mq.ErrConsumerMismatch`: `ack_wait` shorter than asked, or no `max_ack_pending`), ends the worker instead of retrying every tick with a warning, and a bind failure that lasts a minute logs an error. `wavehouse_ingest_rows_held_waits_total` is gone: nothing waits at the cap any more. A renewal at the cap takes a row rather than holding it back: a held-back redelivery goes to the back of the server's redelivery queue, and with max_bytes-1 renewals rows NAKed as 0, 1, 2, 3 came back as 2, 3, 0, 1 (measured); now in order, until a stuck shard is 12 rows past its cap. The server requeues the same way while another process's pull waits for the pin, so rows redelivered across a handover or a stop may come back out of order among themselves, though still ahead of newer rows. The unpin on release names no pin id, so it is preceded by a check that the pin is still this process's; the unit renews its pin until the unpin returns (for up to `ack_wait` and one 5-second renewal after its halt has handed on what it fetched, longer than a handover waits), so the pin cannot lapse between the two, and only a consumer leader change there could move it. +- **External NATS: the shipped manifests fit the shipped volume, boot refuses permissions narrower than the shards, and the tooling covers `js_domain`** (`internal/mq/{nats_manifests,nats_topology,external}.go`, `internal/mq/nats_permprobe.go` (new), `internal/mq/natstest/natstest.go`, `cmd/wavehouse/mq.go` (+ tests; `internal/mq/{nats_manifests,external_perms}_test.go` new), `deployments/nats/{jetstream.yaml,values.yaml}`, `docs/src/content/docs/{deployment,architecture}.md`, `configuration.mdx`): part of [#613](https://github.com/Wave-RF/WaveHouse/issues/613). The shipped streams reserved 225 GiB against the shipped 100 Gi volume, whose size the NATS Helm chart makes each server's `max_file_store`, so the server refused the third partition. `wavehouse mq manifests --file-store` (default `100Gi`) now sizes each partition at 15% of it, the history at 10% and the dead-letter stream at 5%, so the shipped four partitions reserve 75 GiB; a partition's size does not follow N, and the generator refuses streams that reserve more than the store (`--partition-max-bytes` resizes them). The test server takes its file store from the shipped values as the chart does, where it had a limit no test could reach. Boot now probes every shard durable as the connecting user, with a pull and an unpin the server rejects on their merits, and a `required` finding names each durable the user may not pull, where permissions generated for fewer shards passed boot and left those shards' rows unpulled; a consumer request the server refuses later sets `wavehouse_mq_topology_ok` to `0` at once. `wavehouse mq permissions --js-domain` allows and denies the `$JS..API` subjects a client in a JetStream domain sends, beside the plain ones a server in the domain checks after mapping them; `wavehouse mq manifests` takes `--ingest-consumer`, `--history-stream` and `--publish-timeout`, the duplicate window following the timeout; the `wavehouse` user may ack only for its shard durables, under either ack subject layout, where it could ack for any consumer; and an `mq.nats.ingest_consumer` or `history_stream` outside `[a-zA-Z0-9_-]` refuses boot. - **One tenant's failed queue join no longer ends ingest for every tenant** (`internal/mq/{mq,embedded}.go` (+ tests), `docs/src/content/docs/{architecture,durability,ingest-pipeline}.md`, `settings-directory.mdx`): closes [#675](https://github.com/Wave-RF/WaveHouse/issues/675). When a tenant's queue opened while the server ran and the ingest worker's consumer could not join it, the broker reported that as the consumer's delivery ending, so the worker failed and the process restarted — one tenant's failure taking ingest down for all of them — while the stream hub's consumer failing the same way was only logged, and the tenant's `GET /v1/stream` connections got no live rows until a restart. A queue a consumer cannot join is now left unrecorded as open, whichever consumer it is: `SetMaxBytes` returns the error and `MaxBytes` keeps not reporting the budget, so the next reload tries the consumers that had not joined again, and the tenant's publishes answer `503` and try them too, paced as a queue that cannot open already is (one shared attempt, then refusals for five seconds). No row is accepted that a consumer would not read, no other tenant is touched, and nothing restarts. A queue reopened after its ingest stream went missing has its consumers' delivery started again on the new stream, where the stopped delivery on the old one used to stand in for it. A consumer that had joined and whose delivery then ends on its own (a deleted durable, a closed connection) still fails the worker, and boot is unchanged: a flat directory whose queue cannot open refuses boot. - **Dedupe claims an id, publishes, then commits it — and keys it by tenant, table and id** (`internal/dedupe/{dedupe,key,embedded,managed}.go` (+ tests), `internal/dedupe/dedupetest/` (new), `internal/api/ingest.go` (+ tests), `internal/settings/validate_test.go`, `internal/keyenc/keyenc.go`, `internal/testutil/mocks.go`, `internal/app/app_test.go`, `AGENTS.md`, `docs/src/content/docs/{architecture,api,deployment,development}.md`, `settings-directory.mdx`, `sdk/reference.md`): `CheckAndMark` is replaced by a two-phase `Reserve` → `Commit` / `Release` contract with a lease on the pending claim, and every backend now runs one conformance suite. Four bugs go with it. Two concurrent requests carrying one id no longer both publish it: Pebble's check and claim happen under one lock, and the loser answers `503` with `Retry-After` while the winner is still publishing ([#390](https://github.com/Wave-RF/WaveHouse/issues/390)). A publish that fails gives its id back, so the retry a `503` asks for is published instead of skipped as a duplicate of a record that never reached the queue ([#384](https://github.com/Wave-RF/WaveHouse/issues/384)) — the residual case is a publish that fails after it already reached the broker (a timeout, a disconnect), where the released id lets the retry through but that retry publishes a genuine second copy; [#629](https://github.com/Wave-RF/WaveHouse/pull/629) closes that with an idempotency key. The same id in two tables is two ids ([#222](https://github.com/Wave-RF/WaveHouse/issues/222)'s keyspace half). An explicit `null` id is a missing id — rejected under `require_id`, published un-deduped otherwise — instead of the one id `""` that made every null record after the first a duplicate ([#370](https://github.com/Wave-RF/WaveHouse/issues/370)). **Upgrade:** the key layout changes, so an id seen before the upgrade is accepted once more after it; nothing is migrated, and the old keys are left in `/pebble`, unread, deleted by the retention sweep (see Added) ([#220](https://github.com/Wave-RF/WaveHouse/issues/220)) ([Deployment → Upgrading across the dedupe key change](https://github.com/Wave-RF/WaveHouse/blob/main/docs/src/content/docs/deployment.md#upgrading-across-the-dedupe-key-change)). The key is readable text, `/
/` (for example `acme/clicks/evt-123`), with the table and id escaped and joined by `internal/keyenc`, the escaping NATS subject tokens already use, so any table name gets a keyspace of its own, including one holding a NUL byte or a `/`. New metrics: `wavehouse_ingest_dedupe_commit_failed_total` (a published record whose id failed to commit; the claim lapses with its lease) and `wavehouse_dedupe_hashed_id_total` (an id over 1,024 bytes once escaped, stored as its SHA-256). - **Ingest runs in windows of 256 records over the dedupe contract, and a dedupe store that cannot answer is a `503`** (`internal/api/ingest.go` (+ tests), `internal/mq/{mq,embedded}.go` (+ tests), `internal/dedupe/key.go` (+ tests), `internal/testutil/mocks.go`, `AGENTS.md`, `docs/src/content/docs/{api,architecture,durability}.md`, `settings-directory.mdx`, `sdk/reference.md`): each window of a request is prepared, then reserved in one dedupe call, published in order, and committed in one call, so a batch costs one dedupe round trip per phase per window rather than per record — on Pebble, one commit `fsync` per window (a 1,000-record batch: four syncs instead of a thousand, 24 ms against 5.7 s of dedupe time measured with the queue stubbed). Every deduped record is published under an idempotency key (`mq.WithIdempotencyKey`, JetStream's message id, derived by `dedupe.IdempotencyKey`), and each tenant's ingest stream now keeps an explicit two-minute duplicate window, sized to `2 × the 30-second lease + 1s`: an uncertain publish's `503` sends the *full* lease as `Retry-After`, so an obedient client's retry can land up to ~2×lease after the original request, and the `+1s` covers a backend whose claim expiry itself rounds up by that much. That closes the last path of [#384](https://github.com/Wave-RF/WaveHouse/issues/384): a publish that fails with an unknown outcome (anything but a full queue) keeps its record's claim until the lease lapses instead of releasing it, and a retry after the lease but within two minutes of the first publish is dropped by the queue if the first copy was stored (a later one is stored again). A dedupe store that is not open or that reports `dedupe.ErrUnavailable` now answers `503 {"error":"dedupe store unavailable"}` with `Retry-After: 5`, which the SDK retries, rather than `500 dedupe failed`. A mid-body read error or dedupe failure now drops the open window unpublished, where records before it used to be published; `wavehouse_ingest_dedupe_commit_failed_total` and `wavehouse_ingest_dedupe_disabled_total` count records, as before, now added a window at a time. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 70bbb00ba..8ca9c126c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -39,7 +39,7 @@ Open a [feature request issue](https://github.com/Wave-RF/WaveHouse/issues/new?t The pre-push hook (installed by `make tools`) blocks a push until the tree has been validated locally: a code change needs `make ci`, a docs/prose-only change needs only `make verify` (the same split CI makes). `make lint` / `make test` / `make build` are fast inner-loop subsets. -2. Write tests for new functionality. Unit tests go alongside the code in `internal/`. Integration tests carry the `//go:build integration` tag and go in `tests/integration/`, or beside the package when they test one package against its own external server (e.g. `internal/cache/redis_integration_test.go`), with the package added to the `test-integration` target. +2. Write tests for new functionality. Unit tests go alongside the code in `internal/`. Integration tests carry the `//go:build integration` tag and go in `tests/integration/`, or beside the package when they test one package against its own external server (e.g. `internal/cache/redis_integration_test.go`), with the package added to the `test-integration` target. A test that must import NATS, which only `internal/mq` may do, goes in `internal/mq/natsspike`, or in `internal/mq` itself with the `integration` tag when it needs the package's internals (the external NATS broker's tests, which `make test-integration` selects by name). 3. Update documentation if your change affects: - API endpoints → update `docs/src/content/docs/api.md` diff --git a/Makefile b/Makefile index 5fff09f2b..85dc1ddaf 100644 --- a/Makefile +++ b/Makefile @@ -762,9 +762,18 @@ test: test-unit test-integration: go-mod-download ## Run Go integration tests + render coverage + gate threshold (requires Docker) @printf "$(CYAN)==> Running Integration Tests...$(RESET)\n" @rm -rf $(COV_INT)/data && mkdir -p $(COV_INT)/data + @# 480s: the go command kills the package at -timeout + 1m, counting + @# TestMain's wavehouse binary build before m.Run, and the multi-process + @# roles test runs its handover steps in sequence; at 240s CI killed it. + @GOCOVERDIR="$(CURDIR)/$(COV_INT)/data" go tool gotestsum --format $(GOTESTSUM_FMT) -- \ + -tags="integration $(TAGS)" -timeout 480s -coverpkg=./... -race -count=1 \ + ./tests/integration/... ./internal/mq/natsspike/... ./internal/cache/... $(ARGS) \ + -args -test.gocoverdir="$(CURDIR)/$(COV_INT)/data" + @# internal/mq's integration-tagged tests (the external NATS broker) run + @# alone: its untagged tests are the unit suite's. @GOCOVERDIR="$(CURDIR)/$(COV_INT)/data" go tool gotestsum --format $(GOTESTSUM_FMT) -- \ -tags="integration $(TAGS)" -timeout 240s -coverpkg=./... -race -count=1 \ - ./tests/integration/... ./internal/cache/... $(ARGS) \ + -run '^Test(ExternalNATS|NewNATS|NATSPermissions_Refuse|Leases)' ./internal/mq $(ARGS) \ -args -test.gocoverdir="$(CURDIR)/$(COV_INT)/data" @if [ -z "$(COV_DEFER)" ]; then go run ./scripts/cov render integration; fi diff --git a/cmd/wavehouse/main.go b/cmd/wavehouse/main.go index 3a1304f0b..6a29c30b5 100644 --- a/cmd/wavehouse/main.go +++ b/cmd/wavehouse/main.go @@ -97,6 +97,8 @@ func main() { os.Exit(runValidate(os.Args[2:])) case "bootstrap": os.Exit(runBootstrap(os.Args[2:])) + case "mq": + os.Exit(runMQ(os.Args[2:], os.Stdout, os.Stderr)) case "version", "--version", "-v": fmt.Printf("wavehouse %s (commit %s, built %s)\n", Version, GitCommit, BuildTime) os.Exit(0) @@ -138,6 +140,8 @@ func printUsage(w io.Writer) { wavehouse start the server wavehouse validate [dir] validate a settings directory (dir falls back to %[1]s) wavehouse bootstrap [dir] write a starter settings directory, every key at its default (dir falls back to %[1]s) + wavehouse mq manifests print the nack resources for an external NATS JetStream + wavehouse mq permissions print the wavehouse NATS user's permissions for it wavehouse health liveness self-probe against the local server (container HEALTHCHECK) wavehouse version print version, commit, and build time wavehouse help show this help diff --git a/cmd/wavehouse/mq.go b/cmd/wavehouse/mq.go new file mode 100644 index 000000000..bc8d4a190 --- /dev/null +++ b/cmd/wavehouse/mq.go @@ -0,0 +1,182 @@ +package main + +import ( + "errors" + "flag" + "fmt" + "io" + + "github.com/Wave-RF/WaveHouse/internal/dedupe" + "github.com/Wave-RF/WaveHouse/internal/mq" +) + +// runMQ implements `wavehouse mq `: tooling for the message queue. +// Exit codes: 0 ok, 1 failed, 2 usage. +func runMQ(args []string, stdout, stderr io.Writer) int { + usage := func(w io.Writer) { + _, _ = fmt.Fprint(w, `usage: wavehouse mq + +commands: + manifests print the nack resources for an external NATS JetStream + permissions print the wavehouse NATS user's permissions for it +`) + } + if len(args) == 0 { + usage(stderr) + return 2 + } + switch args[0] { + case "manifests": + return runMQManifests(args[1:], stdout, stderr) + case "permissions": + return runMQPermissions(args[1:], stdout, stderr) + case "help", "-h", "--help": + usage(stdout) + return 0 + default: + _, _ = fmt.Fprintf(stderr, "wavehouse mq: unknown command %q\n\n", args[0]) + usage(stderr) + return 2 + } +} + +// runMQManifests implements `wavehouse mq manifests`: print the nack +// Stream, Consumer and KeyValue resources for the topology WaveHouse checks at boot +// under mq.backend: nats, for the operator to apply. +func runMQManifests(args []string, stdout, stderr io.Writer) int { + fs := flag.NewFlagSet("mq manifests", flag.ContinueOnError) + fs.SetOutput(stderr) + partitions := fs.Int("partitions", mq.DefaultNATSPartitions, "number of ingest partition streams (mq.nats.partitions)") + shards := fs.Int("shards", mq.DefaultNATSShards, "shards of every partition, one durable each (mq.nats.shards)") + prefix := fs.String("prefix", mq.DefaultNATSSubjectPrefix, "subject prefix (mq.nats.subject_prefix)") + replicas := fs.Int("replicas", 3, "replicas for every stream and the lease bucket") + consumer := fs.String("ingest-consumer", mq.DefaultNATSIngestConsumer, "the shard durables' name prefix (mq.nats.ingest_consumer)") + history := fs.String("history-stream", "", "the history stream (mq.nats.history_stream); empty is _HISTORY") + publishTimeout := fs.Duration("publish-timeout", mq.DefaultNATSPublishTimeout, "one publish attempt's bound (mq.nats.publish_timeout); the partitions' duplicate window covers every attempt") + bucket := fs.String("coord-bucket", "", "the lease KV bucket (coord.nats.bucket); empty is _coord") + lease := fs.Duration("dedupe-lease", dedupe.DefaultLease, "the dedupe.lease the partitions' duplicate window must cover") + fileStore := fs.String("file-store", mq.FormatStoreSize(mq.DefaultNATSFileStore), "every server's JetStream max_file_store (the Helm chart's fileStore.maxSize, else its PVC size), which the streams' maxBytes must fit in") + partitionBytes := fs.String("partition-max-bytes", "", "each ingest partition's maxBytes; empty is 15% of --file-store") + fs.Usage = func() { + _, _ = fmt.Fprint(fs.Output(), `usage: wavehouse mq manifests [--partitions N] [--shards V] [--prefix wh] [--replicas 3] [--ingest-consumer wh-ingest] [--history-stream S] [--publish-timeout 5s] [--coord-bucket B] [--dedupe-lease 30s] [--file-store 100Gi] [--partition-max-bytes 15Gi] + +Print the nack (jetstream.nats.io/v1beta2) Stream, Consumer and KeyValue +resources for the JetStream topology WaveHouse needs under mq.backend: nats +and coord.backend: nats, as YAML for kubectl apply. WaveHouse never creates these itself; it checks them at boot. +The streams' maxBytes are sized from --file-store: each partition 15%, the +history 10% and the DLQ 5%, so four partitions leave a quarter of it spare. +It refuses streams that together reserve more than --file-store. + +`) + fs.PrintDefaults() + } + if err := fs.Parse(args); err != nil { + if errors.Is(err, flag.ErrHelp) { + return 0 + } + return 2 + } + if fs.NArg() > 0 { + _, _ = fmt.Fprintf(stderr, "wavehouse mq manifests: unexpected argument %q\n", fs.Arg(0)) + fs.Usage() + return 2 + } + if *lease < 0 { + _, _ = fmt.Fprintf(stderr, "wavehouse mq manifests: --dedupe-lease must not be negative\n") + return 2 + } + if *shards < 1 { + _, _ = fmt.Fprintf(stderr, "wavehouse mq manifests: --shards must be at least 1\n") + return 2 + } + if *replicas < 1 { + _, _ = fmt.Fprintf(stderr, "wavehouse mq manifests: --replicas must be at least 1\n") + return 2 + } + if *publishTimeout <= 0 { + _, _ = fmt.Fprintf(stderr, "wavehouse mq manifests: --publish-timeout must be positive\n") + return 2 + } + store, err := positiveSize(*fileStore) + if err != nil { + _, _ = fmt.Fprintf(stderr, "wavehouse mq manifests: --file-store: %v\n", err) + return 2 + } + var partitionMax int64 + if *partitionBytes != "" { + if partitionMax, err = positiveSize(*partitionBytes); err != nil { + _, _ = fmt.Fprintf(stderr, "wavehouse mq manifests: --partition-max-bytes: %v\n", err) + return 2 + } + } + err = mq.WriteNATSManifests(stdout, mq.NATSManifestOptions{ + Topology: mq.NATSTopology{ + Prefix: *prefix, Partitions: *partitions, Shards: *shards, IngestConsumer: *consumer, HistoryStream: *history, + PublishTimeout: *publishTimeout, CoordBucket: *bucket, DedupeLease: *lease, + }, + Replicas: *replicas, + FileStore: store, + PartitionMaxBytes: partitionMax, + }) + if err != nil { + _, _ = fmt.Fprintf(stderr, "wavehouse mq manifests: %v\n", err) + return 1 + } + return 0 +} + +// runMQPermissions implements `wavehouse mq permissions`: print the +// wavehouse NATS user's permissions for the topology, which name every shard +// durable, so an operator regenerates them with the shard count. +func runMQPermissions(args []string, stdout, stderr io.Writer) int { + fs := flag.NewFlagSet("mq permissions", flag.ContinueOnError) + fs.SetOutput(stderr) + shards := fs.Int("shards", mq.DefaultNATSShards, "shards of every partition (mq.nats.shards); while a lower count's durables drain, the higher") + prefix := fs.String("prefix", mq.DefaultNATSSubjectPrefix, "subject prefix (mq.nats.subject_prefix)") + consumer := fs.String("ingest-consumer", mq.DefaultNATSIngestConsumer, "the shard durables' name prefix (mq.nats.ingest_consumer)") + history := fs.String("history-stream", "", "the history stream (mq.nats.history_stream); empty is _HISTORY") + bucket := fs.String("coord-bucket", "", "the lease KV bucket (coord.nats.bucket); empty is _coord") + domain := fs.String("js-domain", "", "the JetStream domain WaveHouse connects to (mq.nats.js_domain); its API requests go to $JS..API.>") + fs.Usage = func() { + _, _ = fmt.Fprint(fs.Output(), `usage: wavehouse mq permissions [--shards V] [--prefix wh] [--ingest-consumer wh-ingest] [--history-stream S] [--coord-bucket B] [--js-domain D] + +Print the permissions block for the wavehouse user of a NATS Helm values +file (config.merge.accounts..users[]), for the topology that +wavehouse mq manifests prints with the same flags. + +`) + fs.PrintDefaults() + } + if err := fs.Parse(args); err != nil { + if errors.Is(err, flag.ErrHelp) { + return 0 + } + return 2 + } + if fs.NArg() > 0 { + _, _ = fmt.Fprintf(stderr, "wavehouse mq permissions: unexpected argument %q\n", fs.Arg(0)) + fs.Usage() + return 2 + } + if *shards < 1 { + _, _ = fmt.Fprintf(stderr, "wavehouse mq permissions: --shards must be at least 1\n") + return 2 + } + err := mq.WriteNATSPermissions(stdout, mq.NATSTopology{ + Prefix: *prefix, Shards: *shards, IngestConsumer: *consumer, HistoryStream: *history, CoordBucket: *bucket, + }, *domain) + if err != nil { + _, _ = fmt.Fprintf(stderr, "wavehouse mq permissions: %v\n", err) + return 1 + } + return 0 +} + +// positiveSize reads a byte count of at least one byte (mq.ParseStoreSize). +func positiveSize(s string) (int64, error) { + n, err := mq.ParseStoreSize(s) + if err == nil && n == 0 { + err = errors.New("must be positive") + } + return n, err +} diff --git a/cmd/wavehouse/mq_test.go b/cmd/wavehouse/mq_test.go new file mode 100644 index 000000000..42b852a6d --- /dev/null +++ b/cmd/wavehouse/mq_test.go @@ -0,0 +1,116 @@ +package main + +import ( + "bytes" + "os" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// The shipped manifests are the generator's output for four partitions of eight shards. +// Regenerate with: go run ./cmd/wavehouse mq manifests --partitions 4 --shards 8 > deployments/nats/jetstream.yaml +// (the file store defaults to the shipped values' 100Gi PVC). +func TestRunMQManifests_MatchesShipped(t *testing.T) { + want, err := os.ReadFile("../../deployments/nats/jetstream.yaml") + require.NoError(t, err) + var out, errOut bytes.Buffer + require.Equal(t, 0, runMQ([]string{"manifests", "--partitions", "4", "--shards", "8"}, &out, &errOut), errOut.String()) + assert.Equal(t, string(want), out.String(), "deployments/nats/jetstream.yaml is stale; regenerate it") +} + +func TestRunMQ_ExitCodes(t *testing.T) { + cases := map[string]struct { + args []string + code int + }{ + "no command": {nil, 2}, + "unknown command": {[]string{"frobnicate"}, 2}, + "help": {[]string{"help"}, 0}, + "manifests help": {[]string{"manifests", "-h"}, 0}, + "stray argument": {[]string{"manifests", "extra"}, 2}, + "unknown flag": {[]string{"manifests", "--nope"}, 2}, + "zero replicas": {[]string{"manifests", "--replicas", "0"}, 2}, + "bad prefix": {[]string{"manifests", "--prefix", "a.b"}, 1}, + "bad partitions": {[]string{"manifests", "--partitions", "-1"}, 1}, + "bad coord bucket": {[]string{"manifests", "--coord-bucket", "a.b"}, 1}, + "negative lease": {[]string{"manifests", "--dedupe-lease", "-1s"}, 2}, + "defaults generate": {[]string{"manifests"}, 0}, + "bad file store": {[]string{"manifests", "--file-store", "100GB"}, 2}, + "zero file store": {[]string{"manifests", "--file-store", "0"}, 2}, + "bad partition cap": {[]string{"manifests", "--partition-max-bytes", "lots"}, 2}, + "over the store": {[]string{"manifests", "--partitions", "6"}, 1}, + "resized to fit": {[]string{"manifests", "--partitions", "6", "--partition-max-bytes", "10Gi"}, 0}, + "zero timeout": {[]string{"manifests", "--publish-timeout", "0s"}, 2}, + "bad consumer": {[]string{"manifests", "--ingest-consumer", "a.b"}, 1}, + "bad history": {[]string{"manifests", "--history-stream", "A B"}, 1}, + "bad js domain": {[]string{"permissions", "--js-domain", "a.b"}, 1}, + } + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + var out, errOut bytes.Buffer + assert.Equal(t, tc.code, runMQ(tc.args, &out, &errOut), errOut.String()) + }) + } +} + +func TestRunMQManifests_NamesTheLeaseBucket(t *testing.T) { + var out, errOut bytes.Buffer + require.Equal(t, 0, runMQ([]string{"manifests", "--prefix", "acme", "--coord-bucket", "acme_leases"}, &out, &errOut), errOut.String()) + assert.Contains(t, out.String(), "kind: KeyValue\nmetadata:\n name: acme-coord\nspec:\n bucket: acme_leases\n") +} + +// A lease past the shipped 2m window's reach widens every partition's window +// to cover it (90s + 90s + 1s), so the manifests pass the boot check. +func TestRunMQManifests_CoversTheDedupeLease(t *testing.T) { + var out, errOut bytes.Buffer + require.Equal(t, 0, runMQ([]string{"manifests", "--partitions", "1", "--dedupe-lease", "90s"}, &out, &errOut), errOut.String()) + assert.Contains(t, out.String(), "duplicateWindow: 3m1s\n") +} + +// The permissions name every shard's durable, and match what the verifier's +// test holds the shipped values to. +func TestRunMQPermissions(t *testing.T) { + var out, errOut bytes.Buffer + require.Equal(t, 0, runMQ([]string{"permissions", "--shards", "3", "--prefix", "acme"}, &out, &errOut), errOut.String()) + for _, want := range []string{"$JS.API.CONSUMER.MSG.NEXT.*.wh-ingest-2", "$JS.API.CONSUMER.UNPIN.*.wh-ingest-0", "$JS.API.CONSUMER.RESET.*.wh-ingest-1", "acme.hist.>", "_INBOX_acme.>"} { + assert.Contains(t, out.String(), want) + } + assert.NotContains(t, out.String(), "wh-ingest-3") + out.Reset() + errOut.Reset() + assert.Equal(t, 2, runMQ([]string{"permissions", "--shards", "0"}, &out, &errOut)) + assert.Contains(t, errOut.String(), "--shards must be at least 1") + errOut.Reset() + assert.Equal(t, 1, runMQ([]string{"permissions", "--shards", "257"}, &out, &errOut)) + assert.Contains(t, errOut.String(), "shards must be from 1 to 256") +} + +// The names and publish timeout the config takes are flags too, and the +// partitions' duplicate window covers every attempt at that timeout +// (3 × 1m + 2 × 250ms). +func TestRunMQManifests_NamesAndPublishTimeout(t *testing.T) { + var out, errOut bytes.Buffer + require.Equal(t, 0, runMQ([]string{"manifests", "--partitions", "1", "--shards", "1", "--ingest-consumer", "acme-ingest", "--history-stream", "ACME_REPLAY", "--publish-timeout", "1m"}, &out, &errOut), errOut.String()) + for _, want := range []string{ + "durableName: acme-ingest-0\n", "name: ACME_REPLAY\n", "duplicateWindow: 3m0.5s\n", + "--ingest-consumer acme-ingest --history-stream ACME_REPLAY --publish-timeout 1m0s", + } { + assert.Contains(t, out.String(), want) + } +} + +// Under a JetStream domain the API and KV subjects come in the domain's form +// as well as the plain one. +func TestRunMQPermissions_JSDomain(t *testing.T) { + var out, errOut bytes.Buffer + require.Equal(t, 0, runMQ([]string{"permissions", "--shards", "1", "--js-domain", "hub"}, &out, &errOut), errOut.String()) + for _, want := range []string{ + "$JS.hub.API.CONSUMER.MSG.NEXT.*.wh-ingest-0", "$JS.API.CONSUMER.MSG.NEXT.*.wh-ingest-0", + "$JS.hub.API.$KV.wh_coord.lease.>", "$JS.hub.API.DIRECT.GET.KV_wh_coord.$KV.wh_coord.lease.>", "$JS.hub.API.STREAM.CREATE.>", + "JetStream domain hub", + } { + assert.Contains(t, out.String(), want) + } +} diff --git a/config.yaml b/config.yaml index 964f37462..e7f82ac40 100644 --- a/config.yaml +++ b/config.yaml @@ -9,11 +9,12 @@ data_dir: ./data # The work this process runs; every role by default. A split (one Deployment -# per role) needs a shared mq.backend and cache.backend, and boot refuses one -# on the in-process backends. +# per role) needs mq.backend: nats and, wherever ingest runs, +# coord.backend: nats (there is no sweeper under nats); separate api and ingest processes also need +# cache.backend: redis. Boot refuses a split the backends cannot serve. roles: [api, ingest, sweeper] -# Names this process: logged at boot today, a lease's holder once a shared -# coord.backend exists. Empty means -<8 hex>, fresh at every boot. +# Names this process: logged at boot, and a lease's holder under +# coord.backend: nats. Empty means -<8 hex>, fresh at every boot. instance_id: "" server: @@ -51,10 +52,18 @@ clickhouse: password: "" max_total_conns: 0 # ceiling on open native connections across pools; 0 = none -# Each layer's implementation, chosen at boot. The in-process backend is -# each layer's default. +# Each layer's implementation, chosen at boot. Every layer defaults to its +# in-process backend; mq and coord also take nats, cache redis, dedupe dynamodb. mq: backend: embedded # NATS JetStream under /nats + # backend: nats reads this block instead: the operator's NATS JetStream + # (see the deployment guide's "External NATS" section). + # nats: + # urls: ["nats://localhost:4222"] + # user: wavehouse + # password_file: ./nats-password + # partitions: 4 + # shards: 8 # must match the generated manifests (default 32) dedupe: backend: pebble # Pebble under /pebble; or dynamodb (below) lease: 30s # how long a claimed id stays pending; at most 59s with the embedded mq (lease + ceil(lease) + 1s within its 2m duplicate window) @@ -68,7 +77,11 @@ dedupe: # retry_mode: standard # or adaptive # create_table: false # dynamodb-local only; refused without endpoint coord: - backend: local # leases (the sweeper's) held in this process + backend: local # leases held in this process + # backend: nats holds them in a KV bucket on mq.nats's connection instead, + # and mq.backend: nats requires it in a process running ingest. + # nats: + # bucket: wh_coord # empty = _coord # The query-result cache: local (in-process, sized by l1_max_cost) or redis # (one Redis-compatible server shared by every instance; see the redis block diff --git a/deployments/nats/jetstream.yaml b/deployments/nats/jetstream.yaml new file mode 100644 index 000000000..7db463ea9 --- /dev/null +++ b/deployments/nats/jetstream.yaml @@ -0,0 +1,772 @@ +# WaveHouse's JetStream topology as nack (jetstream.nats.io/v1beta2) resources: +# 4 ingest partition(s) with work-queue retention, each split into 8 shards +# with a pinned-client durable per shard (wh-ingest-), and republishing every +# row to the WH_HISTORY history stream; the dead-letter stream; and the wh_coord KV +# bucket that coord.backend=nats holds its leases in. +# Generated by: wavehouse mq manifests --partitions 4 --shards 8 --prefix wh --replicas 3 --file-store 100Gi +# Sized for a JetStream file store (max_file_store) of 100Gi on every server: +# the streams' maxBytes reserve 75Gi of it, and JetStream refuses a stream that +# does not fit. The NATS Helm chart sets max_file_store to +# config.jetstream.fileStore.maxSize, else to the JetStream PVC's size. +# Sizes (maxBytes, maxAge, maxMsgsPerSubject) are starting points to tune. +# Set the history's maxAge (15m) to at least the longest +# stream.gap_window_minutes among the tenants served. +# WaveHouse publishes nothing until all of it exists, so apply order is free: +# a partition keeps every row until the ingest worker acks it, even while its +# durable is missing, and the history is best effort (a row republished while +# it is unavailable reaches ClickHouse but not SSE replay). +apiVersion: jetstream.nats.io/v1beta2 +kind: Stream +metadata: + name: wh-ingest-0 +spec: + name: WH_INGEST_0 + subjects: + - wh.ingest.0.> + retention: workqueue + discard: new + discardPerSubject: true + maxBytes: 16106127360 + maxMsgsPerSubject: 1000000 + storage: file + replicas: 3 + duplicateWindow: 2m + denyPurge: true + denyDelete: true + metadata: + wavehouse.dev/partition: "0" + wavehouse.dev/partitions: "4" + wavehouse.dev/shards: "8" + preventDelete: true + republish: + source: wh.ingest.0.*.> + destination: wh.hist.> +--- +apiVersion: jetstream.nats.io/v1beta2 +kind: Consumer +metadata: + name: wh-ingest-0-0 +spec: + streamName: WH_INGEST_0 + durableName: wh-ingest-0 + deliverPolicy: all + ackPolicy: explicit + ackWait: 1m + maxDeliver: -1 + maxAckPending: 10000 + filterSubject: wh.ingest.0.0.> + priorityPolicy: pinned_client + priorityGroups: + - wavehouse + pinnedTtl: 10s + preventDelete: true +--- +apiVersion: jetstream.nats.io/v1beta2 +kind: Consumer +metadata: + name: wh-ingest-0-1 +spec: + streamName: WH_INGEST_0 + durableName: wh-ingest-1 + deliverPolicy: all + ackPolicy: explicit + ackWait: 1m + maxDeliver: -1 + maxAckPending: 10000 + filterSubject: wh.ingest.0.1.> + priorityPolicy: pinned_client + priorityGroups: + - wavehouse + pinnedTtl: 10s + preventDelete: true +--- +apiVersion: jetstream.nats.io/v1beta2 +kind: Consumer +metadata: + name: wh-ingest-0-2 +spec: + streamName: WH_INGEST_0 + durableName: wh-ingest-2 + deliverPolicy: all + ackPolicy: explicit + ackWait: 1m + maxDeliver: -1 + maxAckPending: 10000 + filterSubject: wh.ingest.0.2.> + priorityPolicy: pinned_client + priorityGroups: + - wavehouse + pinnedTtl: 10s + preventDelete: true +--- +apiVersion: jetstream.nats.io/v1beta2 +kind: Consumer +metadata: + name: wh-ingest-0-3 +spec: + streamName: WH_INGEST_0 + durableName: wh-ingest-3 + deliverPolicy: all + ackPolicy: explicit + ackWait: 1m + maxDeliver: -1 + maxAckPending: 10000 + filterSubject: wh.ingest.0.3.> + priorityPolicy: pinned_client + priorityGroups: + - wavehouse + pinnedTtl: 10s + preventDelete: true +--- +apiVersion: jetstream.nats.io/v1beta2 +kind: Consumer +metadata: + name: wh-ingest-0-4 +spec: + streamName: WH_INGEST_0 + durableName: wh-ingest-4 + deliverPolicy: all + ackPolicy: explicit + ackWait: 1m + maxDeliver: -1 + maxAckPending: 10000 + filterSubject: wh.ingest.0.4.> + priorityPolicy: pinned_client + priorityGroups: + - wavehouse + pinnedTtl: 10s + preventDelete: true +--- +apiVersion: jetstream.nats.io/v1beta2 +kind: Consumer +metadata: + name: wh-ingest-0-5 +spec: + streamName: WH_INGEST_0 + durableName: wh-ingest-5 + deliverPolicy: all + ackPolicy: explicit + ackWait: 1m + maxDeliver: -1 + maxAckPending: 10000 + filterSubject: wh.ingest.0.5.> + priorityPolicy: pinned_client + priorityGroups: + - wavehouse + pinnedTtl: 10s + preventDelete: true +--- +apiVersion: jetstream.nats.io/v1beta2 +kind: Consumer +metadata: + name: wh-ingest-0-6 +spec: + streamName: WH_INGEST_0 + durableName: wh-ingest-6 + deliverPolicy: all + ackPolicy: explicit + ackWait: 1m + maxDeliver: -1 + maxAckPending: 10000 + filterSubject: wh.ingest.0.6.> + priorityPolicy: pinned_client + priorityGroups: + - wavehouse + pinnedTtl: 10s + preventDelete: true +--- +apiVersion: jetstream.nats.io/v1beta2 +kind: Consumer +metadata: + name: wh-ingest-0-7 +spec: + streamName: WH_INGEST_0 + durableName: wh-ingest-7 + deliverPolicy: all + ackPolicy: explicit + ackWait: 1m + maxDeliver: -1 + maxAckPending: 10000 + filterSubject: wh.ingest.0.7.> + priorityPolicy: pinned_client + priorityGroups: + - wavehouse + pinnedTtl: 10s + preventDelete: true +--- +apiVersion: jetstream.nats.io/v1beta2 +kind: Stream +metadata: + name: wh-ingest-1 +spec: + name: WH_INGEST_1 + subjects: + - wh.ingest.1.> + retention: workqueue + discard: new + discardPerSubject: true + maxBytes: 16106127360 + maxMsgsPerSubject: 1000000 + storage: file + replicas: 3 + duplicateWindow: 2m + denyPurge: true + denyDelete: true + metadata: + wavehouse.dev/partition: "1" + wavehouse.dev/partitions: "4" + wavehouse.dev/shards: "8" + preventDelete: true + republish: + source: wh.ingest.1.*.> + destination: wh.hist.> +--- +apiVersion: jetstream.nats.io/v1beta2 +kind: Consumer +metadata: + name: wh-ingest-1-0 +spec: + streamName: WH_INGEST_1 + durableName: wh-ingest-0 + deliverPolicy: all + ackPolicy: explicit + ackWait: 1m + maxDeliver: -1 + maxAckPending: 10000 + filterSubject: wh.ingest.1.0.> + priorityPolicy: pinned_client + priorityGroups: + - wavehouse + pinnedTtl: 10s + preventDelete: true +--- +apiVersion: jetstream.nats.io/v1beta2 +kind: Consumer +metadata: + name: wh-ingest-1-1 +spec: + streamName: WH_INGEST_1 + durableName: wh-ingest-1 + deliverPolicy: all + ackPolicy: explicit + ackWait: 1m + maxDeliver: -1 + maxAckPending: 10000 + filterSubject: wh.ingest.1.1.> + priorityPolicy: pinned_client + priorityGroups: + - wavehouse + pinnedTtl: 10s + preventDelete: true +--- +apiVersion: jetstream.nats.io/v1beta2 +kind: Consumer +metadata: + name: wh-ingest-1-2 +spec: + streamName: WH_INGEST_1 + durableName: wh-ingest-2 + deliverPolicy: all + ackPolicy: explicit + ackWait: 1m + maxDeliver: -1 + maxAckPending: 10000 + filterSubject: wh.ingest.1.2.> + priorityPolicy: pinned_client + priorityGroups: + - wavehouse + pinnedTtl: 10s + preventDelete: true +--- +apiVersion: jetstream.nats.io/v1beta2 +kind: Consumer +metadata: + name: wh-ingest-1-3 +spec: + streamName: WH_INGEST_1 + durableName: wh-ingest-3 + deliverPolicy: all + ackPolicy: explicit + ackWait: 1m + maxDeliver: -1 + maxAckPending: 10000 + filterSubject: wh.ingest.1.3.> + priorityPolicy: pinned_client + priorityGroups: + - wavehouse + pinnedTtl: 10s + preventDelete: true +--- +apiVersion: jetstream.nats.io/v1beta2 +kind: Consumer +metadata: + name: wh-ingest-1-4 +spec: + streamName: WH_INGEST_1 + durableName: wh-ingest-4 + deliverPolicy: all + ackPolicy: explicit + ackWait: 1m + maxDeliver: -1 + maxAckPending: 10000 + filterSubject: wh.ingest.1.4.> + priorityPolicy: pinned_client + priorityGroups: + - wavehouse + pinnedTtl: 10s + preventDelete: true +--- +apiVersion: jetstream.nats.io/v1beta2 +kind: Consumer +metadata: + name: wh-ingest-1-5 +spec: + streamName: WH_INGEST_1 + durableName: wh-ingest-5 + deliverPolicy: all + ackPolicy: explicit + ackWait: 1m + maxDeliver: -1 + maxAckPending: 10000 + filterSubject: wh.ingest.1.5.> + priorityPolicy: pinned_client + priorityGroups: + - wavehouse + pinnedTtl: 10s + preventDelete: true +--- +apiVersion: jetstream.nats.io/v1beta2 +kind: Consumer +metadata: + name: wh-ingest-1-6 +spec: + streamName: WH_INGEST_1 + durableName: wh-ingest-6 + deliverPolicy: all + ackPolicy: explicit + ackWait: 1m + maxDeliver: -1 + maxAckPending: 10000 + filterSubject: wh.ingest.1.6.> + priorityPolicy: pinned_client + priorityGroups: + - wavehouse + pinnedTtl: 10s + preventDelete: true +--- +apiVersion: jetstream.nats.io/v1beta2 +kind: Consumer +metadata: + name: wh-ingest-1-7 +spec: + streamName: WH_INGEST_1 + durableName: wh-ingest-7 + deliverPolicy: all + ackPolicy: explicit + ackWait: 1m + maxDeliver: -1 + maxAckPending: 10000 + filterSubject: wh.ingest.1.7.> + priorityPolicy: pinned_client + priorityGroups: + - wavehouse + pinnedTtl: 10s + preventDelete: true +--- +apiVersion: jetstream.nats.io/v1beta2 +kind: Stream +metadata: + name: wh-ingest-2 +spec: + name: WH_INGEST_2 + subjects: + - wh.ingest.2.> + retention: workqueue + discard: new + discardPerSubject: true + maxBytes: 16106127360 + maxMsgsPerSubject: 1000000 + storage: file + replicas: 3 + duplicateWindow: 2m + denyPurge: true + denyDelete: true + metadata: + wavehouse.dev/partition: "2" + wavehouse.dev/partitions: "4" + wavehouse.dev/shards: "8" + preventDelete: true + republish: + source: wh.ingest.2.*.> + destination: wh.hist.> +--- +apiVersion: jetstream.nats.io/v1beta2 +kind: Consumer +metadata: + name: wh-ingest-2-0 +spec: + streamName: WH_INGEST_2 + durableName: wh-ingest-0 + deliverPolicy: all + ackPolicy: explicit + ackWait: 1m + maxDeliver: -1 + maxAckPending: 10000 + filterSubject: wh.ingest.2.0.> + priorityPolicy: pinned_client + priorityGroups: + - wavehouse + pinnedTtl: 10s + preventDelete: true +--- +apiVersion: jetstream.nats.io/v1beta2 +kind: Consumer +metadata: + name: wh-ingest-2-1 +spec: + streamName: WH_INGEST_2 + durableName: wh-ingest-1 + deliverPolicy: all + ackPolicy: explicit + ackWait: 1m + maxDeliver: -1 + maxAckPending: 10000 + filterSubject: wh.ingest.2.1.> + priorityPolicy: pinned_client + priorityGroups: + - wavehouse + pinnedTtl: 10s + preventDelete: true +--- +apiVersion: jetstream.nats.io/v1beta2 +kind: Consumer +metadata: + name: wh-ingest-2-2 +spec: + streamName: WH_INGEST_2 + durableName: wh-ingest-2 + deliverPolicy: all + ackPolicy: explicit + ackWait: 1m + maxDeliver: -1 + maxAckPending: 10000 + filterSubject: wh.ingest.2.2.> + priorityPolicy: pinned_client + priorityGroups: + - wavehouse + pinnedTtl: 10s + preventDelete: true +--- +apiVersion: jetstream.nats.io/v1beta2 +kind: Consumer +metadata: + name: wh-ingest-2-3 +spec: + streamName: WH_INGEST_2 + durableName: wh-ingest-3 + deliverPolicy: all + ackPolicy: explicit + ackWait: 1m + maxDeliver: -1 + maxAckPending: 10000 + filterSubject: wh.ingest.2.3.> + priorityPolicy: pinned_client + priorityGroups: + - wavehouse + pinnedTtl: 10s + preventDelete: true +--- +apiVersion: jetstream.nats.io/v1beta2 +kind: Consumer +metadata: + name: wh-ingest-2-4 +spec: + streamName: WH_INGEST_2 + durableName: wh-ingest-4 + deliverPolicy: all + ackPolicy: explicit + ackWait: 1m + maxDeliver: -1 + maxAckPending: 10000 + filterSubject: wh.ingest.2.4.> + priorityPolicy: pinned_client + priorityGroups: + - wavehouse + pinnedTtl: 10s + preventDelete: true +--- +apiVersion: jetstream.nats.io/v1beta2 +kind: Consumer +metadata: + name: wh-ingest-2-5 +spec: + streamName: WH_INGEST_2 + durableName: wh-ingest-5 + deliverPolicy: all + ackPolicy: explicit + ackWait: 1m + maxDeliver: -1 + maxAckPending: 10000 + filterSubject: wh.ingest.2.5.> + priorityPolicy: pinned_client + priorityGroups: + - wavehouse + pinnedTtl: 10s + preventDelete: true +--- +apiVersion: jetstream.nats.io/v1beta2 +kind: Consumer +metadata: + name: wh-ingest-2-6 +spec: + streamName: WH_INGEST_2 + durableName: wh-ingest-6 + deliverPolicy: all + ackPolicy: explicit + ackWait: 1m + maxDeliver: -1 + maxAckPending: 10000 + filterSubject: wh.ingest.2.6.> + priorityPolicy: pinned_client + priorityGroups: + - wavehouse + pinnedTtl: 10s + preventDelete: true +--- +apiVersion: jetstream.nats.io/v1beta2 +kind: Consumer +metadata: + name: wh-ingest-2-7 +spec: + streamName: WH_INGEST_2 + durableName: wh-ingest-7 + deliverPolicy: all + ackPolicy: explicit + ackWait: 1m + maxDeliver: -1 + maxAckPending: 10000 + filterSubject: wh.ingest.2.7.> + priorityPolicy: pinned_client + priorityGroups: + - wavehouse + pinnedTtl: 10s + preventDelete: true +--- +apiVersion: jetstream.nats.io/v1beta2 +kind: Stream +metadata: + name: wh-ingest-3 +spec: + name: WH_INGEST_3 + subjects: + - wh.ingest.3.> + retention: workqueue + discard: new + discardPerSubject: true + maxBytes: 16106127360 + maxMsgsPerSubject: 1000000 + storage: file + replicas: 3 + duplicateWindow: 2m + denyPurge: true + denyDelete: true + metadata: + wavehouse.dev/partition: "3" + wavehouse.dev/partitions: "4" + wavehouse.dev/shards: "8" + preventDelete: true + republish: + source: wh.ingest.3.*.> + destination: wh.hist.> +--- +apiVersion: jetstream.nats.io/v1beta2 +kind: Consumer +metadata: + name: wh-ingest-3-0 +spec: + streamName: WH_INGEST_3 + durableName: wh-ingest-0 + deliverPolicy: all + ackPolicy: explicit + ackWait: 1m + maxDeliver: -1 + maxAckPending: 10000 + filterSubject: wh.ingest.3.0.> + priorityPolicy: pinned_client + priorityGroups: + - wavehouse + pinnedTtl: 10s + preventDelete: true +--- +apiVersion: jetstream.nats.io/v1beta2 +kind: Consumer +metadata: + name: wh-ingest-3-1 +spec: + streamName: WH_INGEST_3 + durableName: wh-ingest-1 + deliverPolicy: all + ackPolicy: explicit + ackWait: 1m + maxDeliver: -1 + maxAckPending: 10000 + filterSubject: wh.ingest.3.1.> + priorityPolicy: pinned_client + priorityGroups: + - wavehouse + pinnedTtl: 10s + preventDelete: true +--- +apiVersion: jetstream.nats.io/v1beta2 +kind: Consumer +metadata: + name: wh-ingest-3-2 +spec: + streamName: WH_INGEST_3 + durableName: wh-ingest-2 + deliverPolicy: all + ackPolicy: explicit + ackWait: 1m + maxDeliver: -1 + maxAckPending: 10000 + filterSubject: wh.ingest.3.2.> + priorityPolicy: pinned_client + priorityGroups: + - wavehouse + pinnedTtl: 10s + preventDelete: true +--- +apiVersion: jetstream.nats.io/v1beta2 +kind: Consumer +metadata: + name: wh-ingest-3-3 +spec: + streamName: WH_INGEST_3 + durableName: wh-ingest-3 + deliverPolicy: all + ackPolicy: explicit + ackWait: 1m + maxDeliver: -1 + maxAckPending: 10000 + filterSubject: wh.ingest.3.3.> + priorityPolicy: pinned_client + priorityGroups: + - wavehouse + pinnedTtl: 10s + preventDelete: true +--- +apiVersion: jetstream.nats.io/v1beta2 +kind: Consumer +metadata: + name: wh-ingest-3-4 +spec: + streamName: WH_INGEST_3 + durableName: wh-ingest-4 + deliverPolicy: all + ackPolicy: explicit + ackWait: 1m + maxDeliver: -1 + maxAckPending: 10000 + filterSubject: wh.ingest.3.4.> + priorityPolicy: pinned_client + priorityGroups: + - wavehouse + pinnedTtl: 10s + preventDelete: true +--- +apiVersion: jetstream.nats.io/v1beta2 +kind: Consumer +metadata: + name: wh-ingest-3-5 +spec: + streamName: WH_INGEST_3 + durableName: wh-ingest-5 + deliverPolicy: all + ackPolicy: explicit + ackWait: 1m + maxDeliver: -1 + maxAckPending: 10000 + filterSubject: wh.ingest.3.5.> + priorityPolicy: pinned_client + priorityGroups: + - wavehouse + pinnedTtl: 10s + preventDelete: true +--- +apiVersion: jetstream.nats.io/v1beta2 +kind: Consumer +metadata: + name: wh-ingest-3-6 +spec: + streamName: WH_INGEST_3 + durableName: wh-ingest-6 + deliverPolicy: all + ackPolicy: explicit + ackWait: 1m + maxDeliver: -1 + maxAckPending: 10000 + filterSubject: wh.ingest.3.6.> + priorityPolicy: pinned_client + priorityGroups: + - wavehouse + pinnedTtl: 10s + preventDelete: true +--- +apiVersion: jetstream.nats.io/v1beta2 +kind: Consumer +metadata: + name: wh-ingest-3-7 +spec: + streamName: WH_INGEST_3 + durableName: wh-ingest-7 + deliverPolicy: all + ackPolicy: explicit + ackWait: 1m + maxDeliver: -1 + maxAckPending: 10000 + filterSubject: wh.ingest.3.7.> + priorityPolicy: pinned_client + priorityGroups: + - wavehouse + pinnedTtl: 10s + preventDelete: true +--- +apiVersion: jetstream.nats.io/v1beta2 +kind: Stream +metadata: + name: wh-history +spec: + name: WH_HISTORY + subjects: + - wh.hist.> + retention: limits + discard: old + maxBytes: 10737418240 + maxAge: 15m + storage: file + replicas: 3 +--- +apiVersion: jetstream.nats.io/v1beta2 +kind: Stream +metadata: + name: wh-dlq +spec: + name: WH_DLQ + subjects: + - wh.dlq.> + retention: limits + discard: old + maxBytes: 5368709120 + maxMsgsPerSubject: 100000 + storage: file + replicas: 3 +--- +apiVersion: jetstream.nats.io/v1beta2 +kind: KeyValue +metadata: + name: wh-coord +spec: + bucket: wh_coord + history: 1 + storage: file + replicas: 3 diff --git a/deployments/nats/values.yaml b/deployments/nats/values.yaml new file mode 100644 index 000000000..a8d907584 --- /dev/null +++ b/deployments/nats/values.yaml @@ -0,0 +1,125 @@ +# Values for the NATS Helm chart (https://github.com/nats-io/k8s, chart +# nats/nats 1.x) that WaveHouse's mq.backend: nats runs against: JetStream on +# file storage, and an account holding two users — +# nack the JetStream controller that applies jetstream.yaml (full access) +# wavehouse WaveHouse itself, with exactly the permissions it needs: it can +# publish, read and consume, unpin and reset its shard durables, +# read and write the lease keys in the coord bucket, and cannot +# create, change, purge or delete a stream or bucket, create a +# durable on an ingest partition, or publish to the history. +# Passwords come from Secrets through the container env; `<< $VAR >>` is the +# chart's syntax for an unquoted NATS config variable. +# +# The wavehouse user's permissions are for the default subject prefix (wh), +# history stream (WH_HISTORY), shard durables (wh-ingest-) and lease +# bucket (wh_coord), with 8 shards per partition as in jetstream.yaml: they +# name every shard's durable, so regenerate them with +# `wavehouse mq permissions --shards ` (add --js-domain when +# mq.nats.js_domain is set). A test keeps them in step with WaveHouse, and the +# conformance tests connect with them verbatim. WaveHouse refuses to boot when +# they do not let it pull every shard's durable. +# +# The chart sets each server's max_file_store to fileStore.maxSize, else to +# the PVC's size: 100Gi here. Every server holds a replica of every stream +# and reserves its maxBytes against that, and jetstream.yaml is sized to +# reserve 75Gi of it. Resize the two together: `wavehouse mq manifests +# --file-store ` sizes the streams for another volume. +config: + cluster: + enabled: true + replicas: 3 + jetstream: + enabled: true + fileStore: + enabled: true + pvc: + size: 100Gi + merge: + accounts: + WAVEHOUSE: + jetstream: enabled + users: + - user: nack + password: << $NACK_PASSWORD >> + - user: wavehouse + password: << $WAVEHOUSE_PASSWORD >> + permissions: + publish: + allow: + - wh.ingest.> + - wh.dlq.> + - $JS.API.INFO + - $JS.API.STREAM.NAMES + - $JS.API.STREAM.INFO.* + - $JS.API.CONSUMER.INFO.*.* + - $JS.API.CONSUMER.NAMES.* + - $JS.API.CONSUMER.MSG.NEXT.*.wh-ingest-0 + - $JS.API.CONSUMER.MSG.NEXT.*.wh-ingest-1 + - $JS.API.CONSUMER.MSG.NEXT.*.wh-ingest-2 + - $JS.API.CONSUMER.MSG.NEXT.*.wh-ingest-3 + - $JS.API.CONSUMER.MSG.NEXT.*.wh-ingest-4 + - $JS.API.CONSUMER.MSG.NEXT.*.wh-ingest-5 + - $JS.API.CONSUMER.MSG.NEXT.*.wh-ingest-6 + - $JS.API.CONSUMER.MSG.NEXT.*.wh-ingest-7 + - $JS.API.CONSUMER.UNPIN.*.wh-ingest-0 + - $JS.API.CONSUMER.UNPIN.*.wh-ingest-1 + - $JS.API.CONSUMER.UNPIN.*.wh-ingest-2 + - $JS.API.CONSUMER.UNPIN.*.wh-ingest-3 + - $JS.API.CONSUMER.UNPIN.*.wh-ingest-4 + - $JS.API.CONSUMER.UNPIN.*.wh-ingest-5 + - $JS.API.CONSUMER.UNPIN.*.wh-ingest-6 + - $JS.API.CONSUMER.UNPIN.*.wh-ingest-7 + - $JS.API.CONSUMER.RESET.*.wh-ingest-0 + - $JS.API.CONSUMER.RESET.*.wh-ingest-1 + - $JS.API.CONSUMER.RESET.*.wh-ingest-2 + - $JS.API.CONSUMER.RESET.*.wh-ingest-3 + - $JS.API.CONSUMER.RESET.*.wh-ingest-4 + - $JS.API.CONSUMER.RESET.*.wh-ingest-5 + - $JS.API.CONSUMER.RESET.*.wh-ingest-6 + - $JS.API.CONSUMER.RESET.*.wh-ingest-7 + - $JS.ACK.*.wh-ingest-0.> + - $JS.ACK.*.wh-ingest-1.> + - $JS.ACK.*.wh-ingest-2.> + - $JS.ACK.*.wh-ingest-3.> + - $JS.ACK.*.wh-ingest-4.> + - $JS.ACK.*.wh-ingest-5.> + - $JS.ACK.*.wh-ingest-6.> + - $JS.ACK.*.wh-ingest-7.> + - $JS.ACK.*.*.*.wh-ingest-0.> + - $JS.ACK.*.*.*.wh-ingest-1.> + - $JS.ACK.*.*.*.wh-ingest-2.> + - $JS.ACK.*.*.*.wh-ingest-3.> + - $JS.ACK.*.*.*.wh-ingest-4.> + - $JS.ACK.*.*.*.wh-ingest-5.> + - $JS.ACK.*.*.*.wh-ingest-6.> + - $JS.ACK.*.*.*.wh-ingest-7.> + - $JS.API.CONSUMER.CREATE.WH_HISTORY.> + - $JS.API.CONSUMER.MSG.NEXT.WH_HISTORY.> + - $JS.API.CONSUMER.DELETE.WH_HISTORY.> + - $KV.wh_coord.lease.> + - $JS.API.DIRECT.GET.KV_wh_coord.$KV.wh_coord.lease.> + deny: + - wh.hist.> + - $JS.API.STREAM.CREATE.> + - $JS.API.STREAM.UPDATE.> + - $JS.API.STREAM.DELETE.> + - $JS.API.STREAM.PURGE.> + - $JS.API.CONSUMER.DURABLE.CREATE.> + subscribe: + allow: + - _INBOX_wh.> + +container: + image: + tag: 2.14.6-alpine # the server line WaveHouse embeds and tests against + env: + NACK_PASSWORD: + valueFrom: + secretKeyRef: + name: nats-users + key: nack + WAVEHOUSE_PASSWORD: + valueFrom: + secretKeyRef: + name: nats-users + key: wavehouse diff --git a/docs/src/content/docs/api.md b/docs/src/content/docs/api.md index 3e8f6815f..1af6ee209 100644 --- a/docs/src/content/docs/api.md +++ b/docs/src/content/docs/api.md @@ -209,6 +209,22 @@ Where those values come from depends on how the binary was built: --- +### A process without the `api` role — the ops listener + +A process whose [`roles`](/configuration#process-roles) leave out `api` (an ingest worker, possible with [`mq.backend: nats`](/deployment#external-nats)) serves only these routes on `server.port`: + +| Route | Notes | +| ----- | ----- | +| `GET /livez` (and `/healthz`, `/health`) | `200` once booted. It does not wait for schema discovery, which only the API runs. | +| `GET /readyz` (and `/ready`) | In a process running `ingest`, `200` when a ClickHouse pool answers, as above. | +| `GET /version` | As above. | +| The metrics path | When `prometheus.port` is `0`. | +| `POST /v1/ops/settings/reload` | As [below](#post-v1opssettingsreload--reload-settings-directory), but it accepts only the [operator key](#authentication): no token verifier runs without the `api` role, so an admin token is `401`. | + +Every other route answers `404`, including every tenant route. Under `/v1/ops`, the operator-key check comes first, so a request there gets `403` without a credential, and `401` for a bearer token. + +--- + ### `POST /v1/ingest?table={table}` — Ingest Data Accepts a single flat JSON object, a JSON array of objects, or a newline-delimited JSON (NDJSON) batch, validates each record against the ClickHouse schema for `{table}`, and publishes it to the message queue. Returns immediately — ClickHouse insertion happens asynchronously via the batch consumer. @@ -298,8 +314,8 @@ The body is a **flat JSON object** whose keys must match column names in the tar | 503 | `{"error":"dedupe store unavailable"}` | Dedupe is on and its store cannot answer now: it is not open (for example, it failed to open on a reload), or a DynamoDB table is throttling, timing out or unreachable; `Retry-After: 5`. Nothing was published, so the retry is safe | | 503 | `{"error":"schema not loaded yet"}` | The tenant's first schema discovery has not succeeded yet (its ClickHouse unreachable, or [no pool for it](/settings-directory#clickhouse)), so whether the table exists is not known; `Retry-After: 5`. Decided before the body is read | | 500 | `{"error":"publish failed"}` | Message queue error whose outcome is unknown, other than a full queue or an unreachable broker (below): the event may have been stored. With dedupe on, the record's id is left to lapse with the dedupe lease ([`dedupe.lease`](/configuration#dedupe), 30 seconds by default) rather than given back: a retry inside the lease answers the in-flight `503`, and one after it is published under the same idempotency key, which the queue drops if the first copy was stored. The queue's duplicate window (two minutes) covers up to ~2×lease plus a margin, not just the lease itself, so a retry timed off `Retry-After` anywhere in this flow stores no second copy; a much later one is stored again. | -| 503 | `{"error":"service unavailable"}` | The tenant's ingest queue is full (backpressure, for that tenant alone) or not open (see [Message Queue](/settings-directory#message-queue)). Response includes `Retry-After: 30` header. With dedupe on, the record's id is given back, so the retry is published rather than reported as a duplicate. | -| 503 | `{"error":"service unavailable"}` | The message queue could not be reached or did not answer in time (`mq.ErrUnavailable`, a transient broker failure, not a full queue) — reserved for an external broker ([#613](https://github.com/Wave-RF/WaveHouse/issues/613)): the embedded broker never reports this, and its publish failures are the `500` above. As for the `500`, the record's id is left to lapse rather than given back, so a retry cannot land as a second copy; `Retry-After` is that lease, rounded up to whole seconds, when dedupe was on for the record, else the flat `Retry-After: 5`. | +| 503 | `{"error":"service unavailable"}` | The tenant's ingest queue is full (backpressure, for that tenant alone) or not open (see [Message Queue](/settings-directory#message-queue)). Under [`mq.backend: nats`](/deployment#external-nats), the table's partition stream is full, which refuses every table in it, or the tenant's table holds as many unwritten rows as the stream allows one subject. Response includes `Retry-After: 30` header. With dedupe on, the record's id is given back, so the retry is published rather than reported as a duplicate. | +| 503 | `{"error":"service unavailable"}` | The message queue could not be reached or did not answer in time (`mq.ErrUnavailable`, a transient broker failure, not a full queue). Only under [`mq.backend: nats`](/deployment#external-nats), including a partition stream the operator deleted; the embedded broker never reports this, and its publish failures are the `500` above. As for the `500`, the record's id is left to lapse rather than given back, so a retry cannot land as a second copy; `Retry-After` is that lease, rounded up to whole seconds, when dedupe was on for the record, else the flat `Retry-After: 5`. | | 503 | `{"error":"a request with the same dedupe id is in flight"}` | Dedupe is on and another request carrying the same id is still being published — usually a client's timeout-retry racing its own original. Its outcome decides whether this record is a duplicate, so retry after the `Retry-After` header (the dedupe lease, [`dedupe.lease`](/configuration#dedupe), 30 seconds by default). | | 503 | `{"error":"token verifier not ready: the tenant's JWKS has not been fetched yet"}` | A token was supplied, with no valid operator key, while the tenant's JWKS has not been fetched yet; refused before any policy runs, with a `Retry-After: 30` header — see [Authentication](#authentication) | @@ -412,7 +428,7 @@ A `200` is returned whenever the body was read and the records were processed | 415 | `{"error":"no Content-Type: ingest requires one of application/json, application/x-ndjson, …"}` (declared variant: `Content-Type "text/plain": ingest requires one of …` — see the note above on how declarations are echoed; conflicting variant: `conflicting Content-Type declarations "application/json", "application/x-ndjson": ingest reads one format per request, and requires one of …`) | The request declared no `Content-Type`, one whose media type is unsupported or does not parse, a comma-bearing value that does not parse as a single media type, or repeated header lines that disagree — different formats, or one supported and one not. Checked before the body is parsed | | 500 | `{"error":"publish failed"}` / `{"error":"dedupe failed"}` | Message-queue or dedup-backend failure mid-batch, other than a full queue or an unreachable broker (below). After a publish failure the records before it keep their ids, so a whole-batch retry reports those as duplicates; the failing record's id is left to lapse as on the single-object path, and the rest of its window's ids are given back | | 503 | `{"error":"service unavailable"}` | The tenant's ingest queue is full (backpressure) or not open, mid-batch; includes `Retry-After: 30`. The records before the refused one keep their ids, and its id and the rest of its window's are given back | -| 503 | `{"error":"service unavailable"}` | The message queue could not be reached or did not answer in time (`mq.ErrUnavailable`), mid-batch — reserved for an external broker ([#613](https://github.com/Wave-RF/WaveHouse/issues/613)): the embedded broker never reports this, and its publish failures are the `500` above. As for the `500`, the failing record's id is left to lapse rather than given back — so `Retry-After` is that record's dedupe lease, rounded up to whole seconds, when it was deduped; a record published un-deduped has no lapsing claim to wait out, so `Retry-After: 5` | +| 503 | `{"error":"service unavailable"}` | The message queue could not be reached or did not answer in time (`mq.ErrUnavailable`), mid-batch. Only under [`mq.backend: nats`](/deployment#external-nats); the embedded broker never reports this, and its publish failures are the `500` above. As for the `500`, the failing record's id is left to lapse rather than given back — so `Retry-After` is that record's dedupe lease, rounded up to whole seconds, when it was deduped; a record published un-deduped has no lapsing claim to wait out, so `Retry-After: 5` | | 503 | `{"error":"dedupe store unavailable"}` | Dedupe is on and its store cannot answer now; `Retry-After: 5`. Nothing in the window being reserved was published; the windows before it were, and keep their ids | | 503 | `{"error":"a request with the same dedupe id is in flight"}` | A record's dedupe id is held by another request still being published; includes `Retry-After` (the dedupe lease, [`dedupe.lease`](/configuration#dedupe), 30 seconds by default). Nothing in that record's window was published; the windows before it were | | 503 | `{"error":"token verifier not ready: the tenant's JWKS has not been fetched yet"}` | A token was supplied, with no valid operator key, while the tenant's JWKS has not been fetched yet; refused before any policy runs, with a `Retry-After: 30` header — see [Authentication](#authentication) | @@ -778,7 +794,7 @@ Triggers an immediate re-discovery of the `?tenant=`'s ClickHouse table schemas #### `GET /v1/ops/dlq/stats` — DLQ Statistics -Returns per-table message counts in one tenant's Dead Letter Queue: the [tenant](/deployment#the-nested-settings-directory) an optional `?tenant=` names, the default tenant `0` without it, which is the whole settings directory unless it is nested. The tenant is looked up in the message queue, not the settings, so a tenant whose folder was rejected or removed is read like one being served, since its queue is kept (nothing deletes it). The query string is parsed strictly, as on the other admin reads. Admin-only, like the rest of this section. Whether a poison row lands here is the settings directory's [`dlq.enabled`](/settings-directory#dead-letter-queue) switch (global or per table); a tenant's dead-letter stream is opened when the tenant is first served, and this endpoint always exists. Until a row has been parked, the endpoint returns `200` with `{"tables":{},"total":0}`. +Returns per-table message counts in one tenant's Dead Letter Queue: the [tenant](/deployment#the-nested-settings-directory) an optional `?tenant=` names, the default tenant `0` without it, which is the whole settings directory unless it is nested. The tenant is looked up in the message queue, not the settings, so a tenant whose folder was rejected or removed is read like one being served, since its queue is kept (nothing deletes it). The query string is parsed strictly, as on the other admin reads. Admin-only, like the rest of this section. Whether a poison row lands here is the settings directory's [`dlq.enabled`](/settings-directory#dead-letter-queue) switch (global or per table); a tenant's dead-letter stream is opened when the tenant is first served, and this endpoint always exists. Until a row has been parked, the endpoint returns `200` with `{"tables":{},"total":0}`. Under [`mq.backend: nats`](/deployment#external-nats) every tenant's rows are counted on one shared dead-letter stream, so any tenant id reads `200`, with zeros when it has never parked a row, and the `404` below does not occur. **Error responses:** @@ -787,7 +803,7 @@ Returns per-table message counts in one tenant's Dead Letter Queue: the [tenant] | 401 | `{"error":"invalid token"}` / `{"error":"token expired"}` | A present-but-invalid/expired token was supplied and denied (the gate surfaces the token reason) | | 400 | `{"error":"invalid query string: …"}` / `{"error":"invalid ?tenant: …"}` | The query string does not parse (`?tenant=acme;x=1`, a bad `%` escape), or `tenant` is empty, repeated, or not a tenant id | | 403 | `{"error":"forbidden"}` | Caller's role is not the policy `admin_role` (`"admin"` by default) | -| 404 | `{"error":"no dead-letter queue for tenant: "}` | The tenant has no dead-letter queue: it has never been served on this data directory, its queue could not be opened (see [Message Queue](/settings-directory#message-queue)), or the id names no tenant | +| 404 | `{"error":"no dead-letter queue for tenant: "}` | Embedded queue only. The tenant has no dead-letter queue: it has never been served on this data directory, its queue could not be opened (see [Message Queue](/settings-directory#message-queue)), or the id names no tenant | | 500 | `{"error":"stream info failed"}` | NATS JetStream stream-info lookup failed | | 503 | `{"error":"token verifier not ready: the tenant's JWKS has not been fetched yet"}` | A token was supplied, with no valid operator key, while tenant `0`'s JWKS has not been fetched yet (the ops tree verifies as tenant `0`); refused before any policy runs, with a `Retry-After: 30` header — see [Authentication](#authentication) | @@ -911,6 +927,8 @@ Three values, where the envelope above has four: this is the frame a role restri When ClickHouse **rejects** a batch insert (a value it cannot parse, a type mismatch, a table or column it does not have), the worker re-inserts the batch row by row: rows that succeed are acked, and only the rows ClickHouse rejects again are published to the tenant's own DLQ NATS stream (`DLQ_{tenant}`) under subjects `dlq.{tenant}.{table}` (the tenant the row was ingested under; `0` for a settings directory that holds the four files). This prevents infinite retry loops — those messages are ACKed from the main stream and moved to the DLQ for inspection. A ClickHouse that **cannot take** the insert — down, unreachable, timing out, overloaded, read-only, or refusing WaveHouse's credentials — never sends a row here: the batch stays in the tenant's ingest queue and is retried with backoff until it inserts (see [Ingest Pipeline](/ingest-pipeline#when-clickhouse-cannot-take-an-insert)). A batch whose tenant has no ClickHouse connection — one no longer served, or one no pool could be opened for (such as by the connection ceiling) — skips the row-by-row retry, which no row of it could pass, and is parked whole; only a served tenant whose DLQ is off for the table leaves it for redelivery, since a tenant no longer served has no switch to read. A second class lands here too: an envelope the worker cannot *read* at all — malformed JSON, an unknown **or absent** `format`, or `columns` and `row` that do not pair — is parked without ever reaching a table batch. **Two different body shapes land here, and a consumer must not assume one decoder.** A row that failed its INSERT is parked as the `EventMessage` envelope above. An envelope the worker could not *read* is parked as **its original bytes, verbatim** — `parkOnDLQ` republishes what arrived — so it is whatever the producer sent: malformed JSON, an envelope of an unknown `format`, or a v2 envelope whose `columns` and `row` do not pair. Being undecodable as an `EventMessage` is precisely why it was parked, so decode defensively and fall back on the `X-DLQ-Error` header, which names the reason. For the first shape the body is the published `EventMessage` envelope (`{"table_name":…,"scope":"","received_timestamp":…,"format":…,"columns":[…],"row":[…]}` — the failed row is the `row` array, read against `columns`, its `DateTime`/`DateTime64` values as published: canonicalized where WaveHouse could parse them, otherwise the producer's original spelling — see [timestamp canonicalization](#timestamp-canonicalization)); the failure reason, table, and time travel in the `X-DLQ-Table` / `X-DLQ-Error` / `X-DLQ-Timestamp` message headers. +Under [`mq.backend: nats`](/deployment#external-nats) the parked rows of every tenant go to one shared dead-letter stream instead, under `.dlq.{tenant}.{table}`; the bodies and headers are the same. + Use `GET /v1/ops/dlq/stats` to monitor DLQ depth, per tenant (`?tenant=`). ## Generating a JWT for Testing diff --git a/docs/src/content/docs/architecture.md b/docs/src/content/docs/architecture.md index 5c3093985..5938a6fb1 100644 --- a/docs/src/content/docs/architecture.md +++ b/docs/src/content/docs/architecture.md @@ -45,7 +45,7 @@ flowchart TD ## Binaries -WaveHouse ships a single binary, `wavehouse`: an all-in-one process running the API, batch worker, embedded NATS JetStream, and optional embedded Pebble dedup. The only external dependency is ClickHouse. `cmd/wavehouse` is the shell — subcommand dispatch, the logger, `config.Load`, the signal context — and `internal/app` is the process itself (see [`app/`](#app--process-wiring) below). +WaveHouse ships a single binary, `wavehouse`: an all-in-one process running the API, batch worker, embedded NATS JetStream, and optional embedded Pebble dedup. The only external dependency is ClickHouse, unless a shared backend is selected: `cache.backend: redis`, `dedupe.backend: dynamodb`, or `mq.backend: nats`, which points the queue at a NATS cluster the operator runs and lets several processes, each running some of the [roles](/configuration#process-roles), share it. `cmd/wavehouse` is the shell — subcommand dispatch, the logger, `config.Load`, the signal context — and `internal/app` is the process itself (see [`app/`](#app--process-wiring) below). ## Internal Packages @@ -58,10 +58,10 @@ internal/ ├── chconn/ One ClickHouse pool per connection tuple among the served tenants, reconciled on reload under the ceiling ├── chsql/ Shared ClickHouse SQL helpers (identifier quoting, bind-safety) ├── config/ YAML + env var configuration loading -├── coord/ Leases for work that must run in one process at a time (the sweeper), with fencing tokens +├── coord/ Leases for work that must run in one process at a time (the sweeper, the ingest processes' shard claim membership), with fencing tokens (the NATS KV implementation is internal/mq/lease.go) ├── dedupe/ Optional deduplication (Reserve/Commit/Release; Pebble, DynamoDB) ├── discovery/ ClickHouse schema introspection and validation -├── ingest/ Batch buffering, DLQ, and Active Sweeper +├── ingest/ Batch buffering, DLQ, Active Sweeper, and shard claims over a shared queue ├── keyenc/ The one escaping composite keys are built from (NATS subject tokens, cache keys, dedupe keys) ├── mq/ MQ boundary: the only NATS/JetStream importer (owned message/consumer/stream types + embedded server) ├── observability/ OpenTelemetry pipeline (traces/metrics/logs + Prometheus exposition) @@ -92,10 +92,12 @@ The API layer uses [Chi](https://github.com/go-chi/chi) for routing with Request ### `app/` — Process wiring -- **app.go** — `New(ctx, Options)` builds every component from the boot config (`Options.Config`) and the settings directory it names, in dependency order: settings registry, observability (after which each `Config.Warnings` line is logged at `WARN`), ClickHouse pools, schema discovery, the dedupe stores, embedded NATS (ingest + DLQ streams), cache, the lease coordinator, sweeper, streaming (hub, MQ→hub bridge, keepalive wheel), ingest worker, auth, reload triggers, HTTP. The boot config's `roles` decide which of them a process wires: every process gets the settings registry, observability, the MQ, the coordinator, the reload triggers and a listener; `api` adds schema discovery, the dedupe stores, streaming, auth and the full router; `ingest` adds the ingest worker; `sweeper` adds the sweeper; the ClickHouse pools and the cache come with `api` or `ingest`. A process without `api` serves `api.NewOpsRouter` (probes, `/version`, the metrics path, and the settings reload behind the operator key alone, `wireOpsAuth`) on `server.port`. `config.Validate` refuses a role set the backends cannot serve (a split over the embedded MQ, or `api` without `ingest` and the reverse over a local cache), and `New` refuses a `Config` with no roles, which only one built without `config.Load` can have. Each is one `component` value — what it opens, what it loops, what it releases — so a failure part-way releases what was already opened and returns the error. `Run(ctx)` drives every loop under one `errgroup` until `ctx` is canceled (a clean stop: every loop drains, the API server and the ingest worker within `server.shutdown_timeout`; open SSE streams are ended as the drain begins rather than waited on) or a component fails, which stops the rest and returns that error. `Close(ctx)` releases what `New` opened, newest first, under the caller's release budget (`ReleaseTimeout`, 5s), a real bound: a remote implementation's close gives up at the deadline itself, and a close that ignores the context (the local stores) is abandoned at it, with the components below it left unreleased rather than overlapping it, both named in the error — and then flushes telemetry under its own 3s budget, so the flush that reports on the stop is never handed a deadline a slow close already spent. The SIGHUP registration is released last of all. `Handler`, `Registry`, and `MQ` expose the pieces a harness needs; `Options.Listener` lets one serve the API on its own listener instead of `server.port`. -- **wire.go** — one `wire*` function per component, each handed the settings registry whole and deriving the per-call getters the internal packages take (`DLQFor`, `DedupeFor`, `GapWindow`, …) and registering its `AfterAdopt` hook there where it has one. A layer with a choice of implementation — `wireMQ`, `wireCache`, `wireDedupe`, `wireCoord` — picks it there and nowhere else, in a `switch` on the boot config's `.backend` with one case per backend; the default case refuses boot, which only a `config.Config` built without `config.Load` reaches, since `Validate` refuses a value no case handles. `wireCache` has two: `local`, the in-process `LocalCache`, and `redis`, the shared `RedisCache` built from the `cache.redis` block, which boots bypassed rather than failing when its server is unreachable. Those wiring functions are where the per-tenant registry of [#583](https://github.com/Wave-RF/WaveHouse/issues/583) is injected, not `main`: `wireSettings` opens the `settings.Registry`, the HTTP handlers get store-keyed getters (method expressions such as `(*settings.Store).Policy`), and `perTenant` adapts a store accessor into the `func(tenant.ID) T` getter the async packages take, with the tenant each message's `mq.Topic` names for the stream hub and the ingest worker — a tenant the registry is not serving is logged and read as the zero value, except in `dlqFor`, the ingest worker's DLQ switch, where it reads as on so a message the worker cannot read is parked rather than dropped, and a removed or rejected tenant's queued rows are parked rather than left unacked, where each would be redelivered every ack wait for as long as the tenant is away and would hold that tenant's ack floor, so the sweeper could purge none of its queue past it. The ClickHouse pools (`chconn.Pools`) and the per-tenant schema registries (`discoveries`, in `discoveries.go`) are reconciled from `AfterAdopt` after every reload ([#583](https://github.com/Wave-RF/WaveHouse/issues/583) story 6): `wireClickHouse` builds each served tenant's `chconn.Member` from its store and logs what the reconcile refused; `wireDiscovery` builds a registry over `pools.For` for each newly served tenant — a flat directory's tenant `0` refreshed synchronously first, as before — runs its loop under the App's stop context, stops the loop of a tenant no longer served, and drives the `BootState` from the first tenant's first discovery, sticky from there; before that, a diagnostic naming a tenant a reload stopped serving goes back to the no-tenant one. The handlers resolve both per request through store-keyed getters (`chConnFor`, `registryFor`, `chTargetFor`, `queryTimeout`), the hub and the ingest worker through tenant-keyed ones (`discoveries.For`, `pools.Target`) called with the tenant the message's topic names; a tenant on no pool is an untyped nil connection, the handlers' `503`. The ingest worker is handed the cache through `sharedTables`, which bumps each namespace the worker invalidates under every tenant on the same ClickHouse address and database (`pools.SharingTables`), and the pools hook orphans the whole cache — structured-query and pipe results — of a tenant back on a pool after an absence (`Cache.InvalidateTenant`), since it was out of that fan-out while away, and of a tenant moved to another address or database, since it now reads other tables (both returned by `Pools.Reconcile`). The one setting that still follows the default tenant is read per request, the admin role of a flat directory's ops gate: `defaultPolicy` reads it through the registry, where a flat directory's tenant `0` is always served. The auth verifiers are per tenant: `wireAuth` builds one for each tenant being served, its `AfterAdopt` hook reconfigures the adopted tenants' (rebuilt only when their wiring changed) and prunes the ones no longer served, and the operator key's admin role is read from the request tenant's policy. `wireStreaming`'s hook prunes the stream hub the same way (`Hub.Prune`, with the one `served` predicate the auth, dedupe and cache hooks use too), ending the open streams of a tenant no longer served, and `wireCache`'s hook prunes the cache's version index the same way (`LocalCache.Prune`, [#262](https://github.com/Wave-RF/WaveHouse/issues/262)), so a tenant no longer served stops holding it. One setting is shared by folding over the tenants being served rather than by following tenant `0`: the keepalive wheel runs at the shortest `stream.keepalive_interval` among them (`shortestKeepalive`), re-derived after every reload the registry applies — an adoption, a rejection, or a removal — so a dropped tenant's interval leaves the wheel at once ([#597](https://github.com/Wave-RF/WaveHouse/issues/597)). The sweeper is handed each tenant's own `stream.gap_window_minutes` (`gapWindows`, read every sweep over `Registry.Known`, so a rejected tenant keeps the window its folder last had, and all of its history if the folder has been rejected since boot), since each tenant's events have a queue of their own, and runs only while its process holds the `sweeper` lease (`elected`, which wraps `coord.RunElected` over the coordinator `wireCoord` opens; `local`, the only `coord.backend`, keeps leases in the process, so the one process always holds it). The dedupe stores are per tenant ([#583](https://github.com/Wave-RF/WaveHouse/issues/583) story 7): `wireDedupe`'s `pebble` case builds a `dedupe.Stores` over the `Tenant` factory of the embedded Pebble implementation (`dedupe.NewEmbedded`), handing it `data_dir` once; the implementation decides where every tenant's store lives — one instance, each key led by its tenant (story 3) and then its table — and one reconcile closure, the boot apply and the `AfterAdopt` hook alike, sets every store to what the registry says: open exactly when its tenant is served with `dedupe.enabled` on, closed with its seen ids kept when the tenant is switched off, rejected, or removed. An instance that cannot open follows the registry's rule for the shape: fatal at boot over a flat directory, fail-closed for every tenant with dedupe on over a nested one. The system gauges report that one instance's figures (`Embedded.Stats`), not a sum over tenants. `wireDedupe`'s `dynamodb` case is `wireDynamoDedupe`, in wire_dynamodb.go (below). `wireHTTP` hands the ingest handler `dedupe.lease` (`IngestHandler.DedupeLease`) whichever backend is chosen. The ingest handler picks the tenant's store off the request's `settings.Store` (`Store.Tenant()`). The reload triggers only start in `Run`, after `New` has registered every hook, so the watcher's first reload already drives all of them: SIGHUP in both shapes, the directory watcher for a flat directory only. `wireMQ`'s `embedded` case hands each served tenant's `mq.max_bytes_gb` to `mq.Broker.SetMaxBytes` at boot, under `New`'s context (so a stop signaled mid-boot is not held up by opening many queues), and again after every reload, under the App's stop context; the first apply opens that tenant's queue. A queue that cannot be opened or resized follows the registry's rule for the shape — fatal at boot over a flat directory, logged over a nested one — and is retried by the next reload, a queue that did not open by the next publish too. How the budget is split across the tenant's streams, the time bounds, the rollback, and the dead-letter shrink guard are `internal/mq`'s. +- **app.go** — `New(ctx, Options)` builds every component from the boot config (`Options.Config`) and the settings directory it names, in dependency order: settings registry, observability (after which each `Config.Warnings` line is logged at `WARN`), ClickHouse pools, schema discovery, the dedupe stores, the MQ (embedded NATS with its ingest + DLQ streams, or the external NATS), cache, the lease coordinator, sweeper, streaming (hub, MQ→hub bridge, keepalive wheel), ingest worker, auth, reload triggers, HTTP. The boot config's `roles` decide which of them a process wires: every process gets the settings registry, observability, the MQ, the coordinator, the reload triggers and a listener; `api` adds schema discovery, the dedupe stores, streaming, auth and the full router; `ingest` adds the ingest worker; `sweeper` adds the sweeper on `mq.backend: embedded` only (under `nats` the streams' retention replaces it); the ClickHouse pools and the cache come with `api` or `ingest`. A process without `api` serves `api.NewOpsRouter` (probes, `/version`, the metrics path, and the settings reload behind the operator key alone, `wireOpsAuth`) on `server.port`. `config.Validate` refuses a role set the backends cannot serve (a split over the embedded MQ, ingest on a shared MQ over a local coordinator, a sweeper-only process on a shared MQ, or `api` without `ingest` and the reverse over a local cache), and `New` refuses a `Config` with no roles, which only one built without `config.Load` can have. Each is one `component` value — what it opens, what it loops, what it releases — so a failure part-way releases what was already opened and returns the error. `Run(ctx)` drives every loop under one `errgroup` until `ctx` is canceled (a clean stop: every loop drains, the API server and the ingest worker within `server.shutdown_timeout`; open SSE streams are ended as the drain begins rather than waited on) or a component fails, which stops the rest and returns that error. `Close(ctx)` releases what `New` opened, newest first, under the caller's release budget (`ReleaseTimeout`, 5s), a real bound: a remote implementation's close gives up at the deadline itself, and a close that ignores the context (the local stores) is abandoned at it, with the components below it left unreleased rather than overlapping it, both named in the error — and then flushes telemetry under its own 3s budget, so the flush that reports on the stop is never handed a deadline a slow close already spent. The SIGHUP registration is released last of all. `Handler`, `Registry`, and `MQ` expose the pieces a harness needs; `Options.Listener` lets one serve the API on its own listener instead of `server.port`. +- **wire.go** — one `wire*` function per component, each handed the settings registry whole and deriving the per-call getters the internal packages take (`DLQFor`, `DedupeFor`, `GapWindow`, …) and registering its `AfterAdopt` hook there where it has one. A layer with a choice of implementation — `wireMQ`, `wireCache`, `wireDedupe`, `wireCoord` — picks it there and nowhere else, in a `switch` on the boot config's `.backend` with one case per backend; the default case refuses boot, which only a `config.Config` built without `config.Load` reaches, since `Validate` refuses a value no case handles. `wireCache` has two: `local`, the in-process `LocalCache`, and `redis`, the shared `RedisCache` built from the `cache.redis` block, which boots bypassed rather than failing when its server is unreachable. Those wiring functions are where the per-tenant registry of [#583](https://github.com/Wave-RF/WaveHouse/issues/583) is injected, not `main`: `wireSettings` opens the `settings.Registry`, the HTTP handlers get store-keyed getters (method expressions such as `(*settings.Store).Policy`), and `perTenant` adapts a store accessor into the `func(tenant.ID) T` getter the async packages take, with the tenant each message's `mq.Topic` names for the stream hub and the ingest worker — a tenant the registry is not serving is logged and read as the zero value, except in `dlqFor`, the ingest worker's DLQ switch, where it reads as on so a message the worker cannot read is parked rather than dropped, and a removed or rejected tenant's queued rows are parked rather than left unacked, where each would be redelivered every ack wait for as long as the tenant is away and would hold that tenant's ack floor, so the sweeper could purge none of its queue past it. The ClickHouse pools (`chconn.Pools`) and the per-tenant schema registries (`discoveries`, in `discoveries.go`) are reconciled from `AfterAdopt` after every reload ([#583](https://github.com/Wave-RF/WaveHouse/issues/583) story 6): `wireClickHouse` builds each served tenant's `chconn.Member` from its store and logs what the reconcile refused; `wireDiscovery` builds a registry over `pools.For` for each newly served tenant — a flat directory's tenant `0` refreshed synchronously first, as before — runs its loop under the App's stop context, stops the loop of a tenant no longer served, and drives the `BootState` from the first tenant's first discovery, sticky from there; before that, a diagnostic naming a tenant a reload stopped serving goes back to the no-tenant one. The handlers resolve both per request through store-keyed getters (`chConnFor`, `registryFor`, `chTargetFor`, `queryTimeout`), the hub and the ingest worker through tenant-keyed ones (`discoveries.For`, `pools.Target`) called with the tenant the message's topic names; a tenant on no pool is an untyped nil connection, the handlers' `503`. The ingest worker is handed the cache through `sharedTables`, which bumps each namespace the worker invalidates under every tenant on the same ClickHouse address and database (`pools.SharingTables`), and the pools hook orphans the whole cache — structured-query and pipe results — of a tenant back on a pool after an absence (`Cache.InvalidateTenant`), since it was out of that fan-out while away, and of a tenant moved to another address or database, since it now reads other tables (both returned by `Pools.Reconcile`). The one setting that still follows the default tenant is read per request, the admin role of a flat directory's ops gate: `defaultPolicy` reads it through the registry, where a flat directory's tenant `0` is always served. The auth verifiers are per tenant: `wireAuth` builds one for each tenant being served, its `AfterAdopt` hook reconfigures the adopted tenants' (rebuilt only when their wiring changed) and prunes the ones no longer served, and the operator key's admin role is read from the request tenant's policy. `wireStreaming`'s hook prunes the stream hub the same way (`Hub.Prune`, with the one `served` predicate the auth, dedupe and cache hooks use too), ending the open streams of a tenant no longer served, and `wireCache`'s hook prunes the cache's version index the same way (`LocalCache.Prune`, [#262](https://github.com/Wave-RF/WaveHouse/issues/262)), so a tenant no longer served stops holding it. One setting is shared by folding over the tenants being served rather than by following tenant `0`: the keepalive wheel runs at the shortest `stream.keepalive_interval` among them (`shortestKeepalive`), re-derived after every reload the registry applies — an adoption, a rejection, or a removal — so a dropped tenant's interval leaves the wheel at once ([#597](https://github.com/Wave-RF/WaveHouse/issues/597)). The ingest worker's queue, over a `mq.Sharded` broker (`mq.backend: nats`), is wrapped in `ingest.ClaimShards` over the same coordinator, so each process consumes only its share of the shards; the `nats` coordinator holds the ingest processes' membership leases there, since no sweeper is wired under `nats`. The sweeper is handed each tenant's own `stream.gap_window_minutes` (`gapWindows`, read every sweep over `Registry.Known`, so a rejected tenant keeps the window its folder last had, and all of its history if the folder has been rejected since boot), since each tenant's events have a queue of their own, and runs only while its process holds the `sweeper` lease (`elected`, which wraps `coord.RunElected` over the coordinator `wireCoord` opens: `local` keeps leases in the process, so the one process always holds it; `nats` calls `ExternalNATS.Leases` on the MQ's own connection with the bucket `coordBucket` names — `coord.nats.bucket`, or `mq.DefaultNATSCoordBucket` of the subject prefix — and `instance_id` as the holder. `wireNATSMQ` hands the same bucket name to the topology, so boot waits for it with the streams. The coordinator is added after the MQ, so it closes first and resigns its terms while the connection is still up). The dedupe stores are per tenant ([#583](https://github.com/Wave-RF/WaveHouse/issues/583) story 7): `wireDedupe`'s `pebble` case builds a `dedupe.Stores` over the `Tenant` factory of the embedded Pebble implementation (`dedupe.NewEmbedded`), handing it `data_dir` once; the implementation decides where every tenant's store lives — one instance, each key led by its tenant (story 3) and then its table — and one reconcile closure, the boot apply and the `AfterAdopt` hook alike, sets every store to what the registry says: open exactly when its tenant is served with `dedupe.enabled` on, closed with its seen ids kept when the tenant is switched off, rejected, or removed. An instance that cannot open follows the registry's rule for the shape: fatal at boot over a flat directory, fail-closed for every tenant with dedupe on over a nested one. The system gauges report that one instance's figures (`Embedded.Stats`), not a sum over tenants. `wireDedupe`'s `dynamodb` case is `wireDynamoDedupe`, in wire_dynamodb.go (below). `wireHTTP` hands the ingest handler `dedupe.lease` (`IngestHandler.DedupeLease`) whichever backend is chosen. The ingest handler picks the tenant's store off the request's `settings.Store` (`Store.Tenant()`). The reload triggers only start in `Run`, after `New` has registered every hook, so the watcher's first reload already drives all of them: SIGHUP in both shapes, the directory watcher for a flat directory only. `wireMQ`'s `nats` case builds an `mq.NATSConfig` from the boot config's `mq.nats` block and calls `mq.NewNATS`, which waits for the operator's topology under `New`'s context; it hands over no budget, since the operator's streams set every limit. Both cases end in `adoptMQ`, which registers the MQ's close and the system gauges. The `embedded` case hands each served tenant's `mq.max_bytes_gb` to `mq.Broker.SetMaxBytes` at boot, under `New`'s context (so a stop signaled mid-boot is not held up by opening many queues), and again after every reload, under the App's stop context; the first apply opens that tenant's queue. A queue that cannot be opened or resized follows the registry's rule for the shape — fatal at boot over a flat directory, logged over a nested one — and is retried by the next reload, a queue that did not open by the next publish too. How the budget is split across the tenant's streams, the time bounds, the rollback, and the dead-letter shrink guard are `internal/mq`'s. - **wire_dynamodb.go** — `wireDedupe`'s `dynamodb` case, split out of wire.go so the e2e suite's coverage exclude for it (the e2e binary always runs Pebble dedupe, never DynamoDB) doesn't have to blanket wire.go itself: builds the same `dedupe.Stores` over `Dynamo.Tenant`, gated (`Factory.Gated`) on the table's check: boot runs `Dynamo.Check` (after `CreateTable`, when `dedupe.dynamodb.create_table` is on) whether or not any tenant has dedupe on. Boot is refused only for a misconfigured table (an error that is not `ErrUnavailable`) over a flat directory whose tenant has dedupe on; every other failure boots with the switched-on stores closed, the check retried until it passes by a background component that backs off from one second to thirty (a nested directory has no watcher, and a flat one's table can come good with no settings change). The `AfterAdopt` hook never runs the check, since it holds the lock that serializes reloads, and it does not wait on a tenant whose `dedupe.enabled` is unchanged either — `Managed.Apply`'s no-op fast path settles that case under its own read lock, so the hook only takes a store's write lock, and so waits for that tenant's in-flight `Reserve`/`Commit`/`Release` calls to finish, on a genuine flip. It applies every store against the last check's result, so a tenant a reload switches on fails closed meanwhile, and wakes the retry, so a reload still retries at once. It has no Pebble gauges. +- **wire_nats.go** — the `nats` cases of `wireMQ` and `wireCoord` (`wireNATSMQ`, `wireNATSCoord`, and the lease bucket's name, `coordBucket`), split out of wire.go for the same reason as `wire_dynamodb.go`: the e2e binary never runs `mq.backend: nats`. `wireNATSMQ` hands the topology check the dedupe lease and, in a process running `api`, warns at boot and after every reload about a tenant with dedupe on whose finite retention is under the partitions' duplicate window (`ExternalNATS.DuplicateWindow`), and runs `ExternalNATS.CheckReplayWindows(servedGapWindows(...))` at boot and from an `AfterAdopt` hook, the check of a gap window longer than the history keeps that the sweeper made on the embedded MQ. +- **wire_ops.go** — the ops-only listener of a process without the `api` role (`wireOpsAuth`, `wireOpsHTTP`), split out of wire.go for the same reason: the e2e binary runs every role. ### `stream/` — SSE keepalive & fan-out @@ -128,18 +130,19 @@ The SSE fan-out, factored out of `api/` so the delivery hot path ([#294](https:/ - **config.go** — Loads *boot* configuration from a YAML file with environment variable overrides (using [cleanenv](https://github.com/ilyakaznacheev/cleanenv)); every key has a `WH_`-prefixed env var. Boot config is only what can't change under a running process — the implementation each layer runs on, the process's `roles`, resource sizing, listeners, observability exporters, the settings-directory path, and the secrets (`clickhouse.password`, `cache.redis.password`, `auth.jwt_secret`, `auth.operator_key`). Everything tenant-tunable lives in the settings directory (`settings/`). Both sources are strict: `Load` refuses to boot naming every YAML key the struct doesn't declare (`strict.go`) and every `WH_*` environment variable no field binds (`check.go`), so a tunable that moved to the settings directory can't be read, ignored, and believed. Boot is the validator for this half — there is no dry-run command. See [Configuration Reference](/configuration). - **check.go** — `rejectUnboundEnv` is the environment half of the strict loader: `unboundEnv` walks the struct's `env` tags (plus the two process-level names, `WH_CONFIG` and `WH_LOG_LEVEL`) against the environment; `CheckDataDir` probes `data_dir` — run by `main` right after `Load` when `Config.NeedsDataDir` says a selected backend keeps state there, so an unusable `data_dir` refuses boot before anything dials out. It refuses an empty value (reachable through `WH_DATA_DIR=`) outright rather than probing the working directory; a path that exists and is not a directory; a dangling symlink at `data_dir` or any component above it (the walk to the nearest existing ancestor uses `Lstat`, so a failed mount is not skipped over as "does not exist"); and a directory the process cannot write to — or, when it does not exist, an unwritable nearest ancestor — probed by creating and removing one temp file. A permission denial, on the probe or on reaching the path through a parent without search permission, carries the UID-65532 hint, since a bind mount owned by root is the typical cause. -- **backends.go** — the `.backend` keys: one string type per layer (`MQBackend`, `CacheBackend`, `DedupeBackend`, `CoordBackend`), each with its list of the backends this build has, and a `validate` per layer block (`checkBackend`, run by `validateBackends` in `Validate`, between `validateRoles` and `validateTopology`), which refuses a value not on the list and names the ones that are. One rule spans two layers: while `mq.backend` is `embedded`, `dedupe.lease` plus its own ceiling to the next whole second (`ceilSecond`) plus one more second must fit the embedded MQ's 2m duplicate window (`embeddedDuplicateWindow`), a cap of 59s (`maxEmbeddedLease`), because a client obeying the in-flight `503`'s `Retry-After` can republish as late as the lease plus that ceiling plus a second after the claim, since DynamoDB rounds a claim's expiry up to the second. A backend's own settings go in a `.` sub-block that is its `validate`'s case to check. `Distributed` reports whether the MQ is shared with other processes, `NeedsDataDir` whether a selected backend keeps state under `data_dir`, and `Warnings` returns what a valid configuration is still likely to get wrong — the combinations that are correct for one replica only (a shared MQ over a local cache or Pebble dedupe), a `cache.redis` block that is not read, certificate verification turned off — which `app.New` logs at `WARN`. +- **backends.go** — the `.backend` keys: one string type per layer (`MQBackend`, `CacheBackend`, `DedupeBackend`, `CoordBackend`), each with its list of the backends this build has, and a `validate` per layer block (`checkBackend`, run by `validateBackends` in `Validate`, between `validateRoles` and `validateTopology`), which refuses a value not on the list and names the ones that are. One rule spans two layers: while `mq.backend` is `embedded`, `dedupe.lease` plus its own ceiling to the next whole second (`ceilSecond`) plus one more second must fit the embedded MQ's 2m duplicate window (`embeddedDuplicateWindow`), a cap of 59s (`maxEmbeddedLease`), because a client obeying the in-flight `503`'s `Retry-After` can republish as late as the lease plus that ceiling plus a second after the claim, since DynamoDB rounds a claim's expiry up to the second. A backend's own settings go in a `.` sub-block that is its `validate`'s case to check. `Distributed` reports whether the MQ is shared with other processes, `NeedsDataDir` whether a selected backend keeps state under `data_dir`, and `Warnings` returns what a valid configuration is still likely to get wrong — the combinations that are harmless or correct for one replica only (a shared MQ over a local cache or Pebble dedupe; under `nats`, `mq.max_bytes_gb` not applied; an `mq.nats` or `coord.nats` block its layer's backend ignores), a `cache.redis` block that is not read, certificate verification turned off — which `app.New` logs at `WARN`. `mq.backend` has two values, `embedded` and `nats` (`MQNATS`), and `nats` reads the `mq.nats` sub-block, which `MQ.validate` checks only when it is selected. `coord.backend` takes `nats` (`CoordNATS`) too, whose `coord.nats` block holds only the bucket name: the leases ride `mq.nats`'s connection. - **cache_redis.go** — `CacheRedisConfig`, the `cache.redis` sub-block, and its checks: an address (exactly one in `standalone` mode, which dials only the first), each `host:port` with a port from 1 to 65535 (a URL or `user:password@` form refused without repeating it, since it may hold a password), a known mode (`sentinel` is refused until [#656](https://github.com/Wave-RF/WaveHouse/issues/656)), `db` 0 in cluster mode, positive timeouts and sizes, a `timeout` and `dial_timeout` of at most 1 s each (boot and `Close` each wait out a dial: a connect and a handshake bounded by `dial_timeout`, and a cluster's topology read bounded by the larger of the two), a `version_ttl` of at least 2 s, and a `compress_min_bytes` that is not negative (`0` never compresses); its defaults are in `defaults()` with the rest. `CacheRedisTLS.Config` builds the `tls.Config`, reading the files; `Validate` calls it so an unreadable file refuses boot, and `internal/app` calls it again to build the connection. A TLS key set while `tls.enabled` is off is an error rather than a plaintext connection. -- **config.go**, roles — `roles` (`[]Role`: `api`, `ingest`, `sweeper`; `AllRoles` by default; `Has(Role)`) picks which components `internal/app` wires, and `instance_id` names the process (`-<8 hex>` when empty, resolved in `Load`; today only logged at boot, and a distributed coordinator will record it as a lease's holder). `validateRoles` refuses an empty list, an empty entry, an unknown or a repeated role; `validateTopology` refuses a role set the backends cannot serve: any split over the embedded MQ, and a process with exactly one of `api` and `ingest` over a local cache. `NeedsDataDir` counts Pebble only for a process running `api`, and `Warnings` is empty without `api`, since only that role opens a cache it reads or a dedupe store. +- **mq_nats.go** — the two nats blocks and their rules, apart from backends.go for the e2e coverage gate as `cache_redis.go` is: `MQNATSConfig` (URLs, file-path-only credentials, TLS, and the topology to expect; its defaults, `defaultMQNATS`, are in `defaults()`) and its checks, `CoordNATSConfig` (the bucket name), the nats lines of `Warnings` (`natsWarnings`) and the two cross-layer rules `validateTopology` delegates (`validateNATSTopology`: `coord.backend=nats` needs `mq.backend=nats`, a process running `ingest` on a shared queue needs `coord.backend=nats`, and a process running only `sweeper` there is refused). +- **config.go**, roles — `roles` (`[]Role`: `api`, `ingest`, `sweeper`; `AllRoles` by default; `Has(Role)`) picks which components `internal/app` wires, and `instance_id` names the process (`-<8 hex>` when empty, resolved in `Load`; logged at boot, and the holder a NATS lease's value names). `validateRoles` refuses an empty list, an empty entry, an unknown or a repeated role; `validateTopology` refuses a role set the backends cannot serve: any split over the embedded MQ, `coord.backend=nats` without `mq.backend=nats` (the leases ride its connection), `mq.backend=nats` with `coord.backend=local` in a process running `ingest` (the ingest processes share the queue through leases), a process running only `sweeper` under `mq.backend=nats` (it would run nothing), and a process with exactly one of `api` and `ingest` over a local cache. `NeedsDataDir` counts Pebble only for a process running `api`, and the cache and dedupe warnings are skipped without `api`, since only that role opens a cache it reads or a dedupe store. - **strict.go** — `rejectUnknownKeys`, the YAML half: re-reads the file as a generic tree and walks it against the struct's `yaml` tags, listing every key the struct doesn't declare. cleanenv itself is lenient by design, which is exactly wrong for boot config once keys have moved to the settings directory. - **persistence.go** — `WarnIfFreshDataDir` logs the startup `WARN` when `data_dir` is missing or empty (on a redeploy, the sign that the volume didn't persist); `LogStorageInitError` attaches the UID-65532 `permissionHint` to a NATS or Pebble open failure that looks like a permission denial — the same hint string `CheckDataDir` uses. ### `coord/` — Leases -- **coord.go** — `Coordinator` hands out named leases: `TryAcquire(ctx, name)` returns a `Term` if nobody holds a live one, `ErrHeld` if somebody does (this process included), and the term is held until `Resign`, the coordinator's `Close`, or loss; `ctx` bounds the call, not the term. A `Term` carries a fencing `Token` — strictly greater than every earlier term's for the same name on the same backend — and a `Done` channel that closes when it ends, with `Err` saying why (nil after `Resign`/`Close`, wrapping `ErrLost` after a loss). A term can overlap its successor if its holder stalls past the lease duration, so anything that needs strict exclusivity must check `Token` against what it writes; the sweeper does not: an overlap cannot lose ClickHouse data, since every sweep stops at the consumer's ack floor; it can only trim SSE replay history, and only when the two holders' settings views differ (one still reading a shorter `stream.gap_window_minutes`, or missing a tenant, after a reload the other has applied) — which fencing would not prevent either. The package imports only the standard library, so a distributed implementation can live beside the connection it rides on (`internal/mq` for a NATS KV bucket) without a cycle. -- **local.go** — `Local`, the in-process implementation: a mutex-guarded table where the first `TryAcquire` of a name wins and a term never expires. `Peer` returns a second coordinator over the same table, as a second process would hold one over a shared backend (for tests). +- **coord.go** — `Coordinator` hands out named leases (and a backend that can tell whether a lease is held without campaigning implements `Observer.Held`, which the ingest processes' membership reads): `TryAcquire(ctx, name)` returns a `Term` if nobody holds a live one, `ErrHeld` if somebody does (this process included), and the term is held until `Resign`, the coordinator's `Close`, or loss; `ctx` bounds the call, not the term. A `Term` carries a fencing `Token` — strictly greater than every earlier term's for the same name on the same backend — and a `Done` channel that closes when it ends, with `Err` saying why (nil after `Resign`/`Close`, wrapping `ErrLost` after a loss). A term can overlap its successor if its holder stalls past the lease duration, so anything that needs strict exclusivity must check `Token` against what it writes; the sweeper does not: an overlap cannot lose ClickHouse data, since every sweep stops at the consumer's ack floor; it can only trim SSE replay history, and only when the two holders' settings views differ (one still reading a shorter `stream.gap_window_minutes`, or missing a tenant, after a reload the other has applied) — which fencing would not prevent either. The package imports only the standard library, so a distributed implementation can live beside the connection it rides on (`internal/mq` for a NATS KV bucket) without a cycle. +- **local.go** — `Local`, the in-process implementation: a mutex-guarded table where the first `TryAcquire` of a name wins and a term never expires; `Held` reads that table. `Peer` returns a second coordinator over the same table, as a second process would hold one over a shared backend (for tests). - **elect.go** — `RunElected(ctx, c, name, retry, fn)`: campaigns for the lease every `retry` (`RetryPeriod`, 2s), runs `fn` under a context canceled when the term ends, resigns when `fn` returns, and campaigns again, until `ctx` is done or the coordinator is closed (`ErrClosed`, returned). An error `fn` returns while its term is live is returned (fatal to `app.Run`, like any component's); `ErrHeld`, a lost term, and a failed campaign (logged, then retried) are not. -- **coordtest/** — `Conformance(t, factory, opts...)`, the suite every implementation runs against its own backend: one holder at a time, monotonic tokens across holders, `Resign` lets the other in, `Close` resigns every term and refuses more, the context bounds the call and not the term, and — for a backend that can lose a term (`WithLoss`) — loss closes `Done` with `ErrLost`. +- **coordtest/** — `Conformance(t, factory, opts...)`, the suite every implementation runs against its own backend: one holder at a time, a lease seen as held without campaigning (for an `Observer`), monotonic tokens across holders, `Resign` lets the other in, `Close` resigns every term and refuses more, the context bounds the call and not the term, and — for a backend that can lose a term (`WithLoss`) — loss closes `Done` with `ErrLost`. ### `dedupe/` — Deduplication (Optional) @@ -160,28 +163,34 @@ The SSE fan-out, factored out of `api/` so the delivery hot path ([#294](https:/ ### `ingest/` — Ingest Pipeline, DLQ & Sweeping -- **worker.go** — `StartIngestWorker` launches an ingest pipeline: a durable `buffer-consumer` consumer of the ingest queue (created through `mq.ConsumerManager`) reads events, batches them per tenant table — the tenant read off each message's `mq.Topic` — and performs bulk INSERTs to ClickHouse. The pipeline is **insert-only**. The wire format `EventMessage` carries `{table_name, scope, received_timestamp, format, columns, row}` — the row positionally as one `JSONCompactEachRow` line, with `columns` naming its positions (the table's insertable columns — a computed one cannot be named in an `INSERT`); the worker batches per (tenant, table, column list) and writes `INSERT INTO … (cols) FORMAT JSONCompactEachRow`. It accepts any table name (events are addressed by `mq.Topic{Tenant, Table, Scope}` with raw names; `internal/mq` encodes them into subject tokens), then bulk-INSERTs. The embedded NATS server runs with `DontListen: true` (`internal/mq/embedded.go`), so the only publishers that can reach the ingest queue are in-process Go code — today, only the HTTP `/v1/ingest?table={table}` handler. Non-insert mutations (`DELETE`/`UPDATE`/`TRUNCATE`/…) must go through `POST /v1/ops/query` under the admin role (`policy.admin_role`) — see the Query Path section below; the `/v1/ops/*` `RequireAdmin` middleware enforces the check at the API layer, so a no/invalid-token request (resolved to `default_role`, not admin in a production config) never reaches the proxy — or through an operator-authored [pipe that writes](/pipes#pipes-that-write), gated only by its `allowed_roles`. A batch whose tenant has no ClickHouse connection (no longer served, or no pool could be opened for it, such as by the connection ceiling) is never tried: no row of it could pass, so `parkBatch` takes it to the DLQ switch whole, logging once per batch rather than twice per row. Otherwise a bulk-insert failure is first classed by `chconn.Classify`: a ClickHouse that cannot take the insert (unavailable, denied, or no verdict at all) sends the batch back to the MQ for a delayed redelivery (`retryLater` → `mq.Message.NakWithDelay`), under a backoff shared by every table on the same pool (a failure of one table — read-only, too many parts — backs off that table alone), and never to the DLQ — the same when it stops answering mid-isolation. Only when ClickHouse rejects the batch, or refuses a multi-row batch for its size (`chconn.Splittable`: too many partitions for one INSERT, the memory limit), is it re-inserted row by row: rows that succeed are acked, and only the rows ClickHouse rejects again are routed to the DLQ (`sendToDLQ` → `mq.DeadLetterer.DeadLetter`), which parks the as-published `EventMessage` envelope under the topic it arrived on (`dlq.{tenant}.{table}` subjects inside `internal/mq`) with the failure context in `X-DLQ-*` headers when the tenant's `dlq.enabled` is on for the table — see [Ingest Pipeline](/ingest-pipeline) for the worker internals. +- **worker.go** — `StartIngestWorker` launches an ingest pipeline: a durable `buffer-consumer` consumer of the ingest queue (created through `mq.ConsumerManager`) reads events, batches them per tenant table — the tenant read off each message's `mq.Topic` — and performs bulk INSERTs to ClickHouse. The pipeline is **insert-only**. The wire format `EventMessage` carries `{table_name, scope, received_timestamp, format, columns, row}` — the row positionally as one `JSONCompactEachRow` line, with `columns` naming its positions (the table's insertable columns — a computed one cannot be named in an `INSERT`); the worker batches per (tenant, table, column list) and writes `INSERT INTO … (cols) FORMAT JSONCompactEachRow`. It accepts any table name (events are addressed by `mq.Topic{Tenant, Table, Scope}` with raw names; `internal/mq` encodes them into subject tokens), then bulk-INSERTs. The embedded NATS server runs with `DontListen: true` (`internal/mq/embedded.go`), so under `mq.backend: embedded` the only publishers that can reach the ingest queue are in-process Go code — today, only the HTTP `/v1/ingest?table={table}` handler. Under `mq.backend: nats`, anyone the operator lets publish to `.ingest.>` reaches it too, past auth, policy and schema validation, so that right belongs to the `wavehouse` user alone. Non-insert mutations (`DELETE`/`UPDATE`/`TRUNCATE`/…) must go through `POST /v1/ops/query` under the admin role (`policy.admin_role`) — see the Query Path section below; the `/v1/ops/*` `RequireAdmin` middleware enforces the check at the API layer, so a no/invalid-token request (resolved to `default_role`, not admin in a production config) never reaches the proxy — or through an operator-authored [pipe that writes](/pipes#pipes-that-write), gated only by its `allowed_roles`. A batch whose tenant has no ClickHouse connection (no longer served, or no pool could be opened for it, such as by the connection ceiling) is never tried: no row of it could pass, so `parkBatch` takes it to the DLQ switch whole, logging once per batch rather than twice per row. Otherwise a bulk-insert failure is first classed by `chconn.Classify`: a ClickHouse that cannot take the insert (unavailable, denied, or no verdict at all) sends the batch back to the MQ for a delayed redelivery (`retryLater` → `mq.Message.NakWithDelay`), under a backoff shared by every table on the same pool (a failure of one table — read-only, too many parts — backs off that table alone), and never to the DLQ — the same when it stops answering mid-isolation. Only when ClickHouse rejects the batch, or refuses a multi-row batch for its size (`chconn.Splittable`: too many partitions for one INSERT, the memory limit), is it re-inserted row by row: rows that succeed are acked, and only the rows ClickHouse rejects again are routed to the DLQ (`sendToDLQ` → `mq.DeadLetterer.DeadLetter`), which parks the as-published `EventMessage` envelope under the topic it arrived on (`dlq.{tenant}.{table}` subjects inside `internal/mq`) with the failure context in `X-DLQ-*` headers when the tenant's `dlq.enabled` is on for the table — see [Ingest Pipeline](/ingest-pipeline) for the worker internals. - **backoff.go** — The retry backoff behind `retryLater`: a small circuit breaker per ClickHouse pool (the target's URL, user and database), and one per pool and table for a failure of one table (`chconn.TableScoped`). A failure opens it for 1 s, doubling to a 30 s cap, each window jittered down to half; while it is open, flushes and arriving rows are handed back without a request, and once it elapses one flush probes. Any answer that is not an outage closes it. - **types.go** — `EventMessage` struct (TableName, Scope — reserved, always empty today, ReceivedTimestamp, Format, Columns, Row; `Format` is `FormatJSONCompactEachRow` and `Row` is one positional line whose slots `Columns` names) and `BufferConsumerName` constant, shared across API handlers and the ingest pipeline. - **compact.go** — `EncodeCompactRow`, the positional row encoder every published row goes through, rendering one record over the table's **insertable** columns in declaration order. Serialization only: it validates nothing and judges no value. +- **claims.go**, **assign.go** — `ClaimShards` wraps a `Sharded` queue so an ingest process consumes only its share of the units, one process per unit at a time. Each process holds one membership lease, `ingest.m` for the lowest free `j` under the number of configured units (at most 64, read 16 at a time each tick), and every tick (2s) reads which slots are live (`coord.Observer.Held`); `assignUnits` gives every unit to a live slot by rendezvous hashing capped at ⌈units/slots⌉, the same result in every process for the same view; the extras are assigned apart from the configured units, so processes whose lists of extras differ (each refreshes it every five minutes) still agree on every configured unit. A unit no longer assigned halts (`mq.Halter`: stops fetching, keeps its pin, returns once what it fetched reached the worker), waits (bounded by the worker's 60s ack wait, the shortest `ack_wait` a durable may have) for the rows it delivered to settle (`Message.OnSettled`), then releases its pin (`mq.Releaser`). `Halt` on the claiming consumer ends the ticks and halts every unit at once; the worker calls it before its final flush, and the claims' stop releases the units and resigns the membership lease after it. A unit taken over from a slot that is no longer live is reset (`ResetOrphaned`) before it is bound — waiting, unbound, while the broker answers `ErrUnitHeld` (the dead owner's pin has not lapsed, or the unit was active within its pinned TTL), up to 30s — so the dead owner's unacked rows come back at once; on a process's first tick only if it is the one live member (a restart after a clean stop; after a crash the dead run's lease still counts as live for a lease duration). Each unit's rows delivered and unsettled are capped at its share of the worker's 10,000 (`ClaimConfig.MaxHeld`): an even share over the units assigned (`unitShare`, recomputed each tick and read by the broker before every fetch through `ConsumerConfig.MaxHeld`), never under 1,000, two batches; at its share a unit fetches only what keeps its pin, and one stuck unit never takes another's. A row stops counting at its first ack or nak attempt (`Message.OnSettled` fires whether or not the broker confirms), or after `ack_wait` if the worker never settles it, since the broker redelivers it then as a new row. A configured unit whose delivery ends, or whose durable or stream is gone or whose durable no longer fits (`mq.ErrConsumerNotFound`, `mq.ErrConsumerMismatch`) when it is bound, fails the worker; any other bind failure is retried next tick, logged as an error once it has lasted a minute, and an extra's end is logged. The lowest live slot counts the units with rows and no owner (`Sharded.Unowned`) for `wavehouse_ingest_shards_unowned`. - **sweeper.go** — `Sweeper` implements the Active Sweeper pattern. It runs every minute and asks the MQ (`mq.Purger.PurgeAcked`) to drop the ingest events that are **both** ACKed by the buffer consumer (written to ClickHouse) **and** older than the gap window (re-read every sweep: each tenant's own `stream.gap_window_minutes`, a rejected tenant's as its folder last had it (unbounded for one rejected since boot) — `internal/app`'s `gapWindows` — and none for a removed tenant). Finding the purge point is `internal/mq`'s (`purge.go`). ### `mq/` — Message Queue The **only** package that imports NATS/JetStream — a `depguard` rule in `.golangci.yml` fails `make lint` on any `github.com/nats-io` import in every package golangci-lint builds; the `integration`-tagged files under `tests/` sit outside its default build context, so the boundary there rests on convention (AGENTS.md Key Design Decision #20). Every other package talks to the broker through the types below, so a subject, stream, or broker change lands here once. -- **mq.go** — The owned surface, stated as intent rather than broker mechanics. `Topic{Tenant, Table, Scope}` is the only address the rest of the process handles (a validated tenant id and raw names; comparable, so the SSE hub keys its index by the value). `Message` carries `Data`, its topic (`Topic()` decodes the delivered key on demand — the tenant included, which is how the hub bridge and the worker learn whose event it is; `TopicKey()` is the delivered form, for log lines), and the ack family (`DoubleAck(ctx)`, `Ack()`, `Nak()`, `NakWithDelay(d)`, which falls back to `Nak` for a message built without `WithNakDelay`); `Headers` is the message header map (`Add`/`Set`/`Get`, exact-key) that `PublishOpt`s such as `WithHeader` shape; `WithIdempotencyKey` marks a publish so that a second one carrying the same key inside the queue's duplicate window is dropped and reported as success. Interfaces, each speaking per tenant and never per stream: `Publisher` (`ErrQueueFull` when the tenant's ingest queue is at its byte budget or not open yet — the API's 503 + `Retry-After: 30`; `ErrUnavailable` when the broker cannot be reached or does not answer in time — the 503 + `Retry-After: 5`, which no backend returns yet: the embedded broker's publish failures are the `500`), `Subscriber` (every ingest event of every tenant, under a named durable consumer, its fetch-ahead split across the tenants — the hub bridge), `ConsumerManager` → `Consumer` (a durable explicit-ack consumer from a `ConsumerConfig`, whose `MaxAckPending` holds per tenant; `Consume` delivers each tenant's messages on a goroutine of that tenant's, in order, so a blocking handler is backpressure on its own tenant alone, spreads the prefetch across the tenants, and returns a `stop` plus a `failed` channel that reports delivery ending on its own — `ErrDeliveryEnded`, e.g. a deleted consumer or a closed connection — since no message would ever say so) for the ingest worker, `DeadLetterer.DeadLetter` (park a message under its own topic, in its tenant's dead-letter queue; the caller acks), `DeadLetterStats.DeadLetterCounts` (one tenant's; `ErrNoDeadLetterQueue` when it has none), `Purger.PurgeAcked` (drop what is both acked by a consumer and stored before its tenant's cutoff, everything acked for a tenant given none; one error per failed tenant, joined — `ErrConsumerNotFound` for a queue the consumer has not been created on yet, the one failure the sweeper logs as a warning rather than an error) for the sweeper, and `Replayer.ReplaySince` for SSE gap-fill. `Broker` composes them with each tenant's byte budget (`SetMaxBytes`/`MaxBytes`), `Stats`, and `Close`; it is what `internal/app` holds. +- **mq.go** — The owned surface, stated as intent rather than broker mechanics. `Topic{Tenant, Table, Scope}` is the only address the rest of the process handles (a validated tenant id and raw names; comparable, so the SSE hub keys its index by the value). `Message` carries `Data`, its topic (`Topic()` decodes the delivered key on demand — the tenant included, which is how the hub bridge and the worker learn whose event it is; `TopicKey()` is the delivered form, for log lines), and the ack family (`DoubleAck(ctx)`, `Ack()`, `Nak()`, `NakWithDelay(d)`, which falls back to `Nak` for a message built without `WithNakDelay`); `Headers` is the message header map (`Add`/`Set`/`Get`, exact-key) that `PublishOpt`s such as `WithHeader` shape; `WithIdempotencyKey` marks a publish so that a second one carrying the same key inside the queue's duplicate window is dropped and reported as success. Interfaces, each speaking per tenant and never per stream: `Publisher` (`ErrQueueFull` when the tenant's ingest queue is at its byte budget or not open yet — the API's 503 + `Retry-After: 30`; `ErrUnavailable` when the broker cannot be reached or does not answer in time — the 503 + `Retry-After: 5`, which the external NATS backend returns; the embedded broker's publish failures are the `500`), `Subscriber` (every ingest event of every tenant, under a named durable consumer, its fetch-ahead split across the tenants — the hub bridge), `ConsumerManager` → `Consumer` (a durable explicit-ack consumer from a `ConsumerConfig`, whose `MaxAckPending` holds per tenant, and which may name `Units` of a `Sharded` broker — one whose ingest queue is split into units one consumer at a time owns, with `IngestUnits`, `ResetOrphaned` (`ErrUnitHeld` while a consumer holds the unit or was active on it within its pinned TTL) and `Unowned`; such a consumer is a `Releaser`, may be a `Halter` (stops fetching ahead of its stop, keeping its hold on its units, so the worker writes what it holds first), is capped per unit by `ConsumerConfig.MaxHeld`, and `Message.OnSettled` (settle.go, apart from mq.go for the e2e coverage gate) runs a hook once, at the first ack or nak attempt, confirmed or not; `Consume` delivers each tenant's messages on a goroutine of that tenant's, in order, so a blocking handler is backpressure on its own tenant alone, spreads the prefetch across the tenants, and returns a `stop` plus a `failed` channel that reports delivery ending on its own — `ErrDeliveryEnded`, e.g. a deleted consumer or a closed connection — since no message would ever say so) for the ingest worker, `DeadLetterer.DeadLetter` (park a message under its own topic, in its tenant's dead-letter queue; the caller acks), `DeadLetterStats.DeadLetterCounts` (one tenant's; `ErrNoDeadLetterQueue` when it has none), `Purger.PurgeAcked` (drop what is both acked by a consumer and stored before its tenant's cutoff, everything acked for a tenant given none; one error per failed tenant, joined — `ErrConsumerNotFound` for a queue the consumer has not been created on yet, the one failure the sweeper logs as a warning rather than an error) for the sweeper, and `Replayer.ReplaySince` for SSE gap-fill. `Broker` composes them with each tenant's byte budget (`SetMaxBytes`/`MaxBytes`), `Stats`, and `Close`; it is what `internal/app` holds. - **subject.go** — The embedded broker's naming, private to the package: the stream names (`INGEST_` and `DLQ_` — prefixes that differ in their first letter, so no tenant id makes one kind's name the other's — and the one pair an earlier build kept for every tenant together, `WAVEHOUSE`/`WAVEHOUSE_DLQ`, which boot deletes), the `ingest.`/`dlq.` prefixes and `>` wildcards, the subject tokens (`internal/keyenc`: ASCII letters, digits, `_` and `-` pass, everything else is percent-encoded, so a name can never split or wildcard a subject), and `Topic` ↔ subject conversion. A subject is `.
[.]`: the tenant verbatim — its grammar (`tenant.Parse`) makes it one token, and it is checked on the way to the wire, so a topic without one has no subject — then the table and scope as encoded tokens; tenant first so one wildcard selects a tenant's traffic (`ingest.acme.>`). A topic has the same tail on both streams, so parking on the DLQ is a prefix swap on the delivered subject — nothing is decoded or re-encoded — and the tail's first token picks the tenant's stream. - **deadletter.go** — `deadLetterTables`, the per-table count `DeadLetterCounts` reports: a dead-letter stream's per-subject counts, each subject parsed back to its topic and counted under its table — every scope of a table under the table itself, so a dotted table name never shares a count with a table + scope pair — and a table filter keeps that table with all of its scopes. - **purge.go** — The Active Sweeper's arithmetic over JetStream sequences: purge target = `MIN(consumer ack floor + 1, first sequence stored at or after the cutoff)`, the latter found by binary search over message timestamps (~15 lookups). Every uncertainty resolves toward purging less: a sequence that holds no message is kept as a candidate bound rather than discarding the half below it, and a lookup that fails outright aborts that tenant's purge. It runs on each tenant's stream at that tenant's cutoff, and a failure on one tenant's stream is reported without stopping the sweep of the others. Healthy state keeps exactly the gap window; ClickHouse down freezes purging; a catastrophic outage fills the stream to `MaxBytes` and `DiscardNew` pushes back. -- **embedded.go** — `EmbeddedNATS`, the one `Broker`: an in-process NATS server with JetStream, giving each tenant a queue of its own — stream `INGEST_` with subjects `ingest..>`, capped at the tenant's `mq.max_bytes_gb` (`DiscardNew`) and remembering idempotency keys for `EmbeddedDuplicateWindow` (two minutes, sized to `2 × the dedupe lease + 1s` — the in-flight `503` sends the full lease as `Retry-After`, so an obedient client's retry can land up to ~2×lease after the original `Reserve`, and the `+1s` covers a backend whose claim expiry itself rounds up by that much), and stream `DLQ_` (`dlq..>`, `DiscardOld`) at a tenth of it — with the durable consumers on the ingest one; nothing outside the package sees that layout. JetStream's own check of the streams' caps against the disk (75% of the free disk by default) is set out of reach, so a budget is a cap and never a reservation. Boot deletes the pair an earlier build kept for every tenant together — its subjects overlap every tenant's — and takes stock of the tenants' streams on disk with their budgets, so a consumer created later is held on every one, a tenant no longer served included. `SetMaxBytes` opens a tenant's queue the first time — its dead-letter stream first, so no row is queued that could not be parked — and every registered consumer joins it; a queue a consumer cannot join is not recorded open, `SetMaxBytes` returning the error and `MaxBytes` not reporting the budget, so the next reload tries the consumers that had not joined again. A publish or park that finds a stream missing reopens it at the budget last asked for the tenant, and so does a publish to a queue the broker has not recorded open — one a consumer could not join, or an open that timed out, which can leave a stream JetStream creates after all, which no consumer holds — either refused as a full queue with none asked yet. Publishes and parks that find the same queue not open share one attempt (`singleflight`), and after one fails the tenant's publishes and parks are refused at once for five seconds rather than each trying again under the broker's lock, which every tenant's open, resize and reload takes; a reload retries regardless. After that `SetMaxBytes` applies a reloaded budget to the tenant's two live streams as a pair: if the DLQ update fails after the ingest one succeeded, the ingest resize is undone so both stay on the previous budget — best effort, since if that undo also fails the ingest stream stays at the new limit and the DLQ at the previous, and the error says so. A dead-letter stream is never capped below the bytes it holds, which `DiscardOld` would delete to fit ([#532](https://github.com/Wave-RF/WaveHouse/issues/532)): it keeps what it holds, and that is logged. Its JetStream calls are bounded to ten seconds — plus ten more for the consumers joining a queue they do not hold yet, and five for the rollback of a failed resize, each a budget of its own rather than the one that just expired — since a reload holds the settings store's lock while its hooks run; `MaxBytes` reports the budget last applied in full, so a failed resize is retried by the next reload. The consumers `CreateConsumer` and `Subscribe` build hold one durable on each tenant's stream, looked up before anything is written so a boot over many queues writes nothing it need not, each delivering on a goroutine of its own into the one handler. `PurgeAcked` and `DeadLetterCounts` run per tenant stream. `Stats` reports connection and inbound-message counters for `observability.RegisterSystemMetrics`. Trace context rides in the message headers: `Publish` applies `observability.InjectHeaders`, and a message delivered through `Subscribe` (the hub bridge) carries `observability.ExtractHeaders` on its `Ctx`; the worker's `Consumer` path skips the extraction, since it batches across messages and reads no per-message context. +- **external.go** — `ExternalNATS`, the `Broker` over an operator-owned NATS cluster (`mq.backend: nats`): N work-queue ingest partitions shared by every tenant, each split into V shards by a subject token (`natsRoute`: jump hashes of the tenant and table, one for the partition and an independent one for the shard), with one pinned-client durable per shard, each partition republishing every row it stores to the history stream under `.hist.` for SSE replay and the hub, and one dead-letter stream. Ingest publishes do not name the stream they expect, since the republished copy would carry that expectation to the history, which would refuse it; the ack's stream is checked instead. It never creates, changes, purges or deletes a stream or a durable; it creates only auto-expiring consumers on the history stream, one per `Subscribe` and one per replay. `NewNATS` connects and waits for the topology to pass the verifier; publishes carry a `Nats-Msg-Id` reused across retries; a broker that does not answer is `ErrUnavailable`; `PurgeAcked` removes nothing; as a `Sharded` broker it names its units (`/`, the configured N×V then any extras a lower N or V left holding rows, which are drained like the rest); a consumer (external_consumer.go) fetches each unit in the priority group `wavehouse`, 1s pulls sized to what the unit's cap leaves room for, on a goroutine that never runs the handler (another hands the rows on, in order), so the server delivers a unit to one pinned puller at a time and a busy handler never lets the pin lapse; at the cap it renews the pin every 5s with a one-row fetch, up to one `ack_wait` of renewals past the cap (a pull that held a due redelivery back would have the server requeue it behind the others), and otherwise, and once halted, with a pull of `max_bytes` 1, which renews the pin and delivers nothing (the server holds the row back). `Halt` stops fetching and returns once the fetch in flight has ended and its rows reached the handler; a halted unit keeps renewing until `Release`, or until `ack_wait` and one renewal have passed since it handed on what it fetched. `Release` unpins a unit only while the server still holds this consumer's own pin (from the `Nats-Pin-Id` of the last row it received); the check and the unpin are two requests, and the pin cannot lapse between them because the unit renews it until the unpin returns, for up to `ack_wait` (and one renewal) after its halt has handed on what it fetched, which outlasts the handover's own wait. A pull that gets no responders twice in a row, or a renewal that gets one, looks the durable up, and a durable or stream that is gone ends the unit's delivery (`failed`). `ResetOrphaned` resets a unit's durable to its ack floor when no client is pinned, the durable delivered nothing and had no ack within its pinned TTL, and rows await an ack; a caller does it before it pulls the unit, never after, since a reset also redelivers the caller's own unacked rows. `Unowned` applies the same activity test. A consumer of every unit resets orphaned units before pulling and releases each after its halt. It exports the `wavehouse_mq_connected`, `wavehouse_mq_topology_ok` and `wavehouse_mq_history_behind_seconds` gauges. +- **lease.go** — `ExternalNATS.Leases(ctx, bucket, holder)`, the `coord.Coordinator` (and `coord.Observer`: `Held` counts another holder's lease as held until this coordinator has seen its revision unchanged for the lease duration, the clock `TryAcquire` keeps) for `coord.backend: nats`, over the operator's KV bucket (a missing one is `ErrTopology`; WaveHouse never creates it). A lease is the key `lease.`, its value JSON `{holder, duration_ms, session}`; the KV revision a term was taken at is its fencing `Token`. `TryAcquire` creates an absent (or resigned: a delete marker) key; takes over another holder's only after seeing the same revision unchanged for the lease duration on its own monotonic clock (no clocks are compared, and the bucket keeps no per-key TTL, which a renewal could not extend), by a compare-and-set at that revision; and resumes its own write (same `session`) at once. The term renews every `coord.RetryPeriod` (2s) at the revision it last wrote; a write refused for its revision ends it with `ErrLost` unless the key holds its own value at a later revision (a renewal whose answer was lost), and a timer ends it the moment the renew deadline (10s, before the 15s lease duration) passes without a stored renewal, timed from when that renewal was sent, since a candidate's clock can start as soon as it is stored. `Resign` and `Close` cancel a renewal in flight and delete the key at the last revision (or at a later one holding this term's own value, a renewal the cancel cut short), so a successor need not wait. `TryAcquire` holds no lock across a request. `WithLeaseTimings` shortens the timings for tests. +- **nats_topology.go**, **nats_manifests.go**, **subject_nats.go** — what the operator must create (`NATSTopology`: with a dedupe lease, every partition's duplicate window must cover the lease, the lease rounded up to a second, and one more second; with the lease bucket, `CoordBucket`, checked only when a process holds leases there: it must exist, keep a value per key, allow direct gets, and expire nothing), the verifier that checks a live server against it and reports every finding (required or recommended), the nack resources `wavehouse mq manifests` prints from the same spec, sized to fit the servers' file store (`deployments/nats/jetstream.yaml` is its output for N=4, V=8), and the `wavehouse` user's permissions `wavehouse mq permissions` prints (one entry per shard durable: a NATS wildcard is a whole token; both API forms under a JetStream domain), and the external broker's subjects (`.ingest.

...

`, `.hist..
`, `.dlq..
`). +- **nats_permprobe.go** — The verifier's permission probe: a pull and an unpin to each shard durable that the server rejects on their merits, so a refused one (reported only to the connection's error handler) names a shard the user may not pull from or unpin. +- **natstest/** — Test code that stands up NATS as an operator deploys it, from the shipped `deployments/nats` values and manifests: the config for a server, with the file store the Helm chart would give it (in process, or in the integration suite's container) and the operator's hand on it (applying the manifests, the lease bucket included; deleting a durable or the bucket; reading which process holds a lease). It lets `internal/app` and `tests/integration` run against a real server without importing NATS themselves. +- **embedded.go** — `EmbeddedNATS`, the in-process `Broker`: an in-process NATS server with JetStream, giving each tenant a queue of its own — stream `INGEST_` with subjects `ingest..>`, capped at the tenant's `mq.max_bytes_gb` (`DiscardNew`) and remembering idempotency keys for `EmbeddedDuplicateWindow` (two minutes, sized to `2 × the dedupe lease + 1s` — the in-flight `503` sends the full lease as `Retry-After`, so an obedient client's retry can land up to ~2×lease after the original `Reserve`, and the `+1s` covers a backend whose claim expiry itself rounds up by that much), and stream `DLQ_` (`dlq..>`, `DiscardOld`) at a tenth of it — with the durable consumers on the ingest one; nothing outside the package sees that layout. JetStream's own check of the streams' caps against the disk (75% of the free disk by default) is set out of reach, so a budget is a cap and never a reservation. Boot deletes the pair an earlier build kept for every tenant together — its subjects overlap every tenant's — and takes stock of the tenants' streams on disk with their budgets, so a consumer created later is held on every one, a tenant no longer served included. `SetMaxBytes` opens a tenant's queue the first time — its dead-letter stream first, so no row is queued that could not be parked — and every registered consumer joins it; a queue a consumer cannot join is not recorded open, `SetMaxBytes` returning the error and `MaxBytes` not reporting the budget, so the next reload tries the consumers that had not joined again. A publish or park that finds a stream missing reopens it at the budget last asked for the tenant, and so does a publish to a queue the broker has not recorded open — one a consumer could not join, or an open that timed out, which can leave a stream JetStream creates after all, which no consumer holds — either refused as a full queue with none asked yet. Publishes and parks that find the same queue not open share one attempt (`singleflight`), and after one fails the tenant's publishes and parks are refused at once for five seconds rather than each trying again under the broker's lock, which every tenant's open, resize and reload takes; a reload retries regardless. After that `SetMaxBytes` applies a reloaded budget to the tenant's two live streams as a pair: if the DLQ update fails after the ingest one succeeded, the ingest resize is undone so both stay on the previous budget — best effort, since if that undo also fails the ingest stream stays at the new limit and the DLQ at the previous, and the error says so. A dead-letter stream is never capped below the bytes it holds, which `DiscardOld` would delete to fit ([#532](https://github.com/Wave-RF/WaveHouse/issues/532)): it keeps what it holds, and that is logged. Its JetStream calls are bounded to ten seconds — plus ten more for the consumers joining a queue they do not hold yet, and five for the rollback of a failed resize, each a budget of its own rather than the one that just expired — since a reload holds the settings store's lock while its hooks run; `MaxBytes` reports the budget last applied in full, so a failed resize is retried by the next reload. The consumers `CreateConsumer` and `Subscribe` build hold one durable on each tenant's stream, looked up before anything is written so a boot over many queues writes nothing it need not, each delivering on a goroutine of its own into the one handler. `PurgeAcked` and `DeadLetterCounts` run per tenant stream. `Stats` reports connection and inbound-message counters for `observability.RegisterSystemMetrics`. Trace context rides in the message headers: `Publish` applies `observability.InjectHeaders`, and a message delivered through `Subscribe` (the hub bridge) carries `observability.ExtractHeaders` on its `Ctx`; the worker's `Consumer` path skips the extraction, since it batches across messages and reads no per-message context. - **mqtest/** — The conformance suite for `Broker` (`mqtest.Run`): the behavior the rest of the process relies on — publish and consume round trips with names that need encoding, per-tenant order, redelivery, dead-lettering and its counts, replay bounds and isolation, the one `failed` report of a consumer whose delivery ends underneath it — checked through the interfaces alone, with no stream or subject name in sight. Each implementation runs it from a test of its own — the embedded one from `mqtest/embedded_test.go`, a test binary apart from `internal/mq`'s so the two share no 15s budget — handing it a fresh broker per case and flags (`mqtest.Caps`) for the few places where backends legitimately differ: whether a full queue refuses its own tenant alone, whether `PurgeAcked` removes anything, whether a tenant never given a budget has a dead-letter queue to report on, and whether `CreateConsumer` configures the durable or only finds one. ### `observability/` — OpenTelemetry Pipeline - **provider.go** — `InitProvider(ctx, serviceName, ProviderConfig)` wires the OTel pipeline. Each output is independently gated; the W3C TraceContext + Baggage propagator is always installed (cheap, harmless when traces are off). Returns `(shutdown, promHandler http.Handler, err)` — `promHandler` is non-nil only when `PrometheusEnabled` is true and reads from a *private* `prometheus.Registry` to avoid leaking the process/Go collectors that `prometheus.DefaultRegisterer` auto-registers. OTLP-metrics push (`MetricsEnabled`) and Prometheus exposition (`PrometheusEnabled`) are independent: either, both, or neither may be set, and any combination produces a single MeterProvider feeding the active readers. The Endpoint field is only dialed by the OTLP exporters (traces / metrics-OTLP / logs); Prometheus-only operation leaves it untouched. Provider init in `internal/app` runs whenever `otel.enabled` OR `prometheus.enabled` is true, so Prometheus-only operation (Alloy/scrape, no collector) is a first-class mode. - **logger.go** — `NewLogger(component, level, isJSON, otlpSampleRate)` produces a slog logger that fans out to stdout (always 100%) and the OTLP log exporter (DEBUG/INFO sampled at `otlpSampleRate`, WARN/ERROR always 100% as a non-configurable safety floor). `TraceHandler` injects `trace_id`/`span_id` from the active span when one exists. `otlpSamplerFn` is exposed (lowercase) for unit testing the per-level rate logic without driving through the slogmulti middleware. -- **metrics.go** — `RegisterSystemMetrics(mqStats, pebbleStats)` registers observable gauges for embedded NATS connections, in-msgs, and Pebble dedupe storage stats. Both are functions read on every scrape — `mqStats` a `func() (MQStats, error)` (`mq.Broker.Stats` in production; nil skips the MQ gauges), `pebbleStats` a `func() map[string]int64` (`dedupe.Embedded.Stats`, the one instance's figures; nil, or a nil map while it is closed, skips the Pebble gauges) — this package never holds the NATS server or a store. Wired in `internal/app` after the providers are up. +- **metrics.go** — `RegisterSystemMetrics(mqStats, pebbleStats)` registers observable gauges for the MQ's NATS connections and in-msgs (the embedded server's, or under `nats` this process's client connection), and Pebble dedupe storage stats. Both are functions read on every scrape — `mqStats` a `func() (MQStats, error)` (`mq.Broker.Stats` in production; nil skips the MQ gauges), `pebbleStats` a `func() map[string]int64` (`dedupe.Embedded.Stats`, the one instance's figures; nil, or a nil map while it is closed, skips the Pebble gauges) — this package never holds the NATS server or a store. Wired in `internal/app` after the providers are up. - **tracer.go** — W3C TraceContext propagation over message headers (`InjectHeaders` / `ExtractHeaders` on a plain `map[string][]string`, the shape NATS and HTTP headers share) — `internal/mq` injects on every publish and extracts onto the delivered `Message.Ctx` on the `Subscribe` path; no consumer reads it yet (the SSE hub bridge forwards the bytes and starts no span), and the ingest worker's `Consumer` path skips extraction entirely (see `embedded.go` above), so nothing downstream of the queue is linked to the originating request span. The package's design invariants — stdout always 100%, WARN+ERROR always export at 100%, gRPC exporters dial lazily so unreachable collectors never block startup, private Prometheus registry — are documented in AGENTS.md "Key Design Decisions" #15 and must be preserved by anything touching this package. diff --git a/docs/src/content/docs/configuration.mdx b/docs/src/content/docs/configuration.mdx index 8a89f6444..69dfdcc22 100644 --- a/docs/src/content/docs/configuration.mdx +++ b/docs/src/content/docs/configuration.mdx @@ -39,16 +39,57 @@ This page is boot config only — what the platform operator owns (wiring, lifec ### Backends -Each layer's implementation is chosen once, at boot. Every layer defaults to its in-process backend, so a config that sets none of these keys runs as it always has; the cache and dedupe also have a shared one each, `redis` and `dynamodb`. A value this build has no backend for refuses boot and names the valid ones. +Each layer's implementation is chosen once, at boot. Every layer's default is its in-process backend, so a config that sets none of these keys runs as it always has. The message queue and the coordination layer also have a shared backend, `nats`; so do the cache, `redis`, and dedupe, `dynamodb`. A value this build has no backend for refuses boot and names the valid ones. | YAML Key | Env Var | Default | Description | | --- | --- | ------- | ----------- | -| `mq.backend` | `WH_MQ_BACKEND` | `embedded` | The message queue. `embedded`: NATS JetStream inside this process, under `/nats`. It listens on no port, so no other process can reach its queue. | +| `mq.backend` | `WH_MQ_BACKEND` | `embedded` | The message queue. `embedded`: NATS JetStream inside this process, under `/nats`. It listens on no port, so no other process can reach its queue. `nats`: a NATS JetStream cluster you run, shared by every WaveHouse process that names it, configured by [`mq.nats`](#external-nats-mqnats); nothing is kept under `data_dir/nats`. | | `cache.backend` | `WH_CACHE_BACKEND` | `local` | The query-result cache. `local`: in this process, sized by `cache.l1_max_cost`. `redis`: one Redis-compatible server shared by every process, configured by [`cache.redis`](#cache), so an insert one process makes invalidates what every process has cached. | | `dedupe.backend` | `WH_DEDUPE_BACKEND` | `pebble` | Where ingest dedupe keeps the event ids it has seen. `pebble`: in this process, under `/pebble`, open while any tenant has dedupe on; two processes do not share seen ids. `dynamodb`: one DynamoDB table that every tenant and every process shares, configured by [`dedupe.dynamodb`](#dynamodb-dedupe). | -| `coord.backend` | `WH_COORD_BACKEND` | `local` | Where the leases for work only one process may do at a time, such as the sweeper, are held. `local`: in this process, so the one process always holds them. It shares nothing with another process, so every process runs its own sweeper. | +| `coord.backend` | `WH_COORD_BACKEND` | `local` | Where leases are held: the sweeper's on `mq.backend=embedded`, and the ingest processes' on `mq.backend=nats`. `local`: in this process, so the one process always holds them. It shares nothing with another process, so it serves one process on `mq.backend=embedded`, or a process without the `ingest` role. `nats`: a KV bucket you create on the `mq.nats` cluster, reached over the same connection and credentials, so every process sees the same leases; configured by [`coord.nats`](#nats-leases-coordnats). It needs `mq.backend=nats`. | -Settings for one backend go in a sub-block named after it, `.`, read only when that backend is selected. `cache.redis` and `dedupe.dynamodb` are the only ones so far; any other, `mq.embedded` included, is an unknown key and refuses boot. A `cache.redis.addrs` set while `cache.backend` is `local` is logged at `WARN` at boot, since the block is not read. `mq` and `dedupe` also appear in the settings directory's `config.json`, with different keys (`mq.max_bytes_gb`, `dedupe.enabled`, …): those are per-tenant tunables and stay there, and one written in `config.yaml` refuses boot as an unknown key. +Settings for one backend go in a sub-block named after it, `.`, read only when that backend is selected. `mq.nats`, `coord.nats`, `cache.redis` and `dedupe.dynamodb` are the only ones so far; any other, `mq.embedded` included, is an unknown key and refuses boot. A sub-block written while its layer runs another backend is not read. Boot warns about an ignored `mq.nats`, `coord.nats` or `cache.redis.addrs`; a `dedupe.dynamodb` block under `pebble` is ignored silently. `mq` and `dedupe` also appear in the settings directory's `config.json`, with different keys (`mq.max_bytes_gb`, `dedupe.enabled`, …): those are per-tenant tunables and stay there, and one written in `config.yaml` refuses boot as an unknown key. + +### External NATS (`mq.nats`) + +Read only with `mq.backend: nats`. WaveHouse connects to NATS you run and uses streams and durable consumers you create: it never creates, changes, purges or deletes one. [Deployment → External NATS](/deployment#external-nats) covers the cluster, the streams, the user's permissions and the gauges. Secrets are file paths only, such as a mounted Kubernetes Secret; no key takes a secret inline, and `mq.nats.password` or `WH_MQ_NATS_PASSWORD` refuses boot as an unknown key. + +| YAML Key | Env Var | Default | Description | +| --- | --- | ------- | ----------- | +| `mq.nats.urls` | `WH_MQ_NATS_URLS` | *(none)* | Required. The servers to dial: a YAML list, or a comma-separated variable. A URL carrying credentials (`user:password@` or `token@`) refuses boot. | +| `mq.nats.name` | `WH_MQ_NATS_NAME` | `wavehouse-` | The connection name the server reports. | +| `mq.nats.creds_file` | `WH_MQ_NATS_CREDS_FILE` | *(empty)* | A `.creds` file (user JWT and nkey seed), for decentralized auth. | +| `mq.nats.nkey_seed_file` | `WH_MQ_NATS_NKEY_SEED_FILE` | *(empty)* | An nkey seed file. | +| `mq.nats.user` | `WH_MQ_NATS_USER` | *(empty)* | A user name, with `password_file`. | +| `mq.nats.password_file` | `WH_MQ_NATS_PASSWORD_FILE` | *(empty)* | The file holding `user`'s password; a trailing newline is dropped. Refused without `user`. | +| `mq.nats.tls.ca_file` | `WH_MQ_NATS_TLS_CA_FILE` | *(empty)* | CA bundle for the servers' certificates. | +| `mq.nats.tls.cert_file` | `WH_MQ_NATS_TLS_CERT_FILE` | *(empty)* | A client certificate, for mutual TLS. It comes as a pair with `key_file`. | +| `mq.nats.tls.key_file` | `WH_MQ_NATS_TLS_KEY_FILE` | *(empty)* | The client certificate's private key. It comes as a pair with `cert_file`. | +| `mq.nats.tls.server_name` | `WH_MQ_NATS_TLS_SERVER_NAME` | *(empty)* | The name to verify the servers' certificates against, when it is not the host dialed. | +| `mq.nats.tls.handshake_first` | `WH_MQ_NATS_TLS_HANDSHAKE_FIRST` | `false` | Start TLS before the NATS protocol, for servers that require it. | +| `mq.nats.js_domain` | `WH_MQ_NATS_JS_DOMAIN` | *(empty)* | The JetStream domain, for a leafnode or hub-and-spoke deployment. WaveHouse then sends its JetStream API requests and lease writes under `$JS..API.`; generate the user's permissions with `wavehouse mq permissions --js-domain ` ([Permissions](/deployment#permissions)). | +| `mq.nats.subject_prefix` | `WH_MQ_NATS_SUBJECT_PREFIX` | `wh` | Leads every subject WaveHouse uses (`.ingest.…`, `.hist.…`, `.dlq.…`). One token of `[a-z0-9_-]`. | +| `mq.nats.partitions` | `WH_MQ_NATS_PARTITIONS` | `1` | How many ingest partition streams there are. It must equal the number you created; boot finds each one by its subject. | +| `mq.nats.shards` | `WH_MQ_NATS_SHARDS` | `32` | How many shards each partition has: a table's events all go to one shard, by a consistent hash of the tenant and table, and each shard has a durable of its own that one ingest process at a time consumes. It must equal the number you created (`wavehouse mq manifests --shards`); from 1 to 256. It caps the ingest processes that get work at partitions × shards, and at 64. | +| `mq.nats.ingest_consumer` | `WH_MQ_NATS_INGEST_CONSUMER` | `wh-ingest` | Names the shard durables: shard `s` of every partition is `-`, unpadded. Generate them with `wavehouse mq manifests --ingest-consumer`. | +| `mq.nats.history_stream` | `WH_MQ_NATS_HISTORY_STREAM` | `_HISTORY` | The history stream, which SSE replay and the live hub read. WaveHouse looks it up by this name; the partition and dead-letter streams are found by subject and can have any name. The default is the upper-cased prefix, `WH_HISTORY` for `wh`, as `wavehouse mq manifests` names it; `--history-stream` names another. | +| `mq.nats.connect_timeout` | `WH_MQ_NATS_CONNECT_TIMEOUT` | `5s` | Bounds one dial. | +| `mq.nats.publish_timeout` | `WH_MQ_NATS_PUBLISH_TIMEOUT` | `5s` | Bounds one publish attempt. A publish is tried at most three times; a partition's `duplicate_window` must cover all three, or boot refuses. `wavehouse mq manifests --publish-timeout` sizes the window for it. | +| `mq.nats.topology_wait` | `WH_MQ_NATS_TOPOLOGY_WAIT` | `60s` | How long boot waits for the cluster and for your streams and consumers to be right. Boot then refuses with every finding at once. | + +Boot refuses a `nats` block with no URLs, a URL with credentials in it, more than one of `creds_file`, `nkey_seed_file` and `user`, half a certificate pair, a prefix outside the grammar, fewer than one partition, a shard count outside 1 to 256, an `ingest_consumer` or `history_stream` outside `[a-zA-Z0-9_-]`, or a timeout that is not positive. Durations take Go syntax (`5s`, `2m`). + +A tenant's [`mq.max_bytes_gb`](/settings-directory#message-queue) is not applied under `nats`: its events share a partition stream with other tenants, and that stream's limits, which you set, bound them. Boot logs a warning saying so. + +### NATS leases (`coord.nats`) + +Read only with `coord.backend: nats`. It has no connection settings: the leases ride the [`mq.nats`](#external-nats-mqnats) connection, which is why `coord.backend=nats` refuses boot without `mq.backend=nats`. WaveHouse never creates the bucket. Boot waits for it with the rest of your topology (`mq.nats.topology_wait`) and then refuses, naming it; [Deployment → External NATS](/deployment#external-nats) shows how to create it. + +| YAML Key | Env Var | Default | Description | +| --- | --- | ------- | ----------- | +| `coord.nats.bucket` | `WH_COORD_NATS_BUCKET` | `_coord` | The KV bucket the leases live in, one key per lease. The default follows `mq.nats.subject_prefix` (`wh_coord` for `wh`), as `wavehouse mq manifests` names it, so two deployments sharing one NATS account under different prefixes never contend for one lease. To use another name, generate the bucket with `wavehouse mq manifests --coord-bucket ` and change the `wavehouse` user's permissions in `values.yaml` to match. A name outside `[a-zA-Z0-9_-]` refuses boot. | + +A lease is taken over only after its holder has stopped renewing it: a process that wants it must see the same revision unchanged for 15 seconds on its own clock, so no two servers' clocks are compared. The holder renews every 2 seconds and steps down after 10 seconds without a successful renewal, before anyone can take over. A process that stops cleanly deletes its lease, so the next holder takes over at its next attempt (within 2 seconds). The bucket must not expire keys (`ttl` unset), because a lease that expires on the server's clock can end under a live holder. ### Dedupe @@ -56,7 +97,7 @@ Whether a tenant dedupes, and on which field, are settings-directory keys ([Dedu | YAML Key | Env Var | Default | Description | | --- | --- | ------- | ----------- | -| `dedupe.lease` | `WH_DEDUPE_LEASE` | `30s` | How long a record's id stays claimed while the record is published. Another request carrying the same id meanwhile gets `503` with this as `Retry-After`, in whole seconds; a claim that is neither committed nor released, because its process died mid-publish, lapses after it. With `mq.backend: embedded`, the lease plus its own ceiling to the next whole second plus one more second must fit the embedded queue's 2-minute duplicate window, so the lease is at most `59s`: a client that obeys `Retry-After` after a publish whose outcome it never learned can republish as late as the lease plus that ceiling plus a second after the claim, since DynamoDB rounds a claim's expiry up to the second. A longer lease refuses boot. A Go duration (`30s`, `45s`); `0` refuses boot. | +| `dedupe.lease` | `WH_DEDUPE_LEASE` | `30s` | How long a record's id stays claimed while the record is published. Another request carrying the same id meanwhile gets `503` with this as `Retry-After`, in whole seconds; a claim that is neither committed nor released, because its process died mid-publish, lapses after it. With `mq.backend: embedded`, the lease plus its own ceiling to the next whole second plus one more second must fit the embedded queue's 2-minute duplicate window, so the lease is at most `59s`: a client that obeys `Retry-After` after a publish whose outcome it never learned can republish as late as the lease plus that ceiling plus a second after the claim, since DynamoDB rounds a claim's expiry up to the second. A longer lease refuses boot. Under `mq.backend: nats` there is no cap here; instead every partition's `duplicate_window` must cover the same span, or boot refuses with a topology finding ([Deployment](/deployment#create-the-topology)). A Go duration (`30s`, `45s`); `0` refuses boot. | | `dedupe.reserve_concurrency` | `WH_DEDUPE_RESERVE_CONCURRENCY` | `64` | The most parallel calls one Reserve, Commit or Release makes to a remote dedupe backend, and the idle connections per host the DynamoDB client keeps to match, never fewer than the SDK's own default (10). Ingest reserves and commits a window of up to 256 ids per call; `pebble` ignores it. `0` refuses boot. | #### DynamoDB dedupe @@ -73,6 +114,18 @@ Read only when `dedupe.backend` is `dynamodb`. Credentials come from the AWS SDK | `dedupe.dynamodb.retry_mode` | `WH_DEDUPE_DYNAMODB_RETRY_MODE` | `standard` | `standard`, or `adaptive`, which also slows the client down after throttling. Anything else, empty included, refuses boot. | | `dedupe.dynamodb.create_table` | `WH_DEDUPE_DYNAMODB_CREATE_TABLE` | `false` | Development only: create the table at boot if it is missing, with TTL on `ex`. Refused unless `endpoint` is set, so it never creates a table in AWS; the production table belongs to your infrastructure code. | +### Boot warnings + +Boot logs each of these at `WARN` rather than refusing. The first two are right for a single replica only, which one process cannot tell from many; the last depends on a window only your streams know: + +- **`mq.backend=nats` with `cache.backend=local`**, in a process running `api`: an event ingested on another replica never invalidates this one's cache, so its reads stay stale until the cached entry expires. +- **`mq.backend=nats` with `dedupe.backend=pebble`**, in a process running `api`: an id seen by another replica is not seen by this one. +- **`mq.backend=nats`**: `mq.max_bytes_gb` is not applied (above). +- **`mq.nats` set with `mq.backend=embedded`**, or **`coord.nats` set with `coord.backend=local`**: the block is ignored. +- **`cache.redis.addrs` set with `cache.backend=local`**, in a process running `api`: the block is ignored. +- **`cache.redis.tls.insecure_skip_verify` on**, in a process running `api`: the cache accepts any certificate (see [Cache](#cache)). +- **`mq.backend=nats` and a tenant with dedupe on whose finite `dedupe.retention` is under the partitions' `duplicate_window`**, in a process running `api`, at boot and after every reload: see [Deployment → Create the topology](/deployment#create-the-topology). + ### Process roles By default one process does all the work. `roles` splits it, so that the API and the background workers can run in separate processes, for example one Kubernetes Deployment per role (see [Deployment](/deployment#one-deployment-per-role)). The binary and its entry point are the same for every role; only this key differs. @@ -80,20 +133,23 @@ By default one process does all the work. `roles` splits it, so that the API and | YAML Key | Env Var | Default | Description | | --- | --- | ------- | ----------- | | `roles` | `WH_ROLES` | `api,ingest,sweeper` | The roles this process runs: a YAML list, or a comma-separated variable. Order does not matter. An empty list, an empty entry, an unknown role, or a role named twice refuses boot. | -| `instance_id` | `WH_INSTANCE_ID` | *(empty)* | Names this process. Today it is only logged at boot (the `process roles` line); once a shared `coord.backend` exists, it names this process as the holder of a lease. Empty resolves at boot to `-<8 hex>`, with a fresh random suffix every time, so a restarted process is a new instance. | +| `instance_id` | `WH_INSTANCE_ID` | *(empty)* | Names this process: it is logged at boot (the `process roles` line), and under `coord.backend=nats` it is the `holder` a lease's value names, so you can see which process holds a lease. Empty resolves at boot to `-<8 hex>`, with a fresh random suffix every time, so a restarted process is a new instance. | | Role | Runs | | --- | --- | | `api` | The HTTP API, and what answers it: schema discovery, the token verifiers and their JWKS refresh, the dedupe stores, and the SSE hub with its bridge off the queue and its keepalive wheel. Every API process runs its own set of these, and each API process receives every event for its own SSE clients. | -| `ingest` | The ingest worker, which writes the queue to ClickHouse. Every ingest process consumes the same shared durable consumer and competes for its messages. | -| `sweeper` | The sweeper, which purges messages that are written and older than their tenant's gap window. It runs under the `sweeper` lease. With a shared [`coord.backend`](#backends), only one process sweeps at a time, however many run the role; with `local`, each process holds its own lease. | +| `ingest` | The ingest worker, which writes the queue to ClickHouse. Under `mq.backend=nats` the ingest processes share the queue's shards out through membership leases in `coord.nats.bucket`, one process per shard at a time, so a table's rows are written by one process. | +| `sweeper` | The sweeper, which purges messages that are written and older than their tenant's gap window. It runs under the `sweeper` lease, on `mq.backend=embedded` only. Under `mq.backend=nats` your streams' retention does its work, so it is not wired there, and a process whose only role is `sweeper` is refused (below). A tenant whose gap window is longer than the history stream keeps is checked by the API processes instead, at boot and after each reload, with one warning per tenant and window. | -Every process, whatever its roles, reads the settings directory and reloads it (SIGHUP, the directory watcher, and the reload route), and serves `server.port`. A process without the `api` role serves only an ops listener there: `/livez`, `/readyz` and their `/healthz`, `/health`, `/ready` aliases, `/version`, the metrics path when `prometheus.port` is `0`, and `POST /v1/ops/settings/reload`. Every other route answers 404; under `/v1/ops`, only once the operator-key check has passed (403 without a credential; a bearer token is refused with 401, since no token verifier runs without the `api` role). The reload route on that listener accepts only the [operator key](#authentication), because no token verifier runs without the `api` role. `/readyz` is ready when a ClickHouse pool answers in an `ingest` process, and as soon as the process has booted in a `sweeper`-only one. +Every process, whatever its roles, reads the settings directory and reloads it (SIGHUP, the directory watcher, and the reload route), and serves `server.port`. A process without the `api` role serves only an ops listener there: `/livez`, `/readyz` and their `/healthz`, `/health`, `/ready` aliases, `/version`, the metrics path when `prometheus.port` is `0`, and `POST /v1/ops/settings/reload`. Every other route answers 404; under `/v1/ops`, only once the operator-key check has passed (403 without a credential; a bearer token is refused with 401, since no token verifier runs without the `api` role). The reload route on that listener accepts only the [operator key](#authentication), because no token verifier runs without the `api` role. `/readyz` is ready when a ClickHouse pool answers in an `ingest` process. Boot refuses a role set the selected backends cannot serve: -- **Any split with `mq.backend=embedded`.** The embedded queue lives inside its process and listens on no port, so a process without every role could not reach it. Until a shared `mq.backend` exists, every process runs every role. -- **`api` without `ingest`, or `ingest` without `api`, with `cache.backend=local`.** The ingest worker invalidates the cache the API reads, and a local cache in another process never sees that invalidation. Run `api` and `ingest` together, or set [`cache.backend: redis`](#backends), one cache every process shares. A `sweeper`-only process holds no cache, so this rule does not apply to it. +- **Any split with `mq.backend=embedded`.** The embedded queue lives inside its process and listens on no port, so a process without every role could not reach it. Choose `mq.backend=nats` to split. +- **`ingest` with `mq.backend=nats` and `coord.backend=local`.** The ingest processes share the queue through leases, so each needs the shared coordinator. Set `coord.backend=nats`. A process without the `ingest` role holds no lease, so it may keep `local`. +- **Only `sweeper` with `mq.backend=nats`.** The sweeper is not wired there, so the process would run nothing. Remove it. +- **`coord.backend=nats` without `mq.backend=nats`.** The leases ride the NATS connection. +- **`api` without `ingest`, or `ingest` without `api`, with `cache.backend=local`.** The ingest worker invalidates the cache the API reads, and a local cache in another process never sees that invalidation. Run `api` and `ingest` together, or set [`cache.backend: redis`](#backends), one cache every process shares. ### Server @@ -157,7 +213,7 @@ WaveHouse's per-role caps are sent as per-query `SETTINGS` on its connection, so ### Message Queue (NATS) -This section describes the `embedded` [backend](#backends), the only one today. +This section describes the `embedded` [backend](#backends). `mq.backend: nats` takes its settings from [`mq.nats`](#external-nats-mqnats) instead. Each tenant's queue has its own disk budget, `mq.max_bytes_gb`, a hot-reloadable key in the [Settings Directory](/settings-directory#message-queue) — there is no boot-config knob for it. @@ -284,6 +340,13 @@ clickhouse: mq: backend: embedded # in-process NATS JetStream under /nats + # With backend: nats, the operator's cluster instead (read only then): + # nats: + # urls: ["nats://nats.nats.svc:4222"] + # user: wavehouse + # password_file: /var/run/secrets/nats/password + # partitions: 4 # the number of ingest partition streams + # shards: 8 # shards per partition, as generated (default 32) cache: backend: local # local | redis @@ -323,6 +386,9 @@ dedupe: coord: backend: local # in-process leases (the sweeper's) + # With backend: nats (needs mq.backend: nats; read only then): + # nats: + # bucket: "" # empty = _coord auth: jwt_secret: change-me-in-production # jwks_url and role_claim are settings (config.json) @@ -374,6 +440,12 @@ WH_CH_PASSWORD= WH_CH_MAX_TOTAL_CONNS=0 WH_MQ_BACKEND=embedded +# With WH_MQ_BACKEND=nats (read only then): +# WH_MQ_NATS_URLS=nats://nats.nats.svc:4222 +# WH_MQ_NATS_USER=wavehouse +# WH_MQ_NATS_PASSWORD_FILE=/var/run/secrets/nats/password +# WH_MQ_NATS_PARTITIONS=4 +# WH_MQ_NATS_SHARDS=8 WH_CACHE_BACKEND=local WH_CACHE_L1_MAX_COST=67108864 # Read only with WH_CACHE_BACKEND=redis; WH_CACHE_REDIS_ADDRS is then required. @@ -406,6 +478,8 @@ WH_DEDUPE_RESERVE_CONCURRENCY=64 # WH_DEDUPE_DYNAMODB_RETRY_MODE=standard # WH_DEDUPE_DYNAMODB_CREATE_TABLE=false WH_COORD_BACKEND=local +# With WH_COORD_BACKEND=nats (read only then; empty = _coord): +# WH_COORD_NATS_BUCKET= WH_AUTH_JWT_SECRET=change-me-in-production WH_AUTH_OPERATOR_KEY= diff --git a/docs/src/content/docs/deployment.md b/docs/src/content/docs/deployment.md index 9c8029a44..8bcc248d4 100644 --- a/docs/src/content/docs/deployment.md +++ b/docs/src/content/docs/deployment.md @@ -11,7 +11,7 @@ How to run WaveHouse in production — single binary, Docker images, releases, h ## Single binary -WaveHouse runs as one process with embedded NATS and optional Pebble dedup. The only external dependency is ClickHouse. +WaveHouse runs as one process with embedded NATS and optional Pebble dedup. The only external dependency is ClickHouse, unless you select a shared backend: [`mq.backend: nats`](#external-nats) (and `coord.backend: nats`), [`cache.backend: redis`](#multiple-instances-and-the-shared-cache) or [`dedupe.backend: dynamodb`](#a-shared-dedupe-table-on-dynamodb). ### Quick Start with Docker Compose @@ -181,7 +181,7 @@ In a Docker / Podman / Kubernetes deployment, **`data_dir` must resolve to a hos If `data_dir` resolves into the container's writable overlay layer instead, **JetStream state is wiped on every restart**: in-flight events are lost, gap-fill stops bridging restarts, and disk usage accumulates inside `/var/lib/docker` instead of the volume the operator chose. -Beyond persistence, the *speed* of that volume matters: JetStream `fsync`s every event to `/nats` before the ingest endpoint returns `200`, so the volume's `fsync` latency is your ingest latency floor. Managed cloud block storage handles this without thinking; commodity or virtualized substrates (ZFS without a SLOG, qcow2-on-`ext4`, spinning disks) can stall ingest with multi-second `fsync` tails. See [Durability & Storage](/durability) to measure yours before going live. +Beyond persistence, the *speed* of that volume matters: the embedded broker (`mq.backend: embedded`, the default) `fsync`s every event to `/nats` before the ingest endpoint returns `200`, so the volume's `fsync` latency is your ingest latency floor. Managed cloud block storage handles this without thinking; commodity or virtualized substrates (ZFS without a SLOG, qcow2-on-`ext4`, spinning disks) can stall ingest with multi-second `fsync` tails. See [Durability & Storage](/durability) to measure yours before going live. Under [`mq.backend: nats`](#external-nats) the events are not under `data_dir`, and WaveHouse does not require an `fsync` per event: see [Durability](#durability) there. WaveHouse runs a simple existence check on startup and logs a `WARN` if `/nats` (or `/pebble`, when dedupe is on with the `pebble` backend) is missing or empty: @@ -333,6 +333,157 @@ A second `SIGTERM`/`SIGINT` while the stop is running abandons it and exits non- Size the orchestrator's kill grace at `server.shutdown_timeout` plus 8s: at the default a stop needs up to 18s before it should be `SIGKILL`ed, and raising the timeout raises that total by the same amount. Docker's default `stop_grace_period` is 10s, so the [compose file](https://github.com/Wave-RF/WaveHouse/blob/main/deployments/compose/standalone.yaml) sets `stop_grace_period: 25s`, that bound plus headroom; on Kubernetes the equivalent is `terminationGracePeriodSeconds`, whose 30s default already covers it — raise it if you raise `server.shutdown_timeout`. A stop with nothing in flight takes well under a second either way, unless OTLP export is on and the collector is unreachable: the flush then waits out its 3s. +## External NATS + +With `mq.backend: nats`, WaveHouse's message queue is a NATS JetStream cluster you run, shared by every WaveHouse process that points at it. This is what makes more than one replica, or a [split by role](#one-deployment-per-role), possible. **WaveHouse never creates, changes, purges or deletes a stream, a durable consumer or a KV bucket there.** You create them, WaveHouse checks them at boot, and it refuses to start until they are right. The only objects WaveHouse creates are short-lived consumers on the history stream, one per API process for its live SSE events and one per SSE replay, which the server removes on its own when they are idle. + +### What WaveHouse needs + +- **N ingest partition streams.** Partition `p` holds `.ingest.

.>`, and each row's subject is `.ingest.

...

`, where `s` is one of its V shards. It has work-queue retention: a row stays until the ingest worker has written it and acked it, even while no consumer covers it, and is deleted then, so one tenant whose ClickHouse is down keeps only its own rows on disk. Each partition also republishes every row it stores to `.hist..
`, the same subject without the partition and shard, where the history stream captures it. A work queue takes no consumer beside the ingest durables whose subjects overlap theirs, so a debugging consumer on a partition is refused; read the history stream, or get a row directly, instead. A table's events always go to the same partition and shard, chosen by a consistent (jump) hash of the tenant and table: the tenant's other tables spread over the partitions, and raising N to N′ moves (N′−N)/N′ of the tables, each into one of the new partitions (about 1/N′ for one more), and lowering it moves only the removed partitions' tables; V the same. Each partition has no age limit (an age limit would drop rows not yet written), and `discard: new` with a byte limit: a full partition refuses new events with `503` and `Retry-After: 30`, for every tenant in it. +- **A durable per shard on every partition,** `wh-ingest-` for shard `s` (unpadded, so a name never changes when V grows), filtering `.ingest.

..>`, with `priority_policy: pinned_client` in the priority group `wavehouse` and a pinned TTL (`priority_timeout`) of at least 10 seconds, twice the 5 seconds between the pulls that keep the pin of a shard at its share of held rows, or stopping. The ingest processes share the shards out (see [Scaling ingest processes](#scaling-ingest-processes)), and the server delivers a shard to one puller at a time, the one it pinned; the pin lapses once its holder stops pulling for the TTL, and a holder that stops cleanly releases it once it has written what it holds, rather than letting it lapse. +- **The history stream,** which holds `.hist.>` and has no sources: the partitions' republish is what writes it. SSE replay (`Last-Event-ID`) and every API process's live events read from it. It is best effort: a row republished while the history is unavailable (deleted, or electing a leader) reaches ClickHouse but never SSE, and a full or missing history never holds up ingest. Its `max_age` is how far back a replay can reach, so set it to at least the longest [`stream.gap_window_minutes`](/settings-directory#streaming) among the tenants you serve. The generated manifests use `15m`, the seed's default window; the API processes check at boot and after each reload, and warn once per tenant and window (again only if that tenant's window changes) for each tenant whose window is longer. +- **One dead-letter stream** holding `.dlq.>`, shared by every tenant. +- **The lease bucket,** a KV bucket named `_coord` (`wh_coord`; [`coord.nats.bucket`](/configuration#nats-leases-coordnats) names another), where [`coord.backend: nats`](/configuration#backends) holds the ingest processes' leases. Every process running the `ingest` role needs it, because `mq.backend: nats` refuses `coord.backend: local` there. Keep one value per key (`history: 1`), allow direct gets (`allow_direct`, which nack and `nats kv add` always set, because the `wavehouse` user reads leases only that way), and set no `ttl`: a lease expires on its candidates' clocks, and a key the server expires would end a live holder's lease. Boot checks it only in a process with `coord.backend: nats`, and refuses while it is missing. + +### Create the topology + +1. **Run NATS 2.14 or later** with JetStream on file storage. The shards need pinned-client priority groups and unpinning, which arrived in 2.11, and a consumer reset to its ack floor, which arrived in 2.14; boot refuses an older server. 2.14.x, the line WaveHouse embeds, is recommended, and boot warns on a newer line. [`deployments/nats/values.yaml`](https://github.com/Wave-RF/WaveHouse/blob/main/deployments/nats/values.yaml) is a values file for the [NATS Helm chart](https://github.com/nats-io/k8s): a three-node cluster with one account and two users, `nack` for the JetStream controller and `wavehouse` for WaveHouse, whose passwords come from a `nats-users` Secret. +2. **Generate the streams, consumers and lease bucket** as [nack](https://github.com/nats-io/nack) resources (nack's `KeyValue` needs its control-loop mode): + + ```bash + wavehouse mq manifests --partitions 4 --shards 32 --prefix wh --replicas 3 > jetstream.yaml + ``` + + [`deployments/nats/jetstream.yaml`](https://github.com/Wave-RF/WaveHouse/blob/main/deployments/nats/jetstream.yaml) is its output for four partitions of eight shards (`--partitions 4 --shards 8`). Pass the names and timeout you configure: `--ingest-consumer` for `mq.nats.ingest_consumer`, `--history-stream` for `mq.nats.history_stream`, `--publish-timeout` for `mq.nats.publish_timeout` (the partitions' duplicate window follows it), and `--coord-bucket` for `coord.nats.bucket`. The streams' byte limits are sized for the servers' file store, `--file-store` (see [Sizing the file store](#sizing-the-file-store)); the other sizes (the history's `maxAge`, `maxMsgsPerSubject`) are starting points: tune them before you apply. +3. **Generate the `wavehouse` user's permissions for the same V**, and replace the user's `permissions` block in `values.yaml` with the output (see [Permissions](#permissions)). The shipped values cover eight shards. Boot probes every shard durable as the `wavehouse` user and refuses to start while the user may not pull from one, so permissions generated for a smaller V stop boot rather than leave shards unpulled: + + ```bash + wavehouse mq permissions --shards 32 --prefix wh + ``` + +4. **Apply the values and the manifests,** in any order: WaveHouse publishes nothing until all of them pass its boot check. A partition keeps every row until the ingest worker acks it, so a missing durable only delays rows. The history stream should exist before events flow if you want SSE to see them: rows republished before it exists reach ClickHouse but not SSE. +5. **Start WaveHouse** with `mq.backend: nats`, `coord.backend: nats` and the [`mq.nats`](/configuration#external-nats-mqnats) block: the server URLs, the `wavehouse` user and a mounted password file, and `partitions` and `shards` equal to the N and V you generated. Boot waits up to `mq.nats.topology_wait` (60s) for the cluster and your resources, because on Kubernetes they may roll out together, then refuses to start and logs every finding at once. A finding marked `recommended` is logged and does not stop boot. + +The generated manifests satisfy every required finding (pass `--dedupe-lease` when your `dedupe.lease` is not the default). Some you may meet when you write your own: + +- A partition must use `retention: workqueue` and republish to the history: `republish: {source: .ingest.

.*.>, destination: .hist.>}`, which drops the partition and shard tokens. The history must hold exactly `.hist.>` and have no sources. `discard: old` is recommended for it, so a full history keeps the newest rows for SSE; either way it never holds up ingest. +- A partition's `duplicate_window` must cover every attempt of one publish: three times `mq.nats.publish_timeout`, plus half a second. A publish that got no answer is retried with the same message id, so the partition stores it once. The generated `2m` covers the default `5s`; `wavehouse mq manifests --publish-timeout ` widens it for a longer timeout. +- It must also cover [`dedupe.lease`](/configuration#dedupe) twice over, plus a second: the lease, the lease rounded up to whole seconds, and one more second (61s for the default 30s). With dedupe on, a publish whose outcome was unknown keeps its id claimed until the lease lapses, and a client obeying `Retry-After` republishes it as late as that under the same idempotency key; the partition drops the copy only while it still remembers the first. The shipped `2m` covers any lease up to `59s`; for a longer one, `wavehouse mq manifests --dedupe-lease ` widens it. +- A tenant with dedupe on whose finite [`dedupe.retention`](/settings-directory#deduplication), for the tenant or one of its tables, is shorter than the partitions' `duplicate_window` is logged at `WARN`, at boot and after every reload. An id re-sent after its retention but inside the window would be claimed again and then dropped by the partition, while the client is told it was accepted. The settings directory refuses a retention under `2m`, the embedded queue's window, but cannot see yours: keep retention at least as long as the window, or `"0"`. +- Each shard durable needs `max_expires` unset or at least 5 seconds. WaveHouse's pulls wait at most a second today; the floor leaves room for longer ones, since the server refuses a pull that asks for more than `max_expires`. +- Each shard durable needs `max_deliver: -1`. With a limit, a row that failed that many times would stay on its partition and never be delivered again. +- Each shard durable must filter exactly its shard's subjects and use `pinned_client` in the group `wavehouse`, with a pinned TTL of at least 10 seconds. The server renews a pin only when its holder sends a new pull, so a shorter TTL lets a live holder that is at its share of held rows, or stopping, lose its pin between its 5-second renewals. A TTL of 15 seconds or more is recommended against: a dead holder's shard is received by another process only once its pin lapses. + +WaveHouse checks the topology again every five minutes and never repairs it. If you delete a partition, its publishes answer `503` with `Retry-After: 5`. If you delete one of the N×V configured shard durables (`wh-ingest-0` to `wh-ingest-` on each of the N partitions), or the connection is closed for good (for example, its credentials are revoked), the ingest worker ends and the process exits, so that the orchestrator restarts it and the next boot names what is missing. An ingest worker that stayed up without its queue would leave the API accepting events that nothing writes. + +### Sizing the file store + +Each server reserves the `maxBytes` of every stream replica it holds against its JetStream file store, `max_file_store`, and refuses a stream that does not fit (JetStream error `10047`, insufficient storage resources). With three replicas on three servers every server holds a replica of every stream, so the streams' `maxBytes` together must fit in each server's store. The NATS Helm chart sets `max_file_store` to `config.jetstream.fileStore.maxSize`, else to the JetStream PVC's size: `100Gi` in the shipped [`values.yaml`](https://github.com/Wave-RF/WaveHouse/blob/main/deployments/nats/values.yaml). + +`wavehouse mq manifests --file-store ` (default `100Gi`) sizes the streams for that store: each partition gets 15% of it, the history 10% and the dead-letter stream 5%. The shipped four partitions reserve `75Gi` of the `100Gi`, leaving a quarter for what else the store holds (the Raft logs of a replicated stream, the lease bucket, the store's own overhead). A partition's size does not follow N, so lowering N never needs more store while the removed partitions drain. The generator refuses streams that together reserve more than `--file-store`: from six partitions at the defaults, set `--partition-max-bytes` or a larger store. If you change the PVC, pass its size as `--file-store` and apply the regenerated manifests. + +### Durability + +The two backends make a `200` from `POST /v1/ingest` durable in different ways: + +- **Embedded** (`mq.backend: embedded`): the one JetStream server `fsync`s every event to `/nats` before the `200`. There is one copy, so only the disk stands behind it. +- **External NATS** (`mq.backend: nats`): the `200` comes after the partition stream acks the publish, and a stream with 3 or more replicas acks only once a Raft quorum of its servers has stored the event. WaveHouse does not require `sync_always` on your servers, and [`values.yaml`](https://github.com/Wave-RF/WaveHouse/blob/main/deployments/nats/values.yaml) does not set it: an acked event survives the loss of any server short of a quorum, so its durability comes from placing the replicas in separate failure domains (zones, racks or hosts), not from each server's disk. Losing a quorum's servers at once, before they sync, can lose events they acked. +- **External NATS at one replica:** no second copy exists, so the server's `sync_interval` (2 minutes unless you set it) governs. Events stored since the last sync can be lost if that server crashes. Boot logs a `recommended` finding for every partition, history or dead-letter stream with fewer than 3 replicas, and still starts, so a one-server development cluster works. For production, generate the manifests with `--replicas 3` (the default) on a cluster whose servers do not share a failure domain, or set `sync_always` on a one-server cluster and accept the per-event `fsync` cost that [Durability & Storage](/durability) describes. `sync_always` does not reach a stream with `persist_mode: async`, which flushes in the background, so boot refuses that mode on an ingest partition and recommends against it on the dead-letter stream. + +### Permissions + +The `wavehouse` user in `values.yaml` has exactly what WaveHouse needs: it can publish to its subjects, read stream and consumer info and list consumer names, pull from, unpin and reset each shard durable and ack the rows it delivers, create, pull from and delete consumers on the history stream, and read and write the `lease.` keys in the lease bucket (a KV write is a publish to the key's subject, and a read is a direct get). It cannot create, change, purge or delete a stream or a bucket, nor create a durable on a partition, nor ack for another consumer, nor touch another key, nor publish to the history: only the partitions' republish writes `wh.hist.>`. The permissions are written for the default prefix `wh`, history stream `WH_HISTORY`, durables `wh-ingest-` with eight shards, and bucket `wh_coord`. They name every shard's durable one by one, because a NATS wildcard is a whole token (`wh-ingest-*` names no durable), so print them for your settings with `wavehouse mq permissions --shards ` (and `--prefix`, `--ingest-consumer`, `--history-stream`, `--coord-bucket` and `--js-domain` as you set them), and paste the block into the user's entry. For two shards: + +```yaml +publish: + allow: [wh.ingest.>, wh.dlq.>, $JS.API.INFO, $JS.API.STREAM.NAMES, $JS.API.STREAM.INFO.*, + $JS.API.CONSUMER.INFO.*.*, $JS.API.CONSUMER.NAMES.*, + $JS.API.CONSUMER.MSG.NEXT.*.wh-ingest-0, $JS.API.CONSUMER.MSG.NEXT.*.wh-ingest-1, + $JS.API.CONSUMER.UNPIN.*.wh-ingest-0, $JS.API.CONSUMER.UNPIN.*.wh-ingest-1, + $JS.API.CONSUMER.RESET.*.wh-ingest-0, $JS.API.CONSUMER.RESET.*.wh-ingest-1, + $JS.ACK.*.wh-ingest-0.>, $JS.ACK.*.wh-ingest-1.>, + $JS.ACK.*.*.*.wh-ingest-0.>, $JS.ACK.*.*.*.wh-ingest-1.>, + $JS.API.CONSUMER.CREATE.WH_HISTORY.>, $JS.API.CONSUMER.MSG.NEXT.WH_HISTORY.>, + $JS.API.CONSUMER.DELETE.WH_HISTORY.>, $KV.wh_coord.lease.>, + $JS.API.DIRECT.GET.KV_wh_coord.$KV.wh_coord.lease.>] + deny: [wh.hist.>, $JS.API.STREAM.CREATE.>, $JS.API.STREAM.UPDATE.>, $JS.API.STREAM.DELETE.>, + $JS.API.STREAM.PURGE.>, $JS.API.CONSUMER.DURABLE.CREATE.>] +subscribe: + allow: [_INBOX_wh.>] +``` + +An ack goes to the subject the server delivered the row with, which names the stream and the consumer: `$JS.ACK...…` by default, and `$JS.ACK.....…` on a server with the `js_ack_fc_v2` feature flag. The two `$JS.ACK` entries per shard cover both layouts. WaveHouse's history consumers ack nothing. + +Boot checks the permissions against the topology: it sends each shard durable a pull request and an unpin request that the server rejects on their merits (a pull whose heartbeat is more than half its expiry, an unpin naming no priority group), so neither delivers a row nor moves a pin, and a `required` finding names every durable whose request the server refused, with the `wavehouse mq permissions` command to regenerate them. The check runs again every five minutes. A consumer request the server refuses at any other time is logged as an error and sets `wavehouse_mq_topology_ok` to `0` at once, until a check passes again. The reset permission is not probed, since a valid reset request moves the durable back to its ack floor; `wavehouse mq permissions` always grants it with the other two. + +**Under a JetStream domain** (`mq.nats.js_domain`), WaveHouse sends every JetStream API request and every lease write under `$JS..API.`, including the pulls that keep a busy or stopping shard's pin. A server in that domain maps those subjects to the plain `$JS.API.…` before it checks permissions, while a server outside it, such as a leafnode WaveHouse connects through, checks them as sent. `wavehouse mq permissions --js-domain ` allows and denies every such subject in both forms, so the same block works either way. + +WaveHouse's replies arrive under `_INBOX_.>`, which is why the subscribe permission can be that narrow. + +**Publishing to `.ingest.>` or `.dlq.>` is trusted as WaveHouse itself.** The ingest worker takes the tenant from the subject and writes the event as published, so a principal with that right writes to any tenant without passing authentication, policy or schema validation. Grant it to the `wavehouse` user alone. Publishing to `.hist.>` feeds every tenant's live SSE events and replays directly; grant it to no one, since only the partitions' republish writes there. (`nack` has full access to the account; keep its credentials to the controller.) + +### Limits that differ from the embedded queue + +- **Per-tenant budgets are not enforced.** A tenant's [`mq.max_bytes_gb`](/settings-directory#message-queue) is not applied; a partition's byte limit is shared by the tenants in it. `maxMsgsPerSubject` with `discardPerSubject: true`, which the generated manifests set, refuses one tenant's table once it holds that many unwritten rows, before it fills the partition. +- **A shard's delivery can stall on one tenant.** If one tenant's ClickHouse hangs, its unwritten rows can fill its shard's share of the rows the ingest process may hold (see [Scaling ingest processes](#scaling-ingest-processes)), and then delivery pauses for every table in that shard, about 1/(N×V) of the tables. The other shards of the same process keep their own shares and keep flowing. More shards shrink that share. (A ClickHouse that refuses inserts outright does not stall a shard: its rows are handed back to the partition for a delayed retry.) +- **Dead-lettered rows share one stream.** Its `discard: old` evicts the oldest rows when it is full; with `maxMsgsPerSubject` set, it evicts per table, so one tenant's flood evicts only its own rows. `GET /v1/ops/dlq/stats?tenant=` answers `200` with zeros for a tenant that has never parked a row, where the embedded queue answers `404`. +- **SSE replay and live events are best effort.** A row republished while the history stream is unavailable, for example full with `discard: new`, deleted, or electing a leader, is written to ClickHouse but never reaches SSE. `wavehouse_mq_history_behind_seconds` shows a history that stops taking rows, and a deleted one fails the topology check (`wavehouse_mq_topology_ok`). + +### Choosing and changing N + +A table lives in one partition, so one table's ingest rate is bounded by what one stream can take, and a tenant's tables spread over the partitions. N must match `mq.nats.partitions` in every process. Raising it to N′ moves (N′−N)/N′ of the tables, each into one of the new partitions; lowering it moves only the tables of the partitions removed. A moved table's events are not in order across the move. WaveHouse publishes only to partitions `0` to `N−1`, and its ingest worker also drains the shard durables left on any stream holding ingest subjects outside them (the generated `preventDelete` keeps them), so lowering N loses no rows; a stream with no shard durable is only reported, since nothing can drain it. + +Every generated partition records its index and N in its metadata (`wavehouse.dev/partition`, `wavehouse.dev/partitions`), and a process configured for another N refuses them. So change N by regenerating: `wavehouse mq manifests --partitions `, and apply the whole output, which updates every partition's metadata. From then until every process runs the new N, the processes still on the old N report `wavehouse_mq_topology_ok` `0` at their next check and cannot restart, so roll out promptly. + +- **To raise N,** apply the regenerated manifests, then roll WaveHouse out with the new N, promptly. Until every ingest process runs the new N, processes on the old and new N share the shards out differently, so some shards (the new ones among them) may have no owner: their rows wait on the partitions, and `wavehouse_ingest_shards_unowned` reads above `0` until the rollout ends. +- **To lower N,** apply the regenerated manifests, then roll WaveHouse out with the smaller N. Ingest does not need to stop. The regenerated manifests leave the removed partitions out, and the generated resources set `preventDelete`, so each removed partition's stream and its shard durables stay, with their rows. The ingest processes on the new N share each such stream's shard durables out with the rest, and drain them, and processes still on the old N keep publishing to it until they are replaced. Boot warns about each one with the rows it still holds. Once a removed partition holds no rows and no process runs the old N, delete its nack `Stream` and `Consumer` resources if they are still applied, then the stream itself with the operator's credentials (`nats stream rm `): `preventDelete` keeps the stream when only its resources go, and the `wavehouse` user cannot delete it. That ends delivery from that stream only, not the worker. Deleting it while it still holds rows loses them, as deleting any partition does. A removed partition keeps its republish, so rows the old processes publish to it still reach live SSE and replay; the history needs no change when N does. + +### Scaling ingest processes + +Scale the ingest processes as you like; no manifest changes. Each one holds a membership lease in the lease bucket (`lease.ingest.m`, for the lowest free `j`; there are at most 64 such slots, so at most 64 ingest processes get work), and every process computes the same owner for every shard from the live members: rendezvous hashing, capped so that no process owns more than an even share, rounded up. A process that joins or leaves moves roughly one to two times its share of the shards while the even share, rounded up, stays the same (a few times it at worst); past about seven processes, where that share steps down (over 32 shards in all, partitions × shards: at 8, 11, 16 and 32 processes), it moves several times its share. It never moves more than half of them. The cap bounds the most a process owns, not the least: above about half as many processes as shards, some hold none, and a process beyond partitions × shards, or beyond 64, always holds none. Shard durables left over after N or V is lowered are shared out apart from the configured ones, so deleting a drained one moves no configured shard. + +- **A shard changing owner** stops fetching at the old owner, which keeps the shard's pin while it waits for the rows it delivered to be acked, or handed back to the partition, then releases the pin; the new owner receives at once, and only rows newer than the ones the old owner wrote. Fetching stops within a second, the length of one pull; the rows already fetched must then still reach the worker, which takes as long as the worker is behind. The wait for them to be acked is bounded by one minute from when they have reached the worker, the worker's ack wait and the shortest `ack_wait` a shard durable may have, and the old owner keeps the pin that long and a few seconds more; if it runs out, the new owner may receive before the old one has finished, so that one shard's rows are briefly written by two processes, out of order. +- **A clean stop** stops fetching every shard first, keeping their pins, so the worker writes what it holds and whatever those shards still deliver; then it releases every shard at once and resigns the membership lease, and the others take over within about a tick (2 seconds). A row the worker could not take while stopping is handed back to the partition before the release, so it comes back to the next owner ahead of anything newer. Rows the partition redelivers (handed back, or past `ack_wait`) come back ahead of newer rows, but not always in order among themselves across a handover or a stop: while the next owner's pull waits for the pin, the server requeues a due redelivery behind the others. +- **A process that dies** keeps its membership lease until the others have seen it unrenewed for 15 seconds, and its shards' pins until the pinned TTL (10 seconds in the generated manifests) after its last pull. Then each new owner resets the shard to its ack floor before pulling it, so the rows the dead process received and never acked come back at once rather than after `ack_wait` (at least 1 minute). The reset needs more than the membership view: the shard must also hold no pin and have delivered nothing and had no ack for the pinned TTL, so a process the others wrongly think dead, but which still works the shard, is never reset under it. A lone ingest process restarting after a clean stop does the same on its first pass, for every shard nobody holds. One restarting after a crash still sees its predecessor's lease as live for 15 seconds: the shards the new process is assigned beside it come back after `ack_wait`, the rest at once when that lease lapses. Rows it had written but not acked are written again: delivery is at least once, so use a deduplicating table engine, or [`dedupe`](/configuration#dedupe). +- **A live process that stops making progress** (a hung ClickHouse insert) keeps its shards, and their pins, however long its inserts take: liveness is not progress. The worker never holds up a shard's pulls, so its pins never lapse, and no other process takes its shards or resets them. Each shard may hold rows delivered and not yet acked up to its share of 10,000: an even share over the shards the process holds, but never under 1,000 (two batches), so one busy table still fills whole batches. At its share a shard fetches one row every 5 seconds, which keeps its pin (a row the partition redelivers comes back in order that way), up to one `ack_wait` of such rows past its share (12 at the generated 1-minute `ack_wait`); past that it keeps its pin with pulls that deliver nothing, and a stuck shard's redelivered rows may then come back out of order among themselves; the rest stay on the partition, and one stuck shard never takes another shard's share. A process so holds about 10,000 rows at most, or 1,000 per shard when it holds more than ten, plus those 12 per stuck shard. A row stops counting at its first ack or nak, confirmed or not, or once `ack_wait` has passed, when the partition redelivers it anyway. + +### Choosing and changing V + +A shard's rows reach ClickHouse through one ingest process at a time, so V×N caps how many ingest processes get work, and one table's insert rate is bounded by what one process writes. A process splits its prefetch (500 rows) between the shards it pulls, so one shard's pull asks for fewer rows the more shards the process holds. The default V of 32 leaves room to scale out; a larger V costs a consumer per shard on every partition (a Raft group each at 3 replicas), a permission entry per shard, and one pull a second per shard that has nothing to deliver. V must match `mq.nats.shards` in every process, and each partition records it in its metadata (`wavehouse.dev/shards`). + +- **To raise V,** regenerate with `--shards ` and apply the whole output: the new shards' durables are added, and the existing ones keep their names and their rows. Regenerate the permissions with the new V too, then roll WaveHouse out with it promptly: the regenerated metadata says the new V, so processes still on the old V report `wavehouse_mq_topology_ok` `0` at their next check and cannot restart. Raising V to V′ moves (V′−V)/V′ of the tables, each into one of the new shards; a moved table's older rows drain from its old shard while its new rows go to the new one, so for that table alone two processes may insert at once for a while, and its events are not in order across the move. A process still on the old V publishes to the old shards. Until every ingest process runs the new V, the old and new processes share the shards out differently, so some shards (the new ones among them) may have no owner: their rows wait, and `wavehouse_ingest_shards_unowned` reads above `0` until the rollout ends. +- **To lower V,** apply the regenerated manifests (the generated durables set `preventDelete`, so the removed shards' durables stay with their rows) and roll WaveHouse out with the smaller V, promptly, for the same reason. Keep the permissions for the larger V until they drain. The ingest processes share out each durable past the new V and drain it, and boot warns about each with the rows it holds; once one holds none and no process runs the old V, delete its nack `Consumer` resource if it is still applied (nack's control loop would recreate the durable otherwise), then the durable itself with the operator's credentials (`nats consumer rm `): `preventDelete` keeps the durable when only its resource goes. That ends delivery from that durable only, not the worker. Then regenerate the permissions for the smaller V. + +### Before production: manual checks + +Two things the automated tests do not cover. Check them once on your own cluster before production: + +- **nack applies what the manifests say.** Apply the generated manifests with your nack release, then read back one partition and one shard durable (`nats stream info WH_INGEST_0`, `nats consumer info WH_INGEST_0 wh-ingest-0`): the partition needs `retention: workqueue` and its `republish`, the durable `pinned_client`, the group `wavehouse` and its pinned TTL. WaveHouse's boot check reports any field that did not arrive. +- **Three replicas behave as the design assumes.** On a three-server cluster at `--replicas 3`: kill the leader of a shard durable while a worker holds its pin, and check that the worker re-pins with its next pull and no row is lost; kill a partition's stream leader while events flow, and check how many of them the history misses (the rows themselves still reach ClickHouse); and watch the servers' CPU and memory with the full N×V consumers under load. + +### Monitoring + +These gauges are exported through [OpenTelemetry or Prometheus](#observability) under `mq.backend: nats`: + +| Gauge | Meaning | +| --- | --- | +| `wavehouse_mq_connected` | `1` while this process is connected to the cluster, else `0`. | +| `wavehouse_mq_topology_ok` | `1` while the last check found every required stream, consumer and (under `coord.backend: nats`) the lease bucket, and found that the `wavehouse` user may pull from and unpin every shard durable, else `0`. It drops at once when a publish finds a partition deleted, or when the server refuses a consumer request (see [Permissions](#permissions)). | +| `wavehouse_mq_history_behind_seconds` | How far the history's newest row trails the newest row any partition stored, read every 30 seconds. A value that stays up or keeps growing means the history is not taking the rows the partitions republish, so SSE replay and live events miss them; it returns to about `0` with the next row the history takes, and rows missed before that are not counted. A missing history reads as the last value while `wavehouse_mq_topology_ok` goes to `0`. It never affects ingest. | + +The ingest processes export these for their shards: + +| Metric | Meaning | +| --- | --- | +| `wavehouse_ingest_shards_owned` | Shards this process holds and consumes. The sum over processes is partitions × shards, plus any extras still draining, once membership has settled. | +| `wavehouse_ingest_shard_members` | Ingest processes this process last counted. | +| `wavehouse_ingest_shards_unowned` | Shards with rows waiting that no process holds, and that have delivered nothing and had no ack for the pinned TTL, read every 15 seconds by the lowest-ranked member only (the others report nothing, so take the `max` across processes). Alert when it stays above `0` for a minute or more: rows nobody is writing. A shard whose process is stuck on a hung insert keeps its pin, so it does not count. | +| `wavehouse_ingest_rows_held` | Rows delivered to this process that it has not yet acked or nacked (confirmed or not), and that are younger than `ack_wait`, across its shards; each shard holds about its share at most (see [Scaling ingest processes](#scaling-ingest-processes)). | +| `wavehouse_ingest_shard_events_total{event}` | `taken`, `released`, `reset` (a dead owner's rows redelivered at takeover), `handover_timeout` (a shard released before its rows were acked or handed back), `lost` (the membership lease ended under the process), `bind_failed` (a shard could not be bound; tried again next tick, and logged as an error once it has failed for a minute). A configured shard durable (one of the N×V) that is gone, with its stream or alone, or that no longer fits the worker (its `ack_wait` shorter than a minute, or no `max_ack_pending`), ends the worker instead, as a durable deleted while it is pulled does; a leftover durable past a lower N or V is only logged. | +| `wavehouse_ingest_shard_handover_seconds` | From giving a shard up to releasing it. | + +`wavehouse_nats_connections` and `wavehouse_nats_in_msgs_total` describe this process's client connection under `nats` (`1` or `0`, and the messages it has received), where under `embedded` they describe the embedded server. + ## One Deployment per role By default one process runs all of WaveHouse. [`roles`](/configuration#process-roles) (`WH_ROLES`) lets the API and the background workers run as separate processes, so that each scales on its own. On Kubernetes that is one Deployment per role, from the same image, differing only in `WH_ROLES`: @@ -341,17 +492,22 @@ By default one process runs all of WaveHouse. [`roles`](/configuration#process-r | --- | --- | --- | --- | | API | `api` | as many as your request load needs | the full API | | Ingest | `ingest` | as many as your write load needs | the ops listener | -| Sweeper | `sweeper` | 1, or 2 for a warm standby | the ops listener | - **API.** Each API pod runs its own schema discovery, token verifiers, dedupe handle and SSE hub, and receives every event so that it can serve its own SSE clients. Put your Service and ingress in front of these pods only. -- **Ingest.** Every ingest pod consumes the same shared durable consumer and competes for its messages, so throughput scales with the pod count. The rows of one table are then split across pods: each pod writes smaller batches, and rows written by different pods do not reach ClickHouse in publish order. -- **Sweeper.** The sweeper runs under a lease held in the shared `coord.backend`, so only one pod sweeps at a time. A second replica waits and takes over when the first stops. +- **Ingest.** The ingest pods share the shards out, none owning more than an even share rounded up, and each shard is written by one pod at a time, so a table's rows are written by one pod. Scaling the Deployment moves only whole shards, never rows; at most partitions × shards pods get work. See [Scaling ingest processes](#scaling-ingest-processes). + +A split needs backends that every process can reach: a shared `mq.backend`, so that every process reaches the same queue; a shared `coord.backend`, so that the ingest processes' leases span pods; and, when `api` and `ingest` run in separate processes, a shared `cache.backend`, so that the ingest pods' invalidations reach the API pods' cache. This build has four shared backends: [`mq.backend: nats`](#external-nats) and `coord.backend: nats` on the same cluster, [`cache.backend: redis`](#multiple-instances-and-the-shared-cache), and [`dedupe.backend: dynamodb`](#a-shared-dedupe-table-on-dynamodb). Boot refuses a split the selected backends cannot serve, naming the backend to change: + +- **Separate `api` and `ingest` Deployments need `cache.backend: redis`.** With a local cache, boot refuses a process that runs one of them without the other; run them together (`WH_ROLES=api,ingest`) instead, and each replica's cache serves reads that may be stale until an entry expires (boot warns). +- **Dedupe across replicas needs `dedupe.backend: dynamodb`.** With `pebble` each replica dedupes only the event ids it has seen itself, so a retry that lands on another replica is written twice (boot warns). +- **There is no sweeper process.** Under `mq.backend: nats` the streams' own retention replaces the sweeper, so it is not wired, and boot refuses a process whose only role is `sweeper`. The API processes warn, once per tenant and window, about a tenant whose gap window is longer than the history stream keeps. +- **Ingest processes need `coord.backend: nats`:** boot refuses `coord.backend: local` in a process running `ingest` on a shared queue. -A split needs backends that every process can reach: a shared `mq.backend`, so that every process reaches the same queue; a shared `cache.backend` ([`redis`](#multiple-instances-and-the-shared-cache)), so that the ingest pods' invalidations reach the API pods' cache; and a shared `coord.backend`, so that the sweeper lease spans pods. **This build has a shared cache but only the in-process queue and leases, so boot refuses any split** and names the backend to change. Until a shared `mq.backend` and `coord.backend` ship, run every role in one process, the default. +Run every role in one process, the default, until you need more than one. -A pod without the `api` role serves an ops listener on `:8080`: `/livez`, `/readyz` and their aliases, `/version`, the metrics path when `prometheus.port` is `0`, and `POST /v1/ops/settings/reload`. Every other route answers 404 (under `/v1/ops`, 403 without the operator key, and 401 for a bearer token). Point the same probes at it as at an API pod. `/livez` does not wait for schema discovery there, because only the API runs it. `/readyz` checks ClickHouse in an ingest pod, and is ready once a sweeper pod has booted. Every pod reads the settings directory, so mount it in every Deployment. The reload route on the ops listener accepts only the operator key, so whatever reloads your API pods over HTTP must send the operator key to the worker pods too, or rely on `SIGHUP` (or, over a flat directory, the directory watcher) instead. +A pod without the `api` role serves an ops listener on `:8080`: `/livez`, `/readyz` and their aliases, `/version`, the metrics path when `prometheus.port` is `0`, and `POST /v1/ops/settings/reload`. Every other route answers 404 (under `/v1/ops`, 403 without the operator key, and 401 for a bearer token). Point the same probes at it as at an API pod. `/livez` does not wait for schema discovery there, because only the API runs it. `/readyz` checks ClickHouse in an ingest pod. Every pod reads the settings directory, so mount it in every Deployment. The reload route on the ops listener accepts only the operator key, so whatever reloads your API pods over HTTP must send the operator key to the worker pods too, or rely on `SIGHUP` (or, over a flat directory, the directory watcher) instead. -Give each pod a stable `WH_INSTANCE_ID` only if you need one in the logs. The default, the pod's hostname with a random suffix, already names each pod uniquely. +Give each pod a stable `WH_INSTANCE_ID` only if you need one in the logs or in the lease's `holder`. The default, the pod's hostname with a random suffix, already names each pod uniquely. ## Behind a reverse proxy @@ -409,13 +565,13 @@ That is the layout a control plane writes. Each folder's `clickhouse` block is i The folder name is the tenant id, and each folder is a complete settings directory: everything on the [Settings Directory](/settings-directory) page applies to it as written, except where the rules below say otherwise. The two shapes don't mix — a folder beside the four files, or a loose file beside the folders, is a validation error — and a running server keeps the shape it booted with, so switching is stop, restructure, start. The dedupe store needs no restructuring: it keys every tenant's seen ids by tenant, and the four files are tenant `0`, as a `0` folder is. Dot-prefixed entries are ignored in either shape. `wavehouse validate` checks either shape with the same exit codes; a finding in a nested directory names its folder (`acme/policies.json`), and a folder whose name is not a tenant id is a finding of its own — that folder is skipped, and the rest of the directory still loads. -**A rejected folder fails closed, for that tenant alone — tenant `0`'s excepted.** A folder that fails validation stops its tenant being served — its requests answer `503` — while every other tenant carries on, at boot and on a reload alike. Tenant `0` is the exception: the process still draws some shared wiring from that folder, so rejecting it costs every tenant something ("What a lost tenant `0` costs", below, says what). There is no fall back to the tenant's previous settings, unlike [the single-tenant directory](/settings-directory#loading-and-hot-reload): the recovery is fixing the folder and reloading it. A request already in flight finishes on the settings it started with, except an open `GET /v1/stream`, which is ended at once: its reconnect gets the `503` until the folder is fixed — the SDK keeps retrying and then resumes from `Last-Event-ID`, while a browser `EventSource` gives up on the `503` and has to be reopened. The rows the tenant had already accepted but not yet inserted, those of an ingest request in flight included, which still answers `200`, are parked on the DLQ under the tenant's own subject rather than held for the fix, as a removed tenant's are (see [Dead Letter Queue](#dead-letter-queue-dlq)). Its message queue is kept, at the budget it last had, and so is the history that gap-fill replays, for the `stream.gap_window_minutes` its folder last had (all of it, for a folder rejected since the server started, whose window the server never read): a stream resumed after a fix within that window picks up where it left off. The findings go to the log and to the reload response, never into the `503`. A finding about the directory itself — a loose file, an entry or a directory that can't be read, a changed shape — is another matter: it refuses boot, and on a reload it rejects the reload whole and leaves every tenant as it was. +**A rejected folder fails closed, for that tenant alone — tenant `0`'s excepted.** A folder that fails validation stops its tenant being served — its requests answer `503` — while every other tenant carries on, at boot and on a reload alike. Tenant `0` is the exception: the process still draws some shared wiring from that folder, so rejecting it costs every tenant something ("What a lost tenant `0` costs", below, says what). There is no fall back to the tenant's previous settings, unlike [the single-tenant directory](/settings-directory#loading-and-hot-reload): the recovery is fixing the folder and reloading it. A request already in flight finishes on the settings it started with, except an open `GET /v1/stream`, which is ended at once: its reconnect gets the `503` until the folder is fixed — the SDK keeps retrying and then resumes from `Last-Event-ID`, while a browser `EventSource` gives up on the `503` and has to be reopened. The rows the tenant had already accepted but not yet inserted, those of an ingest request in flight included, which still answers `200`, are parked on the DLQ under the tenant's own subject rather than held for the fix, as a removed tenant's are (see [Dead Letter Queue](#dead-letter-queue-dlq)). Under the embedded broker its message queue is kept, at the budget it last had, and so is the history that gap-fill replays, for the `stream.gap_window_minutes` its folder last had (all of it, for a folder rejected since the server started, whose window the server never read); under `mq.backend: nats` the shared history ages out by its `max_age` whatever happens to the folder: a stream resumed after a fix within that window picks up where it left off. The findings go to the log and to the reload response, never into the `503`. A finding about the directory itself — a loose file, an entry or a directory that can't be read, a changed shape — is another matter: it refuses boot, and on a reload it rejects the reload whole and leaves every tenant as it was. -**Reloading is the writer's call.** A nested directory is not watched, because a watcher would validate a folder halfway through being written and drop its tenant. Whoever writes a tenant's folder reloads it once it is complete: `POST /v1/ops/settings/reload?tenant=acme` re-validates that folder and reads nothing else. It must name a tenant the server already holds (`404` otherwise), so a folder the server does not hold yet — one added since the last whole-directory reload — is picked up by a whole-directory reload, not by naming it; a tenant it holds but rejected is reloaded by name like any other. Without the parameter — and on `SIGHUP` — the whole directory is reloaded and mirrors its folders: a new folder becomes a tenant, and a removed one becomes unknown. That is how a tenant is removed: delete its folder, then reload the whole directory. Its open streams end, its routes answer `404`, and its queued rows are parked on the DLQ under its own subject; nothing it stored is deleted — its message queue is kept at the budget it last had, and only the history that gap-fill replays goes from it, at the next sweep — so restoring the folder restores the tenant, seen ids and parked rows included. Reloading a deleted folder by name instead leaves its tenant rejected, answering `503`. The last folder can be removed the same way, with two catches, since `wavehouse validate` and boot both read an emptied directory as the four files missing: `validate` exits `1`, so a writer that gates each reload on it has to skip the check for that one reload, and a server restarted before a folder is written back refuses to boot. A whole-directory reload re-validates every folder, so it carries the exposure the watcher would: a folder caught halfway through being written can fail validation, and its tenant then stops being served until a later reload adopts it. The response is the [single-tenant one](/api#post-v1opssettingsreload--reload-settings-directory). After a whole-directory reload, `adopted: false` with a `422` can mean adopted in part: the folders with an error among their `findings` were rejected and the rest were adopted — warnings included, since `findings` carries every folder's. +**Reloading is the writer's call.** A nested directory is not watched, because a watcher would validate a folder halfway through being written and drop its tenant. Whoever writes a tenant's folder reloads it once it is complete: `POST /v1/ops/settings/reload?tenant=acme` re-validates that folder and reads nothing else. It must name a tenant the server already holds (`404` otherwise), so a folder the server does not hold yet — one added since the last whole-directory reload — is picked up by a whole-directory reload, not by naming it; a tenant it holds but rejected is reloaded by name like any other. Without the parameter — and on `SIGHUP` — the whole directory is reloaded and mirrors its folders: a new folder becomes a tenant, and a removed one becomes unknown. That is how a tenant is removed: delete its folder, then reload the whole directory. Its open streams end, its routes answer `404`, and its queued rows are parked on the DLQ under its own subject; nothing it stored is deleted — under the embedded broker its message queue is kept at the budget it last had, and only the history that gap-fill replays goes from it, at the next sweep (under `mq.backend: nats` the history ages out by its `max_age`) — so restoring the folder restores the tenant, seen ids and parked rows included. Reloading a deleted folder by name instead leaves its tenant rejected, answering `503`. The last folder can be removed the same way, with two catches, since `wavehouse validate` and boot both read an emptied directory as the four files missing: `validate` exits `1`, so a writer that gates each reload on it has to skip the check for that one reload, and a server restarted before a folder is written back refuses to boot. A whole-directory reload re-validates every folder, so it carries the exposure the watcher would: a folder caught halfway through being written can fail validation, and its tenant then stops being served until a later reload adopts it. The response is the [single-tenant one](/api#post-v1opssettingsreload--reload-settings-directory). After a whole-directory reload, `adopted: false` with a `422` can mean adopted in part: the folders with an error among their `findings` were rejected and the rest were adopted — warnings included, since `findings` carries every folder's. -**The admin routes take the operator key only.** `/v1/ops/*` reaches every tenant, so over a nested directory no tenant's admin role opens it: the [operator key](/api#authentication) alone does, and a token carrying an admin role gets `403`. Boot a nested directory without `auth.operator_key` and no caller can reach these routes at all, which leaves `SIGHUP` as the only reload; the server warns about it at boot. `GET /v1/ops/pipes`, `GET /v1/ops/pipes/{name}`, `GET /v1/ops/schema`, `POST /v1/ops/schema/refresh` and `POST /v1/ops/query` take the same `?tenant=`, and address tenant `0` without it; `GET /v1/ops/dlq/stats` takes it too, and reads a rejected or removed tenant's dead-letter queue like a served one's, since the queue is kept; a tenant that has none is a `404`. On the routes that take it the parameter is parsed strictly — a query string that does not parse, an empty or repeated `tenant`, or a malformed id is a `400`, never a silent read of the default tenant or, on the reload route, a reload of every tenant. The SDK sends it as the [`tenant` option](/sdk/admin#settings--whsettings). +**The admin routes take the operator key only.** `/v1/ops/*` reaches every tenant, so over a nested directory no tenant's admin role opens it: the [operator key](/api#authentication) alone does, and a token carrying an admin role gets `403`. Boot a nested directory without `auth.operator_key` and no caller can reach these routes at all, which leaves `SIGHUP` as the only reload; the server warns about it at boot. `GET /v1/ops/pipes`, `GET /v1/ops/pipes/{name}`, `GET /v1/ops/schema`, `POST /v1/ops/schema/refresh` and `POST /v1/ops/query` take the same `?tenant=`, and address tenant `0` without it; `GET /v1/ops/dlq/stats` takes it too, and reads a rejected or removed tenant's dead-letter queue like a served one's, since the queue is kept; under the embedded broker a tenant that has none is a `404`. On the routes that take it the parameter is parsed strictly — a query string that does not parse, an empty or repeated `tenant`, or a malformed id is a `400`, never a silent read of the default tenant or, on the reload route, a reload of every tenant. The SDK sends it as the [`tenant` option](/sdk/admin#settings--whsettings). -**What a tenant's folder decides.** A request is evaluated against its own tenant's `policies.json` and `pipes.json` (ingest, structured queries, pipes), its `query.*` keys, its `cors.allowed_origins`, and its `dedupe` block: whether its records are deduplicated, by which id, against the tenant's own store, which that folder's `dedupe.enabled` opens and closes on reload exactly as [the single-tenant one](/settings-directory#deduplication) does (every tenant's store is a share of the one Pebble instance at `/pebble`, each key led by its tenant and table), so `wavehouse_ingest_dedupe_disabled_total` ticks only across a tenant's own reload, whatever the other tenants' switches say. A tenant's seen ids are its own: the same event id is first seen under each tenant that sends it, and in each table. Its `auth` block is its own too: each tenant's folder wires that tenant's token verifier (`jwks_url`, `role_claim`), built when the folder is adopted and rebuilt when its wiring changes, so a JWKS-issued token verifies only under the tenants whose `jwks_url` names its provider's key set. Under another tenant's header a token is treated as invalid, and the request falls back to that tenant's `default_role` like any other unverifiable token, possibly after a rate-limited key refetch (see [Authentication](/settings-directory#authentication)). Keep `X-Tenant-ID` pinned at the proxy so a token is never presented under the wrong tenant. Tenants can still accept each other's tokens: those that leave `jwks_url` empty share the boot HMAC secret when `auth.jwt_secret` is set, so a token verifies under any of them (with no secret they validate no token at all), and those whose `jwks_url` names the same key set accept each other's tokens; isolate them by provider, or scope rows by a signed claim ([row-level security](/access-control#row-level-security)). A tenant whose `jwks_url` has not been fetched yet answers `503` with `Retry-After` to its token-bearing requests alone. A tenant that stops being served — its folder rejected or removed — loses its verifier and the JWKS refresh with it, and gets a fresh one when its folder is adopted again. The HMAC secret and the operator key stay boot config, shared by every tenant; the operator key is stamped with the request tenant's `admin_role`. A tenant's `clickhouse` and `schema` blocks are its own as well: each tenant reads and writes its own ClickHouse — one native pool per distinct address, database, user, password and `tls` tuple, shared by the tenants naming it, under the process-wide [connection ceiling](/settings-directory#clickhouse) — and discovers its own tables from its own database on its own `schema.refresh_interval`. Its message queue is its own as well: its events are queued on a stream of their own, capped at its own `mq.max_bytes_gb` — at that budget its ingest answers `503` while every other tenant's keeps publishing — beside a dead-letter stream of its own at a tenth of it, and the history that gap-fill replays from it is kept for its own `stream.gap_window_minutes`. Nothing checks what the tenants' budgets add up to against the disk, so size them together ([Message Queue](/settings-directory#message-queue)). An event is published on its tenant's subject (`ingest.{tenant}.{table}`), so a `GET /v1/stream` connection is authorized by its own tenant's `policies.json` and receives its own tenant's rows alone, the ingest worker inserts a row into its own tenant's ClickHouse, a rejected row is parked under its own tenant's `dlq.enabled` and subject (`dlq.{tenant}.{table}`), and two tenants' tables of one name never share a batch. The query cache is one pool, but its entries are keyed by tenant: identical `POST /v1/query` and pipe requests from two tenants are two entries and two queries to ClickHouse, and a tenant is never served another's cached rows. An insert invalidates the table's cached results under every tenant on the same ClickHouse address and database as the tenant it was ingested for, whatever their user or `tls` block, since they read the same tables; a tenant on no pool — its folder rejected or removed, or no pool could be opened for it, such as by the ceiling — is out of that fan-out while it is, and has its cached results (`POST /v1/query` and pipe alike) dropped the moment it is back on one, so a repaired or restored folder never serves rows cached before the inserts it missed. A tenant whose folder moves it to another address or database has them dropped too, since they came from other tables. Apart from those two drops, a cached pipe result stays until its TTL expires, since a pipe names no table and no insert invalidates it. One setting weighs every tenant: the SSE keepalive, where the wheel runs at the shortest `stream.keepalive_interval` among the tenants being served, with that tenant's `stream.keepalive_buckets`. +**What a tenant's folder decides.** A request is evaluated against its own tenant's `policies.json` and `pipes.json` (ingest, structured queries, pipes), its `query.*` keys, its `cors.allowed_origins`, and its `dedupe` block: whether its records are deduplicated, by which id, against the tenant's own store, which that folder's `dedupe.enabled` opens and closes on reload exactly as [the single-tenant one](/settings-directory#deduplication) does (every tenant's store is a share of the one Pebble instance at `/pebble`, each key led by its tenant and table), so `wavehouse_ingest_dedupe_disabled_total` ticks only across a tenant's own reload, whatever the other tenants' switches say. A tenant's seen ids are its own: the same event id is first seen under each tenant that sends it, and in each table. Its `auth` block is its own too: each tenant's folder wires that tenant's token verifier (`jwks_url`, `role_claim`), built when the folder is adopted and rebuilt when its wiring changes, so a JWKS-issued token verifies only under the tenants whose `jwks_url` names its provider's key set. Under another tenant's header a token is treated as invalid, and the request falls back to that tenant's `default_role` like any other unverifiable token, possibly after a rate-limited key refetch (see [Authentication](/settings-directory#authentication)). Keep `X-Tenant-ID` pinned at the proxy so a token is never presented under the wrong tenant. Tenants can still accept each other's tokens: those that leave `jwks_url` empty share the boot HMAC secret when `auth.jwt_secret` is set, so a token verifies under any of them (with no secret they validate no token at all), and those whose `jwks_url` names the same key set accept each other's tokens; isolate them by provider, or scope rows by a signed claim ([row-level security](/access-control#row-level-security)). A tenant whose `jwks_url` has not been fetched yet answers `503` with `Retry-After` to its token-bearing requests alone. A tenant that stops being served — its folder rejected or removed — loses its verifier and the JWKS refresh with it, and gets a fresh one when its folder is adopted again. The HMAC secret and the operator key stay boot config, shared by every tenant; the operator key is stamped with the request tenant's `admin_role`. A tenant's `clickhouse` and `schema` blocks are its own as well: each tenant reads and writes its own ClickHouse — one native pool per distinct address, database, user, password and `tls` tuple, shared by the tenants naming it, under the process-wide [connection ceiling](/settings-directory#clickhouse) — and discovers its own tables from its own database on its own `schema.refresh_interval`. Under the embedded broker its message queue is its own as well (under [`mq.backend: nats`](#limits-that-differ-from-the-embedded-queue) tenants share the partitions): its events are queued on a stream of their own, capped at its own `mq.max_bytes_gb` — at that budget its ingest answers `503` while every other tenant's keeps publishing — beside a dead-letter stream of its own at a tenth of it, and the history that gap-fill replays from it is kept for its own `stream.gap_window_minutes`. Nothing checks what the tenants' budgets add up to against the disk, so size them together ([Message Queue](/settings-directory#message-queue)). An event is published on its tenant's subject (`ingest.{tenant}.{table}`), so a `GET /v1/stream` connection is authorized by its own tenant's `policies.json` and receives its own tenant's rows alone, the ingest worker inserts a row into its own tenant's ClickHouse, a rejected row is parked under its own tenant's `dlq.enabled` and subject (`dlq.{tenant}.{table}`), and two tenants' tables of one name never share a batch. The query cache is one pool, but its entries are keyed by tenant: identical `POST /v1/query` and pipe requests from two tenants are two entries and two queries to ClickHouse, and a tenant is never served another's cached rows. An insert invalidates the table's cached results under every tenant on the same ClickHouse address and database as the tenant it was ingested for, whatever their user or `tls` block, since they read the same tables; a tenant on no pool — its folder rejected or removed, or no pool could be opened for it, such as by the ceiling — is out of that fan-out while it is, and has its cached results (`POST /v1/query` and pipe alike) dropped the moment it is back on one, so a repaired or restored folder never serves rows cached before the inserts it missed. A tenant whose folder moves it to another address or database has them dropped too, since they came from other tables. Apart from those two drops, a cached pipe result stays until its TTL expires, since a pipe names no table and no insert invalidates it. One setting weighs every tenant: the SSE keepalive, where the wheel runs at the shortest `stream.keepalive_interval` among the tenants being served, with that tenant's `stream.keepalive_buckets`. **What a lost tenant `0` costs.** A `0` folder that a reload rejects or removes stops tenant `0` being served like any other, and what becomes of the shared settings depends on how they are read. Tenant `0` leaves its ClickHouse pool (closed only once no served tenant names its tuple), and its schema registry and verifier are released with the folder, like any other tenant's; the `/v1/ops/*` routes, which resolve no tenant, verify against it, so a token there reads as invalid (`401`) rather than merely non-admin (`403`) until tenant `0` is served again — the operator key, which never consults a verifier, is unaffected. CORS does not stay either: the responses that read tenant `0`'s list — the tenant-exempt routes, the refusals, a preflight naming no tenant — carry no CORS headers until the folder is served again, while every other tenant's routes keep their own list. Tenant `0`'s own dedupe store closes, as any rejected or removed tenant's does, its seen ids kept for the folder that restores it. What is read per event follows the event's tenant, so tenant `0`'s events are the ones affected: with no ClickHouse to insert into, its rows fail and are parked on the DLQ whatever its switch said, and its open `GET /v1/stream` connections are ended, as any tenant's are when it stops being served — the other tenants' events are untouched. A nested directory that has never served a tenant `0` — no `0` folder, or one rejected at boot — serves every other tenant from its own ClickHouse. Outside `/v1/ops/*`, a `/v1` request that sends no `X-Tenant-ID` resolves to tenant `0`, so with no `0` folder it answers `404 unknown tenant: 0` (`503` with a rejected one) — the SDK's `/v1/health` reachability ping included. @@ -425,9 +581,9 @@ The folder name is the tenant id, and each folder is a complete settings directo ## Multiple instances and the shared cache -Several WaveHouse instances can serve one ClickHouse behind a load balancer, but most of what each one holds is its own. The message queue is embedded, so an event is inserted by the instance that took its `POST /v1/ingest`, and reaches only that instance's SSE subscribers. With the default `dedupe.backend: pebble` the dedupe store is per instance too, so an id one instance has seen is new to another; [`dedupe.backend: dynamodb`](#a-shared-dedupe-table-on-dynamodb) shares seen ids across instances. +Several WaveHouse instances can serve one ClickHouse behind a load balancer. What they share is decided per layer. With the defaults, most of what each one holds is its own: the embedded message queue means an event is inserted by the instance that took its `POST /v1/ingest` and reaches only that instance's SSE subscribers, and the Pebble dedupe store means an id one instance has seen is new to another. [`mq.backend: nats`](#external-nats) gives every instance one queue, so each one's SSE subscribers see every event, and [`dedupe.backend: dynamodb`](#a-shared-dedupe-table-on-dynamodb) one set of seen ids. -The query-result cache and the dedupe store are the layers that can be shared today. With the default `cache.backend: local`, each instance caches in its own memory, and an insert invalidates only the cache of the instance that made it. Every other instance keeps serving its cached results for the rows before the insert until each entry's TTL runs out, between 10 s and 1 h depending on how long the query took. With [`cache.backend: redis`](/configuration#cache), every instance reads and fills one Redis-compatible server, and an insert on any instance invalidates the cached results of every instance. The server is a standalone one or a Redis Cluster; Sentinel (`mode: sentinel`) refuses boot until [#656](https://github.com/Wave-RF/WaveHouse/issues/656), since the cache does not yet authenticate to the sentinels or refresh their topology. +The query-result cache is shared the same way. With the default `cache.backend: local`, each instance caches in its own memory, and an insert invalidates only the cache of the instance that made it. Every other instance keeps serving its cached results for the rows before the insert until each entry's TTL runs out, between 10 s and 1 h depending on how long the query took. With [`cache.backend: redis`](/configuration#cache), every instance reads and fills one Redis-compatible server, and an insert on any instance invalidates the cached results of every instance. The server is a standalone one or a Redis Cluster; Sentinel (`mode: sentinel`) refuses boot until [#656](https://github.com/Wave-RF/WaveHouse/issues/656), since the cache does not yet authenticate to the sentinels or refresh their topology. **What another instance can see.** Ingest is already asynchronous: `/v1/ingest` answers before the batch is inserted. Once the inserting instance's worker has written the batch to ClickHouse, it replaces the table's version token in Redis, and from then on a lookup on any instance misses and reads the new rows. The cache adds no delay of its own beyond that single write. The exceptions: @@ -581,7 +737,7 @@ If you skipped the drain, the boot's `WARN` line for each deleted stream (`delet ## Dead Letter Queue (DLQ) -A batch insert ClickHouse **rejects** is retried row by row; while the tenant's `dlq.enabled` is `true` for the table (the seed default — a hot-reloadable [settings directory](/settings-directory#dead-letter-queue) key, overridable per table), the rows ClickHouse rejects again are published to the tenant's own dead-letter stream (`DLQ_{tenant}`) under subjects `dlq.{tenant}.{table}` (`0` for a directory that holds the four files) instead of retrying forever. A batch whose tenant has no ClickHouse connection — one no longer served, or one no pool could be opened for, such as by the connection ceiling — skips the row-by-row retry, which no row of it could pass: its tenant's switch is read once for the whole batch, and a tenant no longer served has no switch to read, so its batch is always parked. A ClickHouse that cannot take inserts at all — down, unreachable, overloaded, read-only, or refusing the configured credentials — parks nothing: its rows stay in the tenant's ingest queue and are retried with backoff, counted one per row each time they are handed back by `wavehouse_ingest_retries_total`, so a long outage shows up as a growing ingest stream (and, at the tenant's `mq.max_bytes_gb`, as ingest `503`s), not as a full DLQ — see [Ingest Pipeline](/ingest-pipeline#when-clickhouse-cannot-take-an-insert). Monitor DLQ depth via `GET /v1/ops/dlq/stats`, per tenant (`?tenant=`; tenant `0` without it). +Under [`mq.backend: nats`](#external-nats) every tenant's parked rows go to the one shared dead-letter stream, under `.dlq.{tenant}.{table}`, not `DLQ_{tenant}`, and a long outage fills the partitions its tables are in (or a table's `maxMsgsPerSubject`) rather than its `mq.max_bytes_gb`; the rest of this section holds. A batch insert ClickHouse **rejects** is retried row by row; while the tenant's `dlq.enabled` is `true` for the table (the seed default — a hot-reloadable [settings directory](/settings-directory#dead-letter-queue) key, overridable per table), the rows ClickHouse rejects again are published to the tenant's own dead-letter stream (`DLQ_{tenant}`) under subjects `dlq.{tenant}.{table}` (`0` for a directory that holds the four files) instead of retrying forever. A batch whose tenant has no ClickHouse connection — one no longer served, or one no pool could be opened for, such as by the connection ceiling — skips the row-by-row retry, which no row of it could pass: its tenant's switch is read once for the whole batch, and a tenant no longer served has no switch to read, so its batch is always parked. A ClickHouse that cannot take inserts at all — down, unreachable, overloaded, read-only, or refusing the configured credentials — parks nothing: its rows stay in the tenant's ingest queue and are retried with backoff, counted one per row each time they are handed back by `wavehouse_ingest_retries_total`, so a long outage shows up as a growing ingest stream (and, at the tenant's `mq.max_bytes_gb`, as ingest `503`s), not as a full DLQ — see [Ingest Pipeline](/ingest-pipeline#when-clickhouse-cannot-take-an-insert). Monitor DLQ depth via `GET /v1/ops/dlq/stats`, per tenant (`?tenant=`; tenant `0` without it). ## Observability diff --git a/docs/src/content/docs/development.md b/docs/src/content/docs/development.md index a3d5b0d37..4738b3dda 100644 --- a/docs/src/content/docs/development.md +++ b/docs/src/content/docs/development.md @@ -341,11 +341,11 @@ Each test target writes `covdata` to `tmp/coverage//data/`, renders a tex | -------- | -------- | ------- | ------- | | Unit tests | `internal/*/_test.go` | No | `make test` | | SDK unit tests | `clients/ts/src/**/*.test.ts` | No | `make test-ts` (always includes coverage + gate) | -| Integration tests (Go) | `tests/integration/*_test.go`, `internal/cache/*_integration_test.go` | Yes | `make test-integration` | +| Integration tests (Go) | `tests/integration/*_test.go`, `internal/cache/*_integration_test.go`, plus `internal/mq/natsspike` and `internal/mq`'s integration-tagged tests | Yes | `make test-integration` | | E2E tests (SDK) | `tests/e2e/sdk/*.test.ts` | Yes | `make test-e2e` | - **Unit tests** live beside the code they test (e.g., `internal/discovery/discovery_test.go`). They use mocks or embedded NATS (in-process, no Docker needed). -- **Integration tests** use the `//go:build integration` build tag. In `tests/integration`, `TestMain` starts one ClickHouse testcontainer and a dynamodb-local one (for the DynamoDB dedupe backend's tests), and boots the production wiring against it through `app.New` (embedded NATS, ingest worker, sweeper, hub, the API server on a random loopback port); tests reach it via `env(t)` and create their own tables. DLQ tests use `assert.Eventually` with a 30-second timeout for the 5-second ingest worker batch window. `internal/cache`'s integration tests start their own containers instead — Redis, Valkey, Dragonfly and a one-node Redis Cluster — for the shared backend. `shared_cache_test.go` starts its own Redis testcontainer per test (`startRedis`) and boots extra, independent `cache.backend: redis` instances over that same ClickHouse (`bootRedisApp`), to exercise the cache shared across processes rather than one package in isolation. +- **Integration tests** use the `//go:build integration` build tag. In `tests/integration`, `TestMain` starts one ClickHouse testcontainer and a dynamodb-local one (for the DynamoDB dedupe backend's tests), and boots the production wiring against it through `app.New` (embedded NATS, ingest worker, sweeper, hub, the API server on a random loopback port); tests reach it via `env(t)` and create their own tables. `TestNATSBackend_EndToEnd` also starts a NATS container configured from `deployments/nats/values.yaml`, applies `deployments/nats/jetstream.yaml` to it through `internal/mq/natstest`, and boots two processes on `mq.backend: nats` against it. `TestMain` also builds the `wavehouse` binary while the containers start, so the build is not charged to the `-timeout`: the `TestRoles_*` tests run it as separate OS processes (two API, two then three ingest, one killed with `SIGKILL`) over NATS, Redis and dynamodb-local, and read each ingest process's shard ownership from its `/metrics`. DLQ tests use `assert.Eventually` with a 30-second timeout for the 5-second ingest worker batch window. `internal/cache`'s integration tests start their own containers instead — Redis, Valkey, Dragonfly and a one-node Redis Cluster — for the shared backend. `shared_cache_test.go` starts its own Redis testcontainer per test (`startRedis`) and boots extra, independent `cache.backend: redis` instances over that same ClickHouse (`bootRedisApp`), to exercise the cache shared across processes rather than one package in isolation. The same target also runs `internal/mq/natsspike`. That package pins the nats-server behavior the external-NATS topology depends on, against an in-process server with no Docker. It lives under `internal/mq` because only that tree may import NATS, and it runs here rather than in the unit suite because each test takes seconds and the unit suite has a 15-second limit per package. For the same reason the external NATS broker's tests (`internal/mq/external*_test.go`, including its run of the `mqtest` conformance suite) and the NATS KV lease tests (`internal/mq/lease_test.go`, including their run of the `coordtest` conformance suite) carry the `integration` tag inside `internal/mq`, and the target runs them by name, so the package's untagged tests stay in the unit suite alone. Shared test utilities live in `internal/testutil/`. The packages log through `slog.Default()`, so tests reach log output through `internal/testutil/logtest`: `logtest.Silence()` in a package's `TestMain` discards it, and `logtest.Capture(t, level)` routes it to a buffer for a test that asserts on log lines — such a test must not call `t.Parallel()`, because the default logger is process-wide. A test that starts the embedded broker keeps its store in `internal/testutil/storedir`'s `storedir.New(t)` rather than a bare `t.TempDir()` (`testutil.NewEmbeddedMQ` does): the NATS server can finish writing a consumer's state after `Close` returns, which fails `t.TempDir`'s one-shot removal, and `storedir` removes the store again until those writes have landed ([#442](https://github.com/Wave-RF/WaveHouse/issues/442)). @@ -462,7 +462,7 @@ WaveHouse/ │ ├── coord/ # Leases with fencing tokens (in-process Local, RunElected, coordtest suite) │ ├── dedupe/ # Optional deduplication (Reserve/Commit/Release; Pebble or DynamoDB) │ ├── discovery/ # ClickHouse schema introspection + validation -│ ├── ingest/ # Batch buffering + DLQ + Active Sweeper +│ ├── ingest/ # Batch buffering + DLQ + Active Sweeper + shard claims │ ├── keyenc/ # One escaping for composite keys (NATS subject tokens, cache keys, dedupe keys) │ ├── mq/ # MQ boundary: the only NATS/JetStream importer │ ├── observability/ # OpenTelemetry pipeline (traces/metrics/logs + Prometheus) diff --git a/docs/src/content/docs/durability.md b/docs/src/content/docs/durability.md index 175f9dda9..0d68da222 100644 --- a/docs/src/content/docs/durability.md +++ b/docs/src/content/docs/durability.md @@ -1,13 +1,13 @@ --- title: "Durability & Storage" -description: "What a WaveHouse ingest ack guarantees, why embedded JetStream fsyncs every publish, and how to tell whether your storage substrate can sustain it." +description: "What a WaveHouse ingest ack guarantees, why embedded JetStream fsyncs every publish while an external NATS cluster relies on replicas, and how to tell whether your storage substrate can sustain it." cloudCta: body: "Finding out your disk cannot sustain an fsync per publish is the kind of lesson that arrives at peak traffic. WaveHouse Cloud runs on storage already benchmarked against this page, with the WAL sizing and retention handled for you." sidebar: order: 11 --- -WaveHouse buffers every ingested event in embedded NATS JetStream before the [ingest worker](/ingest-pipeline) drains it into ClickHouse. That buffer lives on disk at `/nats`, and **WaveHouse runs JetStream in its strictest durability mode**: every publish is `fsync`'d to non-volatile storage before the producer is acknowledged. +By default (`mq.backend: embedded`), WaveHouse buffers every ingested event in embedded NATS JetStream before the [ingest worker](/ingest-pipeline) drains it into ClickHouse. That buffer lives on disk at `/nats`, and **WaveHouse runs the embedded JetStream in its strictest durability mode**: every publish is `fsync`'d to non-volatile storage before the producer is acknowledged. An [external NATS cluster](#with-an-external-nats-cluster) is durable through its replicas instead, and WaveHouse does not require it to `fsync` every publish. This is a deliberate, strong guarantee — but it makes your ingest latency a direct function of your storage's `fsync` latency. On managed cloud block storage that is effectively free; on some commodity or virtualized substrates the `fsync` tail balloons into seconds and ingest visibly suffers. This page explains the contract, where it is cheap versus expensive, and how to measure your storage before you trust it. @@ -23,10 +23,19 @@ This is the strongest mode JetStream offers. It is stronger than the default, wh | Mode | Ack means | Crash exposure | Throughput | | --- | --- | --- | --- | -| **`SyncAlways` (WaveHouse today)** | data is `fsync`'d to disk | none for acked events | bounded by `fsync` latency | +| **`SyncAlways` (the embedded broker)** | data is `fsync`'d to disk | none for acked events | bounded by `fsync` latency | | Periodic group commit (default JetStream) | data is in the OS page cache | up to one sync interval of acked-but-unflushed events | bounded by memory/CPU | -WaveHouse does not currently expose a knob to relax this — `SyncAlways` is always on. Exposing a configurable group-commit interval (`mq.sync_interval`) is tracked in [#139](https://github.com/Wave-RF/WaveHouse/issues/139). +WaveHouse does not currently expose a knob to relax this — `SyncAlways` is always on for the embedded broker. Exposing a configurable group-commit interval (`mq.sync_interval`) is tracked in [#139](https://github.com/Wave-RF/WaveHouse/issues/139). + +## With an external NATS cluster + +Under [`mq.backend: nats`](/deployment#external-nats) the buffer is your NATS cluster, not `/nats`, and the `200` means the partition stream has acked the publish: + +- **At 3 or more replicas,** the stream acks only once a Raft quorum of its servers has stored the event. WaveHouse does not require `sync_always` there, and the shipped Helm values do not set it: a quorum spread across failure domains (zones, racks or hosts) survives losing any one of them, which is the durability WaveHouse relies on. What can lose an acked event is losing a quorum's servers at once, before they sync. +- **At one replica,** the server's `sync_interval` (2 minutes unless you set it) governs: this is the periodic group-commit row in the table above, and a crash of that server loses what it stored since its last sync. Boot reports a stream with fewer than 3 replicas as a `recommended` finding and still starts, so a one-server development cluster works. In production, run 3 replicas, or set `sync_always` on a one-server cluster and the rest of this page applies to its disk. A stream with `persist_mode: async` (one replica only) flushes in the background even under `sync_always`, so a crash of the server process alone can lose acked events; boot refuses it on an ingest partition. + +WaveHouse does not choose the cluster's storage settings; the rest of this page describes the embedded server. What does not change is that no limit drops an event before it is written: the partition streams have no age limit, and a full one refuses new events with `503` rather than dropping old ones. A full partition refuses every tenant whose events it holds, not one tenant. The replay history is a separate stream whose `max_age` you set. It is best effort: the partitions republish each row to it, so a row republished while it is unavailable reaches ClickHouse but not SSE replay, and it never holds up ingest. Every tenant's parked rows share one dead-letter stream. ## Why the fsync tail is your ingest floor diff --git a/docs/src/content/docs/ingest-pipeline.md b/docs/src/content/docs/ingest-pipeline.md index ab08c88f1..18bc714e4 100644 --- a/docs/src/content/docs/ingest-pipeline.md +++ b/docs/src/content/docs/ingest-pipeline.md @@ -17,13 +17,15 @@ It is deliberately detailed: this is a hot, concurrency-heavy path, and the goro | `backoff.go` | The retry backoff per ClickHouse pool: one outage backs off every table on it together, probing once per window; a failure of one table (read-only, too many parts or mutations, a missing grant) backs off that table alone — see [When ClickHouse cannot take an insert](#when-clickhouse-cannot-take-an-insert) | | `compact.go` | `EncodeCompactRow` — renders one record as a `JSONCompactEachRow` line over the table's **insertable** columns, in declaration order. Serialization only: it validates nothing and judges no value | | `sweeper.go` | The **Active Sweeper** — every minute, asks the MQ to purge the events that are both written to ClickHouse and past the SSE gap window (the purge arithmetic below lives in `internal/mq/purge.go`) | +| `claims.go` | `ClaimShards` — over the external broker's shards, narrows the worker to this process's share: the membership lease, the per-tick assignment, handover by drain then release, the reset of a dead owner's shards at takeover, each shard's share of the `MaxHeld` budget of rows held, the halt that lets the worker write what it holds before a clean stop releases its shards, and the shard gauges. The behavior is in [Scaling ingest processes](/deployment#scaling-ingest-processes). | +| `assign.go` | `assignUnits` — rendezvous hashing capped at ⌈units/slots⌉, the same owners in every process for the same view | | `types.go` | `EventMessage` wire format and the `BufferConsumerName` constant | The pipeline is **insert-only**. (Upgrading across the v2 envelope? [Drain the queue first](/deployment#upgrading-across-the-v2-ingest-envelope).) The wire format carries `{table_name, scope, received_timestamp, format, columns, row}`: `row` is one `JSONCompactEachRow` line — a positional JSON array — and `columns` names its positions — the table's insertable columns, in declaration order (a `MATERIALIZED` or `ALIAS` column cannot be named in an `INSERT`, so it is not part of the row's contract). (`scope` is reserved and always `""` today.) Each NATS message is its own envelope, so the names ride along per record; where they are carried once is the `INSERT` the worker emits per group. The worker parses the envelope, groups a batch by column list, and bulk-`INSERT`s each group as `INSERT INTO … (cols) FORMAT JSONCompactEachRow` — schema validation already happened at the HTTP ingest handler, before publish. Non-insert mutations go through `POST /v1/ops/query` (admin-only) or an operator-authored [pipe that writes](/pipes#pipes-that-write). ## High-level shape -Each tenant's events are queued on a JetStream stream of its own. One process holds one durable consumer on each tenant's stream, delivered into one handler, and fans events out to a goroutine per tenant table — the tenant is the subject's leading token. Each tenant's table batches independently and POSTs to ClickHouse over the HTTP interface (`JSONCompactEachRow`). When ClickHouse rejects a bulk insert the batch is re-inserted row by row, so a single poison row can't sink it: clean rows ack, and only the rows ClickHouse rejects again go to the dead-letter stream. When ClickHouse cannot take the insert at all — down, overloaded, read-only — nothing is dead-lettered for it: the batch goes back to the queue and is retried with backoff. The one exception is a multi-row batch refused for its size (too many partitions, the memory limit), which is first split row by row like a rejected one; if a row then fails any way but a rejection, that row and every row after it go back. A batch whose tenant has no ClickHouse connection — one no longer served, or one no pool could be opened for (such as by the connection ceiling) — skips the row-by-row retry, which no row of it could pass, and meets the dead-letter switch once, whole; a tenant no longer served has no switch to read, so its batch is parked. An envelope the worker cannot *read* — malformed JSON, an unknown row `format`, or columns and a row that don't pair — never reaches a table loop at all: `parseMsg` parks it on the same dead-letter stream, or, where the DLQ is off for the table, acks and drops it rather than redelivering a message that can never insert. A separate sweeper reclaims stream storage. +With the embedded broker each tenant's events are queued on a JetStream stream of its own, and one process holds one durable consumer on each tenant's stream (for `mq.backend: nats`, see [Scaling to multiple instances](#scaling-to-multiple-instances)), delivered into one handler, and fans events out to a goroutine per tenant table — the tenant is the subject's leading token. Each tenant's table batches independently and POSTs to ClickHouse over the HTTP interface (`JSONCompactEachRow`). When ClickHouse rejects a bulk insert the batch is re-inserted row by row, so a single poison row can't sink it: clean rows ack, and only the rows ClickHouse rejects again go to the dead-letter stream. When ClickHouse cannot take the insert at all — down, overloaded, read-only — nothing is dead-lettered for it: the batch goes back to the queue and is retried with backoff. The one exception is a multi-row batch refused for its size (too many partitions, the memory limit), which is first split row by row like a rejected one; if a row then fails any way but a rejection, that row and every row after it go back. A batch whose tenant has no ClickHouse connection — one no longer served, or one no pool could be opened for (such as by the connection ceiling) — skips the row-by-row retry, which no row of it could pass, and meets the dead-letter switch once, whole; a tenant no longer served has no switch to read, so its batch is parked. An envelope the worker cannot *read* — malformed JSON, an unknown row `format`, or columns and a row that don't pair — never reaches a table loop at all: `parseMsg` parks it on the same dead-letter stream, or, where the DLQ is off for the table, acks and drops it rather than redelivering a message that can never insert. With the embedded broker, a separate sweeper reclaims stream storage. ```mermaid flowchart LR @@ -55,7 +57,7 @@ flowchart LR Stream -.->|"DeliverByStartTime gap-fill"| Replay["replay consumer, one per connection
(the tenant's subject)"] ``` -Note the stream is **dual-use**: it is both the durable buffer feeding the worker and the replay buffer that SSE clients gap-fill from. That is why a custom sweeper exists instead of plain work-queue auto-deletion (see [Scaling out](#scaling-to-multiple-instances)). +Note the embedded broker's stream is **dual-use**: it is both the durable buffer feeding the worker and the replay buffer that SSE clients gap-fill from. That is why a custom sweeper exists there instead of plain work-queue auto-deletion; `mq.backend: nats` splits the two roles instead, with work-queue partitions and a separate history stream (see [Scaling out](#scaling-to-multiple-instances)). :::note[Omitted columns on `Nullable` columns with a default] Inserts also pin `input_format_null_as_default=1`. A positional row has one value per insertable column and no way to say "absent", so a field the record omitted rides as an explicit `null` in its slot. That setting turns the `null` back into the column's default for a **non-nullable** column, matching what omitting the key did under `JSONEachRow` — but on a `Nullable(T) DEFAULT …` column ClickHouse stores `NULL` whatever the setting says, because only an *absent* key ever took the default. So such a column now stores `NULL` where it previously took its default. Verified on ClickHouse 26.6.3. @@ -205,7 +207,7 @@ Messages still sitting in `msgChan` or the consumer's prefetch buffer at shutdow Delivery can end underneath a running worker: the durable consumer is deleted, or the MQ connection closes. The broker client reports either only through an asynchronous error callback and then stops delivering — no message ever arrives to say so, so a loop that only watches `msgChan` would wait forever while the API kept accepting events nothing writes. `mq.Consumer.Consume` therefore returns a `failed` channel next to `stop` (`mq.ErrDeliveryEnded`, wrapping the broker's reason), and `dispatchLoop` selects on it beside `ctx.Done()` and `msgChan`. On a failure it runs the same bottom-up drain as a shutdown — the rows already in hand are flushed and acked, not abandoned — and then reports the error on the worker's own `failed` channel. A consumer that cannot start at all takes the same path. -The worker does not try to revive the consumer. The app's ingest-worker component returns the error from `app.Run`, which stops every other component and exits non-zero, the same way any failed component does; the supervisor's restart recreates the durable consumer at boot, and everything unacked is redelivered (at-least-once). Passing conditions the client also reports through that callback (a missed heartbeat, a leadership change) are logged at `WARN` and do not end the worker. With the embedded broker (`DontListen`, no external client that could delete a durable) this path is hard to reach. A tenant's queue, opened at runtime, that the consumer cannot join is not a way in: the broker keeps that queue closed to the tenant's publishes, which answer `503` until a reload or a publish joins the consumer, and the worker goes on delivering every other tenant's rows. It matters more once a remote broker exists. +The worker does not try to revive the consumer. The app's ingest-worker component returns the error from `app.Run`, which stops every other component and exits non-zero, the same way any failed component does; with the embedded broker the supervisor's restart recreates the durable consumer at boot, and everything unacked is redelivered (at-least-once). Under `mq.backend: nats` WaveHouse never creates the durable: the restarted process waits `mq.nats.topology_wait` for the operator to recreate it, then refuses to boot naming it. Passing conditions the client also reports through that callback (a missed heartbeat) are logged at `WARN` and do not end the worker; a consumer leadership change, and a pull refused because another process holds the shard's pin, are logged at `DEBUG`. With the embedded broker (`DontListen`, no external client that could delete a durable) this path is hard to reach. A tenant's queue, opened at runtime, that the consumer cannot join is not a way in: the broker keeps that queue closed to the tenant's publishes, which answer `503` until a reload or a publish joins the consumer, and the worker goes on delivering every other tenant's rows. Under `nats` it is reachable: an operator deleting one of the configured shard durables, or a connection closed for good. The end of a removed partition's delivery, or of a durable past a lowered V (see [Choosing and changing N](/deployment#choosing-and-changing-n) and [Choosing and changing V](/deployment#choosing-and-changing-v)), only stops draining it. ## When ClickHouse cannot take an insert @@ -224,7 +226,7 @@ Two `Unavailable` codes can be earned by a batch's size alone, each of its rows A batch that is neither `Rejected` nor split is handed back to the queue with `mq.Message.NakWithDelay` (`retryLater`), counted one per row each time it is handed back by `wavehouse_ingest_retries_total{table, reason}` (`reason` is `unavailable`, `denied` or `unknown`, or `backoff` for rows turned away without a try). The delay comes from `backoff.go`, one backoff per ClickHouse pool — the target's URL, user and database, so every table of every tenant sharing that URL, user and database waits together (tenants on the same server under another database or user back off, and probe, on their own): 1 s doubling to a 30 s cap, each delay jittered down to half so the tables of one outage do not come back in step. While the window runs, a flush for any table on that pool makes no request, and a row that arrives is handed straight back rather than buffered, so an outage's backlog waits in the queue, not in the worker; when it elapses one flush probes — rows that arrive meanwhile are handed back with a delay of at least half a second, since a probe to a server dropping packets can take the whole 30 s client timeout — and any answer that is not an outage — a success, or a rejected row — closes it. A failure ClickHouse reports for **one table** — `TABLE_IS_READ_ONLY`, `TABLE_IS_PERMANENTLY_READ_ONLY`, `TOO_MANY_PARTS`, `TOO_MANY_MUTATIONS`, and `ACCESS_DENIED`, which is usually a grant missing on that table (`chconn.TableScoped`) — backs off that table alone, on its own backoff: the server answered, so its other tables keep inserting — until the failing table's waiting rows fill its tenant's `maxAckPending` (below) — and their successes do not close the failing table's backoff. The outage is logged at `WARN` when it starts and at most every 30 s while it lasts, and at `INFO` when ClickHouse takes inserts again. If ClickHouse cannot take a row in the middle of row-by-row isolation — it stops answering, or answers with any class but `Rejected` — isolation stops there: the rows already inserted stay acked, the rows already rejected stay parked, and the row that met the outage and every row after it go back to the queue. -Rows waiting out an outage stay unacked in their tenant's ingest stream, so the [Active Sweeper](#the-active-sweeper) cannot purge past them and they count toward that tenant's `maxAckPending`: a long outage fills each affected tenant's stream to its `mq.max_bytes_gb` and that tenant's ingest answers `503` — backpressure, with nothing lost and nothing parked. The rows of a failure of one table count the same way, so one that lasts — a missing grant, a permanently read-only table — suspends delivery for every table of that tenant once its waiting rows reach `maxAckPending`, and the tenant's other tables stop inserting until it is fixed. A batch whose tenant has no ClickHouse connection at all is a different case and keeps its own rule (`parkBatch`, above). +Rows waiting out an outage stay unacked in their tenant's ingest stream, so the [Active Sweeper](#the-active-sweeper) cannot purge past them and they count toward that tenant's `maxAckPending`: a long outage fills each affected tenant's stream to its `mq.max_bytes_gb` and that tenant's ingest answers `503` — backpressure, with nothing lost and nothing parked. The rows of a failure of one table count the same way, so one that lasts — a missing grant, a permanently read-only table — suspends delivery for every table of that tenant once its waiting rows reach `maxAckPending`, and the tenant's other tables stop inserting until it is fixed. A batch whose tenant has no ClickHouse connection at all is a different case and keeps its own rule (`parkBatch`, above). This paragraph describes the embedded broker; under `mq.backend: nats` a stall is per shard and a full partition refuses every tenant in it (see [Limits that differ from the embedded queue](/deployment#limits-that-differ-from-the-embedded-queue)). A retry does not keep arrival order, and it can insert a row twice. Rows handed back return after their delay mixed in with rows that arrived since, so after an outage a table's rows reach ClickHouse out of the order they were ingested. A plain `MergeTree` sorts every part by its `ORDER BY` key, so no query sees the order; it matters where insert order picks the winner — a `ReplacingMergeTree` without a version column keeps the last row inserted, and a `CollapsingMergeTree` expects a cancel row after the row it cancels. Give such a table a version column the producer sets in the record (`ReplacingMergeTree(ver)`, or `VersionedCollapsingMergeTree`) — not an insert-time `DEFAULT now64()`, which stamps a delayed retry later than the rows that overtook it. And a retried failure can follow an INSERT ClickHouse did commit — the client's 30 s timeout, a connection reset or EOF after the body was sent, `UNKNOWN_STATUS_OF_INSERT` — so delivery after one of them is at-least-once: the first attempt may have landed, and because the retried rows come back regrouped with newer ones, ClickHouse's insert deduplication does not match them. Nor does `dedupe.enabled`, which drops a repeated publish at the HTTP edge, before the queue. @@ -236,7 +238,7 @@ Several layers throttle the pipeline, inner to outer: 2. **`msgChan`** (cap `maxBatch*2`) — when full, the consume callback blocks and delivery pauses. 3. **`pullMaxMessages`** — nats.go's client-side prefetch buffer in front of `msgChan`, shared by the tenants' streams (at least one message each). 4. **`maxAckPending`** — the server suspends a tenant's delivery once this many of its messages are delivered-but-unacked; no other tenant's delivery waits on it. The outermost in-memory bound, and a per-tenant one: while ClickHouse stalls, the worker can hold up to `maxAckPending` rows for every tenant served. -5. **`MaxBytes` + `DiscardNew`** on each tenant's stream (its `mq.max_bytes_gb` in the [settings directory](/settings-directory#message-queue), resized in place on reload) — when it fills (e.g. ClickHouse is down so nothing acks/purges), that tenant's new publishes are rejected and the API returns 503. +5. **`MaxBytes` + `DiscardNew`** on each tenant's stream (its `mq.max_bytes_gb` in the [settings directory](/settings-directory#message-queue), resized in place on reload) — when it fills (e.g. ClickHouse is down so nothing acks/purges), that tenant's new publishes are rejected and the API returns 503. Steps 4 and 5 are the embedded broker's; under `mq.backend: nats` a shard slows to what keeps its pin at its share of the process's 10,000-row budget of unsettled rows (an even share over its shards, never under 1,000, so a process with more than ten shards holds 1,000 per shard), which pauses every table in that shard, and a full partition refuses every tenant in it (see [Limits that differ from the embedded queue](/deployment#limits-that-differ-from-the-embedded-queue)). | Knob | Default | Meaning / invariant | | --- | --- | --- | @@ -261,39 +263,41 @@ flowchart TD Purge -->|"deletes msgs that are BOTH
written to ClickHouse AND past the gap window"| Stream[("INGEST_TENANT stream")] ``` -`MIN(ackFloor+1, gapSeq)` is the safety argument: never purge past what is in ClickHouse, and never past the SSE replay window. If ClickHouse is down the `AckFloor` stops advancing, purging freezes, and the stream fills toward `MaxBytes` — backpressure by construction. The sweeper is one of `app.Run`'s components (`Sweeper.Start` blocks until the run context is canceled), but an interrupted sweep is harmless and idempotent, so it returns on `ctx.Done()` with no drain of its own — unlike the worker's bounded `stopFunc`. It runs under the `sweeper` lease (`coord.RunElected`), so only the process holding the lease sweeps; with the in-process coordinator that is always the one process. +`MIN(ackFloor+1, gapSeq)` is the safety argument: never purge past what is in ClickHouse, and never past the SSE replay window. If ClickHouse is down the `AckFloor` stops advancing, purging freezes, and the stream fills toward `MaxBytes` — backpressure by construction. The sweeper is one of `app.Run`'s components (`Sweeper.Start` blocks until the run context is canceled), but an interrupted sweep is harmless and idempotent, so it returns on `ctx.Done()` with no drain of its own — unlike the worker's bounded `stopFunc`. It runs under the `sweeper` lease (`coord.RunElected`), so only the process holding the lease sweeps; with the in-process coordinator that is always the one process. Under [`mq.backend: nats`](/deployment#external-nats) no sweeper is wired: the partition streams use work-queue retention, so the server deletes each row once the worker acks it, and the replay history is a separate stream the server expires by its `max_age`. The API processes instead warn (`ExternalNATS.CheckReplayWindows`), at boot and after each reload, once per tenant and window, about a gap window longer than that `max_age`. ## Scaling to multiple instances -Today this is a **single-process** design (embedded, in-process NATS — the "connection" cannot blip independently of the process, so there is intentionally no reconnect logic). Running multiple instances against a real/clustered NATS changes several things: +The embedded broker is single-process by construction: it listens on no port, so its "connection" cannot blip independently of the process. [`mq.backend: nats`](/deployment#external-nats) is how several processes share one queue: ```mermaid flowchart TD - subgraph Cluster["Clustered NATS (Replicas: 3)"] - S["one shared ingest stream"] + subgraph NATS["Operator's NATS JetStream"] + P0[("ingest partition 0
work queue")] + P1[("ingest partition N-1")] + H[("history stream
limits, max_age")] + D[("dead-letter stream")] end - S --> P0["partition 0"] - S --> P1["partition 1"] - S --> P2["partition 2"] - P0 --> IA["instance A (pinned owner)"] - P1 --> IB["instance B (pinned owner)"] - P2 --> IA - IA --> CH[("ClickHouse
idempotent inserts")] - IB --> CH + API["API processes"] -->|"publish: jump hash of tenant.table → (partition, shard)"| P0 + API --> P1 + P0 -->|"one pinned-client durable per shard"| W["ingest workers (one per shard at a time)"] + P1 --> W + P0 -. republish .-> H + P1 -. republish .-> H + H -->|"per-process consumer"| Hub["each API process's SSE hub + replay"] + W --> CH[("ClickHouse")] + W --> D ``` -What will need to change, and the trade-offs (discussed at length on the batching work): - -- **Work distribution.** Either a *shared* durable pull consumer (competing consumers — coordination-free, but a hot table's rows spread across instances, shrinking per-instance batches), or **partitioned consumer groups** that hash by the tenant and table subject tokens so a tenant's table always lands on one owner (pinned consumer → per-table affinity + automatic failover, at the cost of an assignment layer). -- **Idempotent inserts become mandatory.** At-least-once + redelivery-on-crash means another instance can re-insert a batch the dead one had written but not acked. Use `ReplacingMergeTree` (or a dedup key). A single instance already re-inserts after a crash, or after an insert whose outcome it could not see ([When ClickHouse cannot take an insert](#when-clickhouse-cannot-take-an-insert)); more instances make it routine. -- **NATS resilience.** Remote NATS needs explicit reconnect/backoff for the connection itself — the embedded path never dials out, so there is nothing to reconnect. The `Consume` error handler that detects a dead consumer already lives in `embedded.go` and needs no change for a remote broker. -- **The sweeper.** Its single-`AckFloor` model assumes one consumer. With per-table/partition consumers you either rework it to purge below the *minimum* AckFloor across consumers, or — cleaner — **split the dual-use stream**: a `WorkQueuePolicy` work stream (auto-deletes on ack, no sweeper) plus a `MaxAge` replay stream (server-expired by time, no sweeper), joined by stream sourcing. That deletes the sweeper entirely, at the cost of duplicating the in-flight overlap on disk. Keeping the sweeper instead needs one sweeper per shared stream: it already campaigns for a lease (`internal/coord`), so this is a shared coordinator backend rather than new election code — and a brief overlap during a handoff is tolerable: an overlap cannot lose ClickHouse data, since every sweep stops at the consumer's ack floor; it can only trim SSE replay history, and only when the two holders' settings views differ (one still reading a shorter `stream.gap_window_minutes`, or missing a tenant, after a reload the other has applied) — which fencing would not prevent either. +- **Work distribution.** Each partition is split into V shards, and a table's rows all go to one shard. Each shard has its own durable with the `pinned_client` priority policy, so the server delivers a shard to one pulling process at a time: a table's rows are batched and written by one process at a time. The ingest processes share the shards out: each holds a membership lease (`ingest.m`), and every process computes the same owner for every shard from the live members, by rendezvous hashing capped at an even share (`ClaimShards`). A shard given up stops fetching but keeps its pin until the rows it delivered are written, then releases it, so the next owner receives at once and only newer rows; a stopping process first halts every shard the same way (`mq.Halter`), writes what it holds, and only then releases them. A shard whose owner died is reset to its ack floor before its new owner pulls it, so the rows the dead owner held come back at once rather than after `ack_wait`; the broker refuses the reset while the shard is pinned or was active within its pinned TTL. The worker never blocks a shard's pulls: at the shard's share of the process's 10,000-row budget of unsettled rows (an even share over its shards, never under 1,000, so a process with more than ten shards holds 1,000 per shard), the shard fetches only what keeps its pin (a row every 5 seconds, at most one `ack_wait` of them past its share, then pulls that take none), so a stuck table neither loses its shard nor stalls the process's other shards. +- **Idempotent inserts matter more.** At-least-once delivery plus redelivery after a crash means another process can re-insert a batch the dead one had written but not acked. Use `ReplacingMergeTree` (or a dedup key). A single process already re-inserts after a crash, or after an insert whose outcome it could not see ([When ClickHouse cannot take an insert](#when-clickhouse-cannot-take-an-insert)); several make it routine. +- **NATS resilience.** The external broker reconnects on its own, with backoff; while it is disconnected a publish answers `503` with `Retry-After: 5`, and consumption resumes after the reconnect. A consumer whose delivery ends for good (one of its configured shard durables deleted, or its stream, even while no pull of it is waiting, which the next pull finds; a durable that no longer fits the worker's config; or the connection closed) ends the worker and the process, as the embedded one does. +- **The sweeper.** Work-queue retention deletes each row once it is acked, one row at a time, so one tenant's unwritten rows never hold back another's reclaim, which a shared ack floor would, and it keeps a row no consumer covers rather than dropping it. SSE replay reads the history stream, which the partitions republish to and which expires by `max_age`; it is best effort, so an outage of the history costs SSE the rows republished meanwhile and never holds up ingest. So there is nothing for the sweeper to purge. ## Deferred / not yet implemented Tracked under [#191](https://github.com/Wave-RF/WaveHouse/issues/191): - **Pipelining beyond coalescing** — more than one insert in flight per tenant table (with a documented bound), once benchmarks justify the added concurrency. -- **`tableLoop` reaping** — loops are spawned per distinct tenant table and never reaped; safe while tenants and table names are bounded (a settings folder per tenant, schema-validated tables, in-process publishers only). Needs idle-reaping before untrusted/remote publishers can create unbounded cardinality. Tracked in [#263](https://github.com/Wave-RF/WaveHouse/issues/263). -- **Per-table / partitioned consumers** and the **two-stream retention redesign**. +- **`tableLoop` reaping** — loops are spawned per distinct tenant table and never reaped; safe while tenants and table names are bounded (a settings folder per tenant, schema-validated tables, only WaveHouse publishing). Needs idle-reaping before untrusted publishers can create unbounded cardinality. Under `mq.backend: nats`, keep publish rights on `.ingest.>` to the `wavehouse` user alone: any other publisher bypasses schema validation. Tracked in [#263](https://github.com/Wave-RF/WaveHouse/issues/263). +- **A hot table**: a table lives in one shard, so one process writes it. Spreading one table's rows over several shards, which gives up its order and single writer, waits for a measured need. The embedded broker keeps one stream per tenant and the sweeper. - **Parallel e2e test files.** The e2e suite now isolates tables **per file** (`tests/e2e/sdk/tables.ts` — each file gets its own `clicks_`/`events_`/`users_`), so cross-file *data* contamination is structurally impossible. Running the files in parallel (dropping `maxWorkers: 1` in `vitest.config.ts`) is still deferred: several files do read-modify-write on the **single global policy document** and `streaming.test.ts` flips the global `default_role`, so concurrent files would race those writes. Parallelism needs per-table policy storage with atomic per-table updates first — tracked in [#214](https://github.com/Wave-RF/WaveHouse/issues/214). diff --git a/docs/src/content/docs/settings-directory.mdx b/docs/src/content/docs/settings-directory.mdx index 5d54d7ba0..dc4be85c7 100644 --- a/docs/src/content/docs/settings-directory.mdx +++ b/docs/src/content/docs/settings-directory.mdx @@ -132,7 +132,7 @@ The tenant tunables. Every key is required (a missing one is a validation error) | `schema.refresh_interval` | `60` | Seconds between ClickHouse table-schema re-discoveries (`>= 1`), per tenant; a reloaded value takes effect from the next refresh cycle, and the first periodic refresh lands at a random point within the interval. Schemas are also refreshable on demand via `POST /v1/ops/schema/refresh` (admin-only). | | `stream.keepalive_interval` | `30` | Seconds (`>= 1`) a quiet `GET /v1/stream` connection may go without a write before the server sends a `:` keepalive comment — keep it under your proxy's idle timeout; see [Streaming](#streaming). | | `stream.keepalive_buckets` | `3` | Load-spreading (`>= 1`): connections are spread across N buckets so each tick nudges ~1/N of live streams. Most deployments leave it. | -| `stream.gap_window_minutes` | `15` | Minutes (`>= 0`) of written-to-ClickHouse history the Active Sweeper keeps in NATS for `Last-Event-ID` gap-fill; applies from the next sweep. | +| `stream.gap_window_minutes` | `15` | Minutes (`>= 0`) of written-to-ClickHouse history the Active Sweeper keeps in NATS for `Last-Event-ID` gap-fill; applies from the next sweep. Under [`mq.backend: nats`](/deployment#external-nats) the history stream's `max_age` decides instead, and the API processes check at boot and after each reload, warning once per tenant and window about a tenant whose window is longer. | | `mq.max_bytes_gb` | `50` | Disk budget (GB, `>= 1`) for the tenant's embedded NATS ingest stream (`INGEST_{tenant}`); its dead-letter stream (`DLQ_{tenant}`) gets a tenth of it. A reload updates the live streams in place. See [Message Queue](#message-queue). | | `cors.allowed_origins` | `["*"]` | Allowed CORS origins, applied per request. `"*"` allows any browser origin. WaveHouse is a Bearer-token API — `Access-Control-Allow-Credentials` is intentionally never sent, so this allowlist controls *which origins can read responses*, not cookie scope. Tighten to your frontend's exact origin(s) in production (e.g. `["https://dashboard.example.com", "http://localhost:3000"]`). An empty list `[]` denies every browser origin (no `Access-Control-Allow-Origin` is ever sent); `"*"` is the only allow-all spelling. Over [a nested settings directory](/deployment#the-nested-settings-directory) each tenant's list decorates its own responses, the preflight included; which list answers a preflight, the tenant-exempt routes, and a refused request is [spelled out there](/deployment#multi-tenant-deployments). | @@ -181,7 +181,7 @@ The tenant tunables. Every key is required (a missing one is a validation error) } ``` -What stays in boot config is only what cannot change under a running process — the implementation each layer runs on (`mq.backend`, `cache.backend`, `dedupe.backend`, `coord.backend`) and a shared backend's connection (`cache.redis`, `dedupe.dynamodb`), how a dedupe claim behaves (`dedupe.lease`, `dedupe.reserve_concurrency`), the process's `roles`, resource sizing (`data_dir`, `cache.l1_max_cost`, `clickhouse.max_total_conns`), the listeners, the observability exporters — and the **secrets**: `clickhouse.password`, `cache.redis.password`, `auth.jwt_secret`, `auth.operator_key`. Secrets never belong in a tracked JSON file, so they stay in the environment and are combined with the wiring here on every (re)connect; rotating one is a restart. See [Configuration](/configuration). Everything else lives here and reloads. +What stays in boot config is only what cannot change under a running process — the implementation each layer runs on (`mq.backend`, `cache.backend`, `dedupe.backend`, `coord.backend`) and a shared backend's connection (`mq.nats`, `coord.nats`, `cache.redis`, `dedupe.dynamodb`), how a dedupe claim behaves (`dedupe.lease`, `dedupe.reserve_concurrency`), the process's `roles`, resource sizing (`data_dir`, `cache.l1_max_cost`, `clickhouse.max_total_conns`), the listeners, the observability exporters — and the **secrets**: `clickhouse.password`, `cache.redis.password`, `auth.jwt_secret`, `auth.operator_key`. Secrets never belong in a tracked JSON file, so they stay in the environment and are combined with the wiring here on every (re)connect; rotating one is a restart. See [Configuration](/configuration). Everything else lives here and reloads. ## Deduplication @@ -190,7 +190,7 @@ Every per-tenant dedupe knob lives here. Where the seen ids are kept (`dedupe.ba - `dedupe.enabled` (seed default `false`) — turns deduplication on. Hot-reloadable: a reload that flips it opens or closes this tenant's store (in the embedded Pebble instance at `/pebble`, or its share of the DynamoDB table under `dedupe.backend: dynamodb`), so no restart is needed; seen ids persist across an off/on cycle. If the store fails to open on a reload, the failure is logged and ingest fails closed (`503 dedupe store unavailable`, `Retry-After: 5`) until it opens — the files asked for dedupe, so publishing un-deduped is not a fallback. With `dedupe.backend: pebble` that is the next reload or restart, and at boot a failed open over a flat directory refuses to start, like every other store; with `dynamodb` it is the background retry described below. A record that lands in the instant of the flip itself is published un-deduped: if the settings already say on but the store is not yet open, it's counted by `wavehouse_ingest_dedupe_disabled_total`; in the reverse case (settings already say off, store still open) the handler skips dedupe like any other disabled record and nothing is counted. That counter should only ever tick during a reload, so a steadily climbing rate means the store and the settings have come apart. Over [a nested directory](/deployment#the-nested-settings-directory) with `dedupe.backend: pebble`, every tenant's seen ids live in that one instance, each key led by its tenant and table, and it is open while any tenant's switch is on: each tenant's store follows its own folder's `dedupe.enabled` the same way; a tenant's seen ids are never another's; a rejected or removed folder closes its tenant's store and keeps its seen ids for the folder that restores it; and if that instance fails to open, at boot or on reload, every tenant with dedupe on fails closed — its ingest answers `503 dedupe store unavailable` (`Retry-After: 5`) until a reload opens it — while the tenants with dedupe off carry on. Under `dedupe.backend: dynamodb` the table check plays the instance's part, in either shape: the table is checked whether or not any tenant's switch is on, and a table that fails it fails every tenant with dedupe on closed until the check, retried in the background and at once after every reload, passes. Only a misconfigured table (missing, the wrong key schema, access denied) over a flat directory whose tenant has dedupe on refuses boot instead ([Configuration](/configuration#dynamodb-dedupe)). - `dedupe.id_field` (seed default `event_id`) — JSON field name in the ingest body used as the dedup key. An id is a duplicate only within its own tenant and table: the same value in two tables is two ids. An id longer than 1,024 bytes once escaped (every byte but an ASCII letter, digit, `_` or `-` takes three) is stored as its SHA-256, counted by `wavehouse_dedupe_hashed_id_total`. While its record is being published, an id is held for its lease ([`dedupe.lease`](/configuration#dedupe), 30 seconds by default): another request carrying the same id meanwhile gets `503` (`a request with the same dedupe id is in flight`) with the lease, in whole seconds, as `Retry-After` — see [the ingest errors](/api#post-v1ingesttabletable--ingest-data). An id is committed only after its record is published; if that commit fails (counted by `wavehouse_ingest_dedupe_commit_failed_total`, which should stay at zero), the record is still answered `ok` and the id lapses with its lease: a retry of it before then answers in-flight, one inside the ingest queue's two-minute duplicate window is dropped there by its idempotency key, and one after that is stored again. - `dedupe.require_id` (seed default `false`) — controls what happens to a row missing `id_field`, or carrying it as `null` (which can't be deduped, so idempotency wouldn't apply to it). Such a row is always logged at `WARN` and counted by `wavehouse_ingest_dedupe_missing_id_total`, in both modes. `false`: it is then published un-deduped. `true` rejects it instead (`400` for a single insert; a per-record failure in a batch) — a server-side tripwire for producers that must guarantee the id. -- `dedupe.retention` (optional; seed default `"0"`) — how long a committed id stays a duplicate, as a Go duration string: `"24h"`, `"720h"` (30 days), `"90m"`. There is no day unit. `"0"` keeps every id forever, which was the only behavior before this key existed, and a `config.json` without the key means the same. Once an id's retention has ended, the next record carrying it is published as new. With `dedupe.backend: pebble`, a background sweep over the shared Pebble instance deletes the expired id: first about a minute after the instance opens (when the first tenant switches dedupe on), then hourly while any tenant keeps it on, counted by `wavehouse_dedupe_swept_keys_total{reason="expired"}`. With `dynamodb`, no sweep runs: the table's TTL on `ex` deletes the item ([Deployment](/deployment#a-shared-dedupe-table-on-dynamodb)). A finite retention must be at least `"2m"`, the ingest queue's duplicate window: every deduped record is published under an idempotency key derived from its id, so an id re-sent after a shorter retention would be claimed again and then dropped by the queue as a copy, while the client was told it was accepted. A retention below that is refused, not raised to the minimum; so are a negative value and anything that is not a duration, such as `"30d"`, a number with no unit (`"300"` needs one: `"300s"`; `"0"` is the one exception), or a JSON number rather than a string. Hot-reloadable: a change applies to ids committed after the reload, and an id already committed keeps the expiry it was stored with. +- `dedupe.retention` (optional; seed default `"0"`) — how long a committed id stays a duplicate, as a Go duration string: `"24h"`, `"720h"` (30 days), `"90m"`. There is no day unit. `"0"` keeps every id forever, which was the only behavior before this key existed, and a `config.json` without the key means the same. Once an id's retention has ended, the next record carrying it is published as new. With `dedupe.backend: pebble`, a background sweep over the shared Pebble instance deletes the expired id: first about a minute after the instance opens (when the first tenant switches dedupe on), then hourly while any tenant keeps it on, counted by `wavehouse_dedupe_swept_keys_total{reason="expired"}`. With `dynamodb`, no sweep runs: the table's TTL on `ex` deletes the item ([Deployment](/deployment#a-shared-dedupe-table-on-dynamodb)). A finite retention must be at least `"2m"`, the embedded ingest queue's duplicate window (under `mq.backend: nats`, keep it at least the partitions' `duplicate_window` too, which boot warns about but this file cannot check; see [External NATS](/deployment#create-the-topology)): every deduped record is published under an idempotency key derived from its id, so an id re-sent after a shorter retention would be claimed again and then dropped by the queue as a copy, while the client was told it was accepted. A retention below that is refused, not raised to the minimum; so are a negative value and anything that is not a duration, such as `"30d"`, a number with no unit (`"300"` needs one: `"300s"`; `"0"` is the one exception), or a JSON number rather than a string. Hot-reloadable: a change applies to ids committed after the reload, and an id already committed keeps the expiry it was stored with. - `dedupe.tables.

.{id_field, require_id, retention}` — per-table overrides; each entry overrides only the fields it names and inherits the rest, so a table with no `retention` keeps the tenant's (forever when the tenant sets none). A table can keep ids for a shorter time than its tenant, or for longer, or forever (`"retention": "0"`) under a finite tenant retention. ## ClickHouse @@ -224,7 +224,7 @@ A tenant's dead-letter stream is opened when the tenant is first served (an empt ## Message Queue -- `mq.max_bytes_gb` (seed default `50`) — disk budget for the tenant's embedded JetStream ingest stream (`INGEST_{tenant}`), which buffers its ingested events until the worker writes them to ClickHouse; its dead-letter stream (`DLQ_{tenant}`) gets a tenth of it. Each tenant's pair of streams is its own, opened when the tenant is first served and kept, at the budget it last had, when its folder is rejected or removed. The ingest stream runs `DiscardNew`, so when it's full new publishes are rejected and `POST /v1/ingest` returns `503` for that tenant alone — [backpressure by construction](/ingest-pipeline#backpressure-and-durability-knobs). A reload updates both streams' limits in place without touching what's buffered: growing takes effect immediately; shrinking below what's currently on disk makes the ingest stream refuse new publishes until the Active Sweeper purges it back under the limit — what it purges is what is both written to ClickHouse and past the tenant's `stream.gap_window_minutes`, and nothing already accepted is dropped — and a dead-letter stream holding more than a tenth of the new budget is kept at what it holds rather than shrunk, since shrinking it would delete its oldest parked rows; that is logged, and the stream then makes room for each new row by dropping its oldest, as a full one always does. If NATS rejects the update, the rest of the reload is still adopted, the failure is logged, and the next reload retries it. A queue NATS will not open at all refuses boot, like every other store; over [a nested directory](/deployment#the-nested-settings-directory) it costs that tenant alone, at boot or on reload — its ingest answers `503`, each reload trying the queue again, and so does a publish, at most once every five seconds — while every other tenant carries on. A queue that opens while the server runs but that a consumer — the ingest worker's or the stream hub's — cannot join is treated the same way: it costs that tenant alone, its ingest answering `503` until a reload or a publish joins the consumer, so no row is accepted that the worker would not write or the tenant's `GET /v1/stream` connections would not see. The two streams are resized as a pair: a failed DLQ resize undoes the ingest one so both stay on the previous budget, but if that undo fails too the ingest stream keeps the new limit and the DLQ the previous one until a later reload succeeds — the log line says which happened. +- `mq.max_bytes_gb` (seed default `50`) — not applied under [`mq.backend: nats`](/deployment#external-nats), where the partition streams' limits are the operator's; everything below is the embedded queue. The disk budget for the tenant's embedded JetStream ingest stream (`INGEST_{tenant}`), which buffers its ingested events until the worker writes them to ClickHouse; its dead-letter stream (`DLQ_{tenant}`) gets a tenth of it. Each tenant's pair of streams is its own, opened when the tenant is first served and kept, at the budget it last had, when its folder is rejected or removed. The ingest stream runs `DiscardNew`, so when it's full new publishes are rejected and `POST /v1/ingest` returns `503` for that tenant alone — [backpressure by construction](/ingest-pipeline#backpressure-and-durability-knobs). A reload updates both streams' limits in place without touching what's buffered: growing takes effect immediately; shrinking below what's currently on disk makes the ingest stream refuse new publishes until the Active Sweeper purges it back under the limit — what it purges is what is both written to ClickHouse and past the tenant's `stream.gap_window_minutes`, and nothing already accepted is dropped — and a dead-letter stream holding more than a tenth of the new budget is kept at what it holds rather than shrunk, since shrinking it would delete its oldest parked rows; that is logged, and the stream then makes room for each new row by dropping its oldest, as a full one always does. If NATS rejects the update, the rest of the reload is still adopted, the failure is logged, and the next reload retries it. A queue NATS will not open at all refuses boot, like every other store; over [a nested directory](/deployment#the-nested-settings-directory) it costs that tenant alone, at boot or on reload — its ingest answers `503`, each reload trying the queue again, and so does a publish, at most once every five seconds — while every other tenant carries on. A queue that opens while the server runs but that a consumer — the ingest worker's or the stream hub's — cannot join is treated the same way: it costs that tenant alone, its ingest answering `503` until a reload or a publish joins the consumer, so no row is accepted that the worker would not write or the tenant's `GET /v1/stream` connections would not see. The two streams are resized as a pair: a failed DLQ resize undoes the ingest one so both stay on the previous budget, but if that undo fails too the ingest stream keeps the new limit and the DLQ the previous one until a later reload succeeds — the log line says which happened. **Sizing the volume.** Nothing checks the budget against the disk — neither one tenant's nor what the tenants' add up to ([#138](https://github.com/Wave-RF/WaveHouse/issues/138)) — so keep within the free space of the `/nats` volume every tenant's budget plus its dead-letter stream's cap: a tenth of the budget, or what the stream held when a smaller budget arrived, if that is more. Count every tenant ever served on the volume, not only those served now: a rejected or removed tenant's queue is kept and nothing deletes it, so what it holds goes on holding disk — a rejected tenant's replay history, and the rows parked on either one's dead-letter stream, up to that cap. A disk that fills before a budget does fails every tenant's writes, not one: the failed write is logged, the publish goes unanswered until it times out, and ingest answers `500` (`publish failed`) for every tenant on that volume, not the `503` with `Retry-After` of a full budget. It does not clear on its own: a stream that failed a write refuses every later one until WaveHouse restarts, so free the space and then restart. @@ -234,4 +234,4 @@ A tenant's dead-letter stream is opened when the tenant is first served (an empt - `stream.keepalive_interval` (seed default `30`) — seconds a quiet connection may go without a write before the server sends a `:` keepalive comment. It exists to stay under whatever idle timeout sits between WaveHouse and the client; the default clears the common 55–60s proxy windows with margin, and a tighter edge (Azure Application Gateway 20s, CloudFront 30s) wants a lower value — see [Behind a reverse proxy → Idle timeouts](/reverse-proxy#idle-timeouts-by-provider). A reload rebuilds the keepalive wheel in place: live connections stay open and are redistributed across the new ring, each getting at most one full new period before its next keepalive. - `stream.keepalive_buckets` (seed default `3`) — spreads the keepalive writes across the interval (one bucket fires every `keepalive_interval ÷ keepalive_buckets`) so the server nudges ~1/N of connections per tick instead of all at once. It changes only how the writes are spread in time, never the period. -- `stream.gap_window_minutes` (seed default `15`) — minutes of already-written-to-ClickHouse history the Active Sweeper keeps in NATS so a reconnecting client's `Last-Event-ID` replay can bridge the gap; a drop longer than this resumes with a hole. Bounded by the tenant's disk budget, [`mq.max_bytes_gb`](#message-queue). A reload applies from the next sweep (every minute). +- `stream.gap_window_minutes` (seed default `15`) — minutes of already-written-to-ClickHouse history the Active Sweeper keeps in NATS so a reconnecting client's `Last-Event-ID` replay can bridge the gap; a drop longer than this resumes with a hole. Bounded by the tenant's disk budget, [`mq.max_bytes_gb`](#message-queue). A reload applies from the next sweep (every minute). Under [`mq.backend: nats`](/deployment#external-nats) the history stream's `max_age` decides instead, and the API processes warn once per tenant and window about a tenant whose window is longer. diff --git a/go.mod b/go.mod index ee399126b..9ff8605e3 100644 --- a/go.mod +++ b/go.mod @@ -34,8 +34,11 @@ require ( github.com/klauspost/compress v1.19.2 github.com/moby/moby/api v1.55.0 github.com/moby/moby/client v0.5.1 + github.com/nats-io/jwt/v2 v2.8.2 github.com/nats-io/nats-server/v2 v2.14.6 github.com/nats-io/nats.go v1.53.1 + github.com/nats-io/nkeys v0.4.16 + github.com/nats-io/nuid v1.0.1 github.com/prometheus/client_golang v1.24.1 github.com/redis/rueidis v1.0.78 github.com/samber/slog-multi v1.8.0 @@ -177,9 +180,6 @@ require ( github.com/muesli/termenv v0.16.0 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/narqo/go-badge v0.0.0-20230821190521-c9a75c019a59 // indirect - github.com/nats-io/jwt/v2 v2.8.2 // indirect - github.com/nats-io/nkeys v0.4.16 // indirect - github.com/nats-io/nuid v1.0.1 // indirect github.com/nikolaydubina/treemap v1.2.5 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.1 // indirect diff --git a/internal/app/app.go b/internal/app/app.go index fc028cb76..8dfb8eea6 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -199,10 +199,12 @@ func New(ctx context.Context, opts Options) (app *App, err error) { return nil, err } } - if err := a.wireCoord(); err != nil { + if err := a.wireCoord(ctx); err != nil { return nil, err } - if a.cfg.Has(config.RoleSweeper) { + // Under nats, retention is the operator's stream policy, so the sweeper + // has nothing to purge and is not wired. + if a.cfg.Has(config.RoleSweeper) && a.cfg.MQ.Backend == config.MQEmbedded { a.wireSweeper() } if apiRole { diff --git a/internal/app/mq_nats_test.go b/internal/app/mq_nats_test.go new file mode 100644 index 000000000..ba5199937 --- /dev/null +++ b/internal/app/mq_nats_test.go @@ -0,0 +1,127 @@ +package app + +import ( + "context" + "log/slog" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/Wave-RF/WaveHouse/internal/config" + "github.com/Wave-RF/WaveHouse/internal/mq" + "github.com/Wave-RF/WaveHouse/internal/mq/natstest" + "github.com/Wave-RF/WaveHouse/internal/testutil/logtest" +) + +// natsConfig is testConfig on mq.backend: nats against url, connected as the +// shipped wavehouse user, the block otherwise as config.Load leaves it. +func natsConfig(t *testing.T, url string) *config.Config { + t.Helper() + pw := filepath.Join(t.TempDir(), "nats-password") + require.NoError(t, os.WriteFile(pw, []byte(natstest.Password(natstest.WaveHouseUser)+"\n"), 0o600)) + cfg := testConfig(t, writeSettings(t, nil)) + cfg.MQ = config.MQ{Backend: config.MQNATS, NATS: config.MQNATSConfig{ + URLs: []string{url}, User: natstest.WaveHouseUser, PasswordFile: pw, + SubjectPrefix: "wh", Partitions: 4, Shards: 8, IngestConsumer: "wh-ingest", + ConnectTimeout: 5 * time.Second, PublishTimeout: 5 * time.Second, TopologyWait: 10 * time.Second, + }} + return cfg +} + +// mq.backend: nats wires the external broker in place of the embedded one, +// and a role split that the embedded MQ cannot serve boots on it. The worker +// consumes the operator's durable; the operator deleting it ends the worker, +// and with it Run, naming the component. +func TestNew_NATSBackend(t *testing.T) { + srv := natstest.Start(t) + cfg := natsConfig(t, srv.URL()) + cfg.Roles = []config.Role{config.RoleAPI, config.RoleIngest} + a := newApp(t, cfg, Options{}) + + _, ok := a.MQ().(*mq.ExternalNATS) + require.True(t, ok, "mq.backend: nats wires mq.ExternalNATS, got %T", a.MQ()) + assert.Equal(t, []string{ + "clickhouse", "schema discovery", "dedupe", "mq", "cache", "coord", + "hub bridge", "keepalive", "ingest worker", + "auth", "sighup", "settings watcher", "http server", + }, componentNames(a)) + _, err := os.Stat(filepath.Join(cfg.DataDir, "nats")) + assert.True(t, os.IsNotExist(err), "nothing is kept under data_dir/nats") + + ctx, cancel := context.WithTimeout(t.Context(), 20*time.Second) + defer cancel() + done := make(chan error, 1) + go func() { done <- a.Run(ctx) }() + // Once the worker has bound it, the durable has a pull waiting. + require.Eventually(t, func() bool { + c, err := srv.Operator.JetStream().Consumer(ctx, "WH_INGEST_0", "wh-ingest-0") + return err == nil && c.CachedInfo().NumWaiting > 0 + }, 10*time.Second, 20*time.Millisecond, "the ingest worker pulls from the operator's durable") + require.NoError(t, srv.Operator.DeleteIngestDurables(ctx)) + err = <-done + require.ErrorIs(t, err, mq.ErrDeliveryEnded) + assert.True(t, strings.HasPrefix(err.Error(), "ingest worker: "), "the failing component names itself: %v", err) +} + +// Every role under mq.backend: nats wires no sweeper (retention is the +// operator's stream policy there), and a tenant's gap window is checked +// against what the history keeps instead, at boot and after a reload. +func TestNew_NATSWiresNoSweeper(t *testing.T) { //nolint:paralleltest // captures the default logger + srv := natstest.Start(t) + cfg := natsConfig(t, srv.URL()) + cfg.Roles = config.AllRoles() + cfg.Coord.Backend = config.CoordNATS + day := map[string]any{"keepalive_interval": 30, "keepalive_buckets": 3, "gap_window_minutes": 24 * 60} + rewriteSettings(t, cfg.Settings.Dir, map[string]any{"stream": day}) + a := newApp(t, cfg, Options{}) + assert.NotContains(t, componentNames(a), "sweeper") + + // New's observability wiring replaced the default logger; capture after. + // The boot check already warned for this window, so reloading it again + // warns nobody: only a changed window does. + logs := logtest.Capture(t, slog.LevelWarn) + warned := func() int { return strings.Count(logs.String(), "keeps less than this tenant's gap window") } + _, adopted := a.tenants.Reload("test") + require.True(t, adopted) + assert.Zero(t, warned(), "boot checked a day's window against the shipped history already") + + day["gap_window_minutes"] = 2 * 24 * 60 + rewriteSettings(t, cfg.Settings.Dir, map[string]any{"stream": day}) + _, adopted = a.tenants.Reload("test") + require.True(t, adopted) + assert.Equal(t, 1, warned(), "a reload checks the new window") +} + +// A cluster never reached within topology_wait refuses boot as unavailable. +func TestNew_NATSUnreachable(t *testing.T) { + guardGlobals(t) + cfg := natsConfig(t, "nats://"+closedAddr(t)) + cfg.MQ.NATS.TopologyWait = time.Millisecond + _, err := New(t.Context(), Options{Config: cfg}) + require.ErrorIs(t, err, mq.ErrUnavailable) + assert.ErrorContains(t, err, "mq open") +} + +// The operator's topology missing a piece refuses boot with the finding. +func TestNew_NATSTopologyMissing(t *testing.T) { + srv := natstest.Start(t) + require.NoError(t, srv.Operator.JetStream().DeleteStream(t.Context(), "WH_DLQ")) + guardGlobals(t) + cfg := natsConfig(t, srv.URL()) + cfg.MQ.NATS.TopologyWait = time.Millisecond + _, err := New(t.Context(), Options{Config: cfg}) + require.ErrorIs(t, err, mq.ErrTopology) + assert.ErrorContains(t, err, "dead-letter stream") +} + +// config bounds mq.nats.shards by internal/mq's own bound, which it cannot +// import. +func TestMQNATSShards_BoundMatchesMQ(t *testing.T) { + t.Parallel() + assert.Equal(t, mq.MaxNATSShards, config.MaxNATSShards) +} diff --git a/internal/app/retention_warn_test.go b/internal/app/retention_warn_test.go new file mode 100644 index 000000000..5a7913b9a --- /dev/null +++ b/internal/app/retention_warn_test.go @@ -0,0 +1,44 @@ +package app + +import ( + "bytes" + "log/slog" + "testing" + "time" + + "github.com/Wave-RF/WaveHouse/internal/settings" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// Under mq.backend=nats the operator's duplicate_window can exceed the 2m +// floor settings enforces, so boot and every reload warn about each served +// tenant with dedupe on whose finite retention, default or per table, is +// under it. Forever ("0"), a longer retention and a tenant with dedupe off +// are quiet. +func TestWarnShortRetention(t *testing.T) { + guardGlobals(t) + dedupe := func(enabled bool, retention string, tables map[string]any) map[string]any { + return map[string]any{"dedupe": map[string]any{ + "enabled": enabled, "id_field": "event_id", "require_id": false, "retention": retention, "tables": tables, + }} + } + root := writeNestedSettings(t, map[string]map[string]any{ + "acme": dedupe(true, "5m", map[string]any{"clicks": map[string]any{"retention": "10m"}, "views": map[string]any{"retention": "3m"}}), + "globex": dedupe(true, "0", nil), + "initech": dedupe(false, "3m", nil), + }) + tenants, findings := settings.Open(root) + require.NotNil(t, tenants, "findings: %v", findings) + + var buf bytes.Buffer + slog.SetDefault(slog.New(slog.NewTextHandler(&buf, nil))) + (&App{tenants: tenants}).warnShortRetention(8 * time.Minute) + + out := buf.String() + assert.Equal(t, 2, bytes.Count(buf.Bytes(), []byte("dedupe retention is shorter")), out) + assert.Contains(t, out, `tenant=acme table="" retention=5m0s duplicate_window=8m0s`) + assert.Contains(t, out, `tenant=acme table=views retention=3m0s`) + assert.NotContains(t, out, "globex") + assert.NotContains(t, out, "initech") +} diff --git a/internal/app/wire.go b/internal/app/wire.go index a18ab5b45..4286341aa 100644 --- a/internal/app/wire.go +++ b/internal/app/wire.go @@ -547,11 +547,29 @@ func (a *App) wireMQ(ctx context.Context) error { switch b := a.cfg.MQ.Backend; b { case config.MQEmbedded: return a.wireEmbeddedMQ(ctx) + case config.MQNATS: + return a.wireNATSMQ(ctx) default: return unreachableBackend("mq.backend", b) } } +// adoptMQ makes broker the process's MQ, closed with it. +func (a *App) adoptMQ(broker mq.Broker) { + a.mq = broker + a.add(component{name: "mq", close: withoutContext(broker.Close)}) + + // Only register system metric gauges when a real MeterProvider is in + // place — otherwise `otel.GetMeterProvider()` returns the no-op SDK + // provider and RegisterCallback silently no-ops, making this look + // authoritative when it's actually doing nothing. + if a.cfg.OTel.Enabled || a.cfg.Prometheus.Enabled { + if err := observability.RegisterSystemMetrics(broker.Stats, a.dedupeStats); err != nil { + slog.Error("failed to register system metrics", "error", err) + } + } +} + // wireEmbeddedMQ starts the embedded NATS under data_dir/nats and hands it // each served tenant's mq.max_bytes_gb, which opens that tenant's queue the // first time. The budget is hot-reloadable: after every @@ -576,18 +594,7 @@ func (a *App) wireEmbeddedMQ(ctx context.Context) error { config.LogStorageInitError("mq", dir, err) return fmt.Errorf("mq open: %w", err) } - a.mq = broker - a.add(component{name: "mq", close: withoutContext(broker.Close)}) - - // Only register system metric gauges when a real MeterProvider is in - // place — otherwise `otel.GetMeterProvider()` returns the no-op SDK - // provider and RegisterCallback silently no-ops, making this look - // authoritative when it's actually doing nothing. - if a.cfg.OTel.Enabled || a.cfg.Prometheus.Enabled { - if err := observability.RegisterSystemMetrics(broker.Stats, a.dedupeStats); err != nil { - slog.Error("failed to register system metrics", "error", err) - } - } + a.adoptMQ(broker) // The hook's apply is rooted in the App's stop context, so a reload // caught mid-hook by SIGTERM gives up rather than holding the drain past @@ -697,14 +704,20 @@ func unreachableBackend[T ~string](key string, got T) error { return fmt.Errorf("%s %q has no wiring: a Config built without config.Load must name the backend of every layer it wires", key, got) } -// wireCoord opens the lease coordinator the singleton loops campaign on. -func (a *App) wireCoord() error { +// wireCoord opens the lease coordinator the singleton loops campaign on: +// in-process, or the operator's KV bucket on the external broker's +// connection (config refuses coord.backend=nats without mq.backend=nats). It +// is added after the MQ, so it closes first and its terms are resigned while +// the connection is still up. +func (a *App) wireCoord(ctx context.Context) error { switch b := a.cfg.Coord.Backend; b { case config.CoordLocal: c := coord.NewLocal() a.coord = c a.add(component{name: "coord", close: c.Close}) return nil + case config.CoordNATS: + return a.wireNATSCoord(ctx) default: return unreachableBackend("coord.backend", b) } @@ -783,10 +796,21 @@ func (a *App) wireStreaming() { // wireIngestWorker adds the batch consumer (JetStream → ClickHouse). Its // consumer is created when Run starts it; at shutdown the in-flight batches -// drain within the shutdown timeout. +// drain within the shutdown timeout. Over a sharded queue (mq.backend: nats) +// it consumes only the shards this process is assigned (ingest.ClaimShards), +// so each table has one writer. func (a *App) wireIngestWorker() { a.add(component{name: "ingest worker", run: func(ctx context.Context) error { - stop, failed, err := ingest.StartIngestWorker(ctx, a.mq, sharedTables{Cache: a.cache, sharing: a.pools.SharingTables}, a.pools.Target, dlqFor(a.tenants)) + var queue ingest.Queue = a.mq + if _, ok := a.mq.(mq.Sharded); ok { + // One process per shard at a time, balanced over the ingest + // processes through the shared coordinator. + var err error + if queue, err = ingest.ClaimShards(a.mq, a.coord, ingest.ClaimConfig{}); err != nil { + return err + } + } + stop, failed, err := ingest.StartIngestWorker(ctx, queue, sharedTables{Cache: a.cache, sharing: a.pools.SharingTables}, a.pools.Target, dlqFor(a.tenants)) if err != nil { return err } @@ -896,22 +920,6 @@ func (a *App) wireAuth() func(http.Handler) http.Handler { return authn.Middleware() } -// wireOpsAuth is the authentication of a process without the api role: the -// operator key and nothing else. Token verifiers — and the JWKS fetches that -// keep them — are per API process, so no token validates here and the reload -// route admits the operator alone (api.NewOpsRouter). -func (a *App) wireOpsAuth() func(http.Handler) http.Handler { - operatorKey := strings.TrimSpace(a.cfg.Auth.OperatorKey) - switch { - case operatorKey == "" && a.tenants.Nested(): - slog.Warn("no auth.operator_key set: a process without the api role takes only the operator key on POST /v1/ops/settings/reload, and a nested settings directory has no watcher, so its settings can only be reloaded by SIGHUP") - case operatorKey == "": - slog.Warn("no auth.operator_key set: a process without the api role takes only the operator key on POST /v1/ops/settings/reload, so its settings can only be reloaded by SIGHUP or the directory watcher") - } - authn := auth.NewAuthenticator(auth.Config{OperatorKey: operatorKey}, nil, nil) - return authn.Middleware() -} - // wireReloadTriggers adds SIGHUP and the directory watcher. All three // triggers (these two and POST /v1/ops/settings/reload) funnel into the same // serialized Registry.Reload, and a rejected reload keeps the previous good @@ -1027,26 +1035,6 @@ func (a *App) wireHTTP(authMW func(http.Handler) http.Handler) { a.wireServers(func() { close(closing) }) } -// wireOpsHTTP serves the ops-only router of a process without the api role: -// the probes, /version, the metrics endpoint, and the settings reload. -// Readiness pings the ClickHouse pools when the process has them (the ingest -// role); a sweeper-only process is ready once booted. -func (a *App) wireOpsHTTP(authMW func(http.Handler) http.Handler) { - health := api.NewHealthHandler(nil) - if a.pools != nil { - health.Ping = a.pools.Ping - } - deps := api.OpsDependencies{ - Health: health, - Version: api.NewVersionHandler(a.build.Version, a.build.GitCommit, a.build.BuildTime), - Settings: api.NewSettingsHandler(a.tenants), - AuthMW: authMW, - } - deps.MetricsHandler, deps.MetricsPath = a.inlineMetrics() - a.handler = api.NewOpsRouter(deps) - a.wireServers(nil) -} - // inlineMetrics is the metrics endpoint to mount on the main router: with // prometheus.port 0 only, since a non-zero port gets its own listener. func (a *App) inlineMetrics() (http.Handler, string) { diff --git a/internal/app/wire_nats.go b/internal/app/wire_nats.go new file mode 100644 index 000000000..16ad60f94 --- /dev/null +++ b/internal/app/wire_nats.go @@ -0,0 +1,139 @@ +// The wiring for mq.backend=nats and coord.backend=nats, apart from +// wire.go so the e2e coverage gate, whose stack runs the embedded broker, +// can leave it to the integration suite (.testcoverage.yml). + +package app + +import ( + "context" + "fmt" + "log/slog" + "time" + + "github.com/Wave-RF/WaveHouse/internal/config" + "github.com/Wave-RF/WaveHouse/internal/dedupe" + "github.com/Wave-RF/WaveHouse/internal/mq" + "github.com/Wave-RF/WaveHouse/internal/settings" + "github.com/Wave-RF/WaveHouse/internal/tenant" +) + +// wireNATSMQ connects to the operator's NATS (mq.backend: nats) and waits, +// up to mq.nats.topology_wait, for the streams and durables it needs; a +// topology still wrong then refuses boot with every finding. The operator +// owns every limit, so a tenant's mq.max_bytes_gb is not handed over +// (config.Warnings says so at boot). +func (a *App) wireNATSMQ(ctx context.Context) error { + n := a.cfg.MQ.NATS + broker, err := mq.NewNATS(ctx, mq.NATSConfig{ + URLs: n.URLs, + Name: n.Name, + CredsFile: n.CredsFile, + NKeySeedFile: n.NKeySeedFile, + User: n.User, + PasswordFile: n.PasswordFile, + TLS: mq.NATSTLS{ + CAFile: n.TLS.CAFile, CertFile: n.TLS.CertFile, KeyFile: n.TLS.KeyFile, + ServerName: n.TLS.ServerName, HandshakeFirst: n.TLS.HandshakeFirst, + }, + JSDomain: n.JSDomain, + // AckWait, MaxAckPending and Prefetch are left to mq's defaults, + // which are the ingest worker's own. + Topology: mq.NATSTopology{ + Prefix: n.SubjectPrefix, + Partitions: n.Partitions, + Shards: n.Shards, + IngestConsumer: n.IngestConsumer, + HistoryStream: n.HistoryStream, + PublishTimeout: n.PublishTimeout, + DedupeLease: a.dedupeLease(), + // Boot waits for the lease bucket with the rest of the topology. + CoordBucket: a.coordBucket(), + }, + ConnectTimeout: n.ConnectTimeout, + TopologyWait: n.TopologyWait, + }) + if err != nil { + return fmt.Errorf("mq open: %w", err) + } + a.adoptMQ(broker) + if !a.cfg.Has(config.RoleAPI) { + return nil // replay and dedupe run on the API path only + } + // No sweeper runs here, so what it warned about (a gap window longer than + // the history keeps) is checked by the processes that replay. + checkWindows := func() { broker.CheckReplayWindows(servedGapWindows(a.tenants)) } + checkWindows() + a.tenants.AfterAdopt(func([]tenant.ID) { checkWindows() }) + window, err := broker.DuplicateWindow(ctx) + if err != nil { + slog.Warn("mq: could not read the partitions' duplicate window; dedupe retention is not checked against it", "error", err) + return nil + } + a.warnShortRetention(window) + a.tenants.AfterAdopt(func([]tenant.ID) { a.warnShortRetention(window) }) + return nil +} + +// servedGapWindows is each served tenant's stream.gap_window_minutes. +func servedGapWindows(tenants *settings.Registry) map[tenant.ID]time.Duration { + windows := map[tenant.ID]time.Duration{} + for id, store := range tenants.All() { + windows[id] = store.GapWindow() + } + return windows +} + +// dedupeLease is the lease ingest runs with: dedupe.lease, or the default for 0. +func (a *App) dedupeLease() time.Duration { + if l := a.cfg.Dedupe.Lease; l > 0 { + return l + } + return dedupe.DefaultLease +} + +// warnShortRetention logs each served tenant with dedupe on whose finite +// retention, default or per table, is under the operator's duplicate window. +// settings refuses one under the embedded window; a longer operator window +// can't be seen there. Such an id re-sent after it expires but inside the +// window is claimed again, then dropped by the queue while the client hears +// it was accepted. +func (a *App) warnShortRetention(window time.Duration) { + for id, store := range a.tenants.All() { + if !store.DedupeEnabled() { + continue + } + for table, r := range store.DedupeRetentions() { + if r > 0 && r < window { + slog.Warn("dedupe retention is shorter than the nats partitions' duplicate_window: an id re-sent between the two is dropped by the queue while the client is told it was accepted; use a retention of at least the window, or \"0\"", + "tenant", id, "table", table, "retention", r, "duplicate_window", window) + } + } + } +} + +// coordBucket is the lease bucket under coord.backend=nats, "" otherwise. +func (a *App) coordBucket() string { + if a.cfg.Coord.Backend != config.CoordNATS { + return "" + } + if b := a.cfg.Coord.NATS.Bucket; b != "" { + return b + } + return mq.DefaultNATSCoordBucket(a.cfg.MQ.NATS.SubjectPrefix) +} + +// wireNATSCoord holds the leases in the operator's KV bucket, on the MQ's own +// connection (coord.backend: nats). +func (a *App) wireNATSCoord(ctx context.Context) error { + broker, ok := a.mq.(*mq.ExternalNATS) + if !ok { + return fmt.Errorf("coord.backend=nats needs mq.backend=nats, got %T", a.mq) + } + c, err := broker.Leases(ctx, a.coordBucket(), a.cfg.InstanceID) + if err != nil { + return fmt.Errorf("coord open: %w", err) + } + a.coord = c + a.add(component{name: "coord", close: c.Close}) + return nil +} diff --git a/internal/app/wire_ops.go b/internal/app/wire_ops.go new file mode 100644 index 000000000..2f3c460c9 --- /dev/null +++ b/internal/app/wire_ops.go @@ -0,0 +1,49 @@ +// The ops-only listener of a process without the api role, apart from +// wire.go for the same reason as wire_nats.go: the e2e stack runs every role. + +package app + +import ( + "log/slog" + "net/http" + "strings" + + "github.com/Wave-RF/WaveHouse/internal/api" + "github.com/Wave-RF/WaveHouse/internal/auth" +) + +// wireOpsAuth is the authentication of a process without the api role: the +// operator key and nothing else. Token verifiers — and the JWKS fetches that +// keep them — are per API process, so no token validates here and the reload +// route admits the operator alone (api.NewOpsRouter). +func (a *App) wireOpsAuth() func(http.Handler) http.Handler { + operatorKey := strings.TrimSpace(a.cfg.Auth.OperatorKey) + switch { + case operatorKey == "" && a.tenants.Nested(): + slog.Warn("no auth.operator_key set: a process without the api role takes only the operator key on POST /v1/ops/settings/reload, and a nested settings directory has no watcher, so its settings can only be reloaded by SIGHUP") + case operatorKey == "": + slog.Warn("no auth.operator_key set: a process without the api role takes only the operator key on POST /v1/ops/settings/reload, so its settings can only be reloaded by SIGHUP or the directory watcher") + } + authn := auth.NewAuthenticator(auth.Config{OperatorKey: operatorKey}, nil, nil) + return authn.Middleware() +} + +// wireOpsHTTP serves the ops-only router of a process without the api role: +// the probes, /version, the metrics endpoint, and the settings reload. +// Readiness pings the ClickHouse pools when the process has them (the ingest +// role); a process without them is ready once booted. +func (a *App) wireOpsHTTP(authMW func(http.Handler) http.Handler) { + health := api.NewHealthHandler(nil) + if a.pools != nil { + health.Ping = a.pools.Ping + } + deps := api.OpsDependencies{ + Health: health, + Version: api.NewVersionHandler(a.build.Version, a.build.GitCommit, a.build.BuildTime), + Settings: api.NewSettingsHandler(a.tenants), + AuthMW: authMW, + } + deps.MetricsHandler, deps.MetricsPath = a.inlineMetrics() + a.handler = api.NewOpsRouter(deps) + a.wireServers(nil) +} diff --git a/internal/config/backends.go b/internal/config/backends.go index 2c955e194..06c857fad 100644 --- a/internal/config/backends.go +++ b/internal/config/backends.go @@ -22,16 +22,29 @@ type MQBackend string // /nats. const MQEmbedded MQBackend = "embedded" -var mqBackends = []MQBackend{MQEmbedded} +// MQNATS is a NATS JetStream cluster the operator runs, holding the streams +// and durables deployments/nats describes; every process naming it shares +// one queue. Its settings are the mq.nats block. +const MQNATS MQBackend = "nats" + +var mqBackends = []MQBackend{MQEmbedded, MQNATS} // MQ selects the message queue. The per-tenant byte budget, mq.max_bytes_gb, // is a settings-directory key, not this block's. type MQ struct { Backend MQBackend `yaml:"backend" env:"WH_MQ_BACKEND"` + // NATS is read only when Backend is nats. + NATS MQNATSConfig `yaml:"nats"` } func (m MQ) validate() error { - return checkBackend("mq.backend", "WH_MQ_BACKEND", m.Backend, mqBackends) + if err := checkBackend("mq.backend", "WH_MQ_BACKEND", m.Backend, mqBackends); err != nil { + return err + } + if m.Backend == MQNATS { + return m.NATS.validate() + } + return nil } // CacheBackend names the query-result cache implementation. @@ -147,22 +160,36 @@ func (d DedupeDynamoDBConfig) validate() error { return nil } -// CoordBackend names where leases for singleton work (the sweeper) are held. +// CoordBackend names where leases are held: the sweeper's on the embedded +// MQ, and the ingest processes' on a shared one. type CoordBackend string // CoordLocal holds leases in this process, which is enough while no other // process shares its queue. const CoordLocal CoordBackend = "local" -var coordBackends = []CoordBackend{CoordLocal} +// CoordNATS holds leases in a KV bucket on mq.nats's connection, so every +// process on the shared queue contends for the same ones. It needs +// mq.backend=nats; its settings are the coord.nats block. +const CoordNATS CoordBackend = "nats" + +var coordBackends = []CoordBackend{CoordLocal, CoordNATS} // Coord selects the coordination layer. type Coord struct { Backend CoordBackend `yaml:"backend" env:"WH_COORD_BACKEND"` + // NATS is read only when Backend is nats. + NATS CoordNATSConfig `yaml:"nats"` } func (c Coord) validate() error { - return checkBackend("coord.backend", "WH_COORD_BACKEND", c.Backend, coordBackends) + if err := checkBackend("coord.backend", "WH_COORD_BACKEND", c.Backend, coordBackends); err != nil { + return err + } + if c.Backend == CoordNATS { + return c.NATS.validate() + } + return nil } // checkBackend refuses a backend this build has no implementation for, @@ -234,17 +261,17 @@ func (c *Config) NeedsDataDir() bool { } // Warnings returns what a valid configuration is still likely to get wrong, -// one line each, for boot to log at WARN. The shared-queue ones are not -// errors because each is correct for a single replica, and one process -// cannot count its replicas. +// one line each, for boot to log at WARN. None is an error: an ignored block +// is harmless, and the shared-queue ones are correct for a single replica, +// which one process cannot tell from many. func (c *Config) Warnings() []string { - // All are the api role's: a process without it opens no cache it reads - // and no dedupe store, and a split's Deployments differ only in roles, so - // the API's warnings cover the others'. + out := c.natsWarnings() + // The rest are the api role's: a process without it opens no cache it + // reads and no dedupe store, and a split's Deployments differ only in + // roles, so the API's warnings cover the others'. if !c.Has(RoleAPI) { - return nil + return out } - var out []string if c.Cache.Backend == CacheRedis && c.Cache.Redis.TLS.InsecureSkipVerify { out = append(out, "cache.redis.tls.insecure_skip_verify is on: the cache accepts any certificate, so whoever can intercept the connection can read and replace cached query results") } diff --git a/internal/config/backends_test.go b/internal/config/backends_test.go index 6b6081644..3d8a9f301 100644 --- a/internal/config/backends_test.go +++ b/internal/config/backends_test.go @@ -52,10 +52,10 @@ func TestLoad_BackendsFromEnv(t *testing.T) { } func TestLoad_BackendFromEnvRefusesAnUnknownValue(t *testing.T) { - t.Setenv("WH_MQ_BACKEND", "nats") + t.Setenv("WH_MQ_BACKEND", "kafka") _, err := Load("nonexistent.yaml") require.Error(t, err) - assert.Contains(t, err.Error(), `mq.backend (WH_MQ_BACKEND) "nats" is not a backend this build has; valid: embedded`) + assert.Contains(t, err.Error(), `mq.backend (WH_MQ_BACKEND) "kafka" is not a backend this build has; valid: embedded, nats`) } func TestLoad_BackendsFromYAML(t *testing.T) { @@ -90,14 +90,14 @@ func TestLoad_BackendBlocksRefuseUnknownKeys(t *testing.T) { mq: backend: embedded max_bytes_gb: 5 - nats: - urls: nats://localhost:4222 + redis: + addr: localhost:6379 dedupe: enabled: true `), 0o600)) _, err := Load(path) require.Error(t, err) - assert.Contains(t, err.Error(), "dedupe.enabled, mq.max_bytes_gb, mq.nats") + assert.Contains(t, err.Error(), "dedupe.enabled, mq.max_bytes_gb, mq.redis") assert.Contains(t, err.Error(), EnvSettingsDir) } @@ -116,10 +116,10 @@ func TestValidate_UnknownBackend(t *testing.T) { set func(*Config) want string }{ - {"mq", func(c *Config) { c.MQ.Backend = "kafka" }, `mq.backend (WH_MQ_BACKEND) "kafka" is not a backend this build has; valid: embedded`}, + {"mq", func(c *Config) { c.MQ.Backend = "kafka" }, `mq.backend (WH_MQ_BACKEND) "kafka" is not a backend this build has; valid: embedded, nats`}, {"cache", func(c *Config) { c.Cache.Backend = "memcached" }, `cache.backend (WH_CACHE_BACKEND) "memcached" is not a backend this build has; valid: local, redis`}, {"dedupe", func(c *Config) { c.Dedupe.Backend = "redis" }, `dedupe.backend (WH_DEDUPE_BACKEND) "redis" is not a backend this build has; valid: pebble, dynamodb`}, - {"coord", func(c *Config) { c.Coord.Backend = "nats" }, `coord.backend (WH_COORD_BACKEND) "nats" is not a backend this build has; valid: local`}, + {"coord", func(c *Config) { c.Coord.Backend = "kubernetes" }, `coord.backend (WH_COORD_BACKEND) "kubernetes" is not a backend this build has; valid: local, nats`}, // The zero value, which a Config built without Load carries. {"empty", func(c *Config) { c.MQ.Backend = "" }, `mq.backend (WH_MQ_BACKEND) "" is not a backend`}, } @@ -136,7 +136,7 @@ func TestValidate_UnknownBackend(t *testing.T) { } } -// Every warning keys on a shared queue, which no backend offers yet, so the +// These warnings key on any shared queue, not on nats alone, so a stand-in // value is set directly: Warnings reads the choice, it doesn't validate it. func TestWarnings_SharedQueue(t *testing.T) { t.Parallel() diff --git a/internal/config/config.go b/internal/config/config.go index c6abd263d..99016ac9a 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -25,8 +25,8 @@ type Config struct { // a Deployment per role differs only in this. See Role. Roles []Role `yaml:"roles" env:"WH_ROLES"` // InstanceID names this process: logged at boot, and the holder a - // distributed coordinator will record. Empty resolves to -<8 hex> - // at Load. + // coord.backend=nats lease names. Empty resolves to -<8 hex> at + // Load. InstanceID string `yaml:"instance_id" env:"WH_INSTANCE_ID"` Server Server `yaml:"server"` ClickHouse ClickHouse `yaml:"clickhouse"` @@ -175,10 +175,11 @@ const ( // its bridge off the queue and its keepalive wheel. Per process: every // API process runs its own. RoleAPI Role = "api" - // RoleIngest runs the ingest worker, queue to ClickHouse. Every ingest - // process consumes the one shared durable, competing for messages. + // RoleIngest runs the ingest worker, queue to ClickHouse. Under nats each + // shard of the queue is delivered to one ingest process at a time. RoleIngest Role = "ingest" - // RoleSweeper runs the sweeper, one per queue, under the sweeper lease. + // RoleSweeper runs the sweeper, one per queue, under the sweeper lease, on + // the embedded MQ only: under nats the streams' own retention replaces it. RoleSweeper Role = "sweeper" ) @@ -217,6 +218,9 @@ func (c *Config) validateTopology() error { if c.MQ.Backend == MQEmbedded && len(c.Roles) != len(allRoles) { return fmt.Errorf("roles %s with mq.backend=embedded: the embedded MQ lives inside this process, and a process without it cannot reach its queue — run every role (%s), or set a shared mq.backend", joinRoles(c.Roles), joinRoles(allRoles)) } + if err := c.validateNATSTopology(); err != nil { + return err + } if c.splitsCache() && c.Cache.Backend == CacheLocal { return fmt.Errorf("roles %s with cache.backend=local: api and ingest run in different processes, and the ingest worker's cache invalidation would never reach the API's cache — run api and ingest together, or set cache.backend=redis, one cache every process shares", joinRoles(c.Roles)) } @@ -255,7 +259,7 @@ func defaults() Config { DataDir: "./data", Roles: AllRoles(), Server: Server{Port: 8080, ShutdownTimeout: 10}, - MQ: MQ{Backend: MQEmbedded}, + MQ: MQ{Backend: MQEmbedded, NATS: defaultMQNATS()}, Cache: Cache{ Backend: CacheLocal, L1MaxCost: 64 << 20, Redis: CacheRedisConfig{ @@ -381,6 +385,7 @@ func Load(path string) (*Config, error) { for i, r := range cfg.Roles { cfg.Roles[i] = Role(strings.TrimSpace(string(r))) } + cfg.MQ.NATS.trimURLs() if cfg.InstanceID = strings.TrimSpace(cfg.InstanceID); cfg.InstanceID == "" { cfg.InstanceID = defaultInstanceID() } diff --git a/internal/config/coord_nats_test.go b/internal/config/coord_nats_test.go new file mode 100644 index 000000000..17527de8b --- /dev/null +++ b/internal/config/coord_nats_test.go @@ -0,0 +1,112 @@ +package config + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestLoad_CoordNATS(t *testing.T) { + t.Setenv("WH_MQ_BACKEND", "nats") + t.Setenv("WH_MQ_NATS_URLS", "nats://nats:4222") + t.Setenv("WH_COORD_BACKEND", "nats") + t.Setenv("WH_COORD_NATS_BUCKET", "leases") + cfg, err := Load("nonexistent.yaml") + require.NoError(t, err) + assert.Equal(t, Coord{Backend: CoordNATS, NATS: CoordNATSConfig{Bucket: "leases"}}, cfg.Coord) +} + +func TestLoad_CoordNATSFromYAML(t *testing.T) { + t.Parallel() + path := filepath.Join(t.TempDir(), "config.yaml") + require.NoError(t, os.WriteFile(path, []byte(` +mq: + backend: nats + nats: + urls: ["nats://a:4222"] +coord: + backend: nats + nats: + bucket: prod_leases +`), 0o600)) + cfg, err := Load(path) + require.NoError(t, err) + assert.Equal(t, CoordNATSConfig{Bucket: "prod_leases"}, cfg.Coord.NATS) + assert.Empty(t, CoordNATSConfig{}.Bucket, "empty is the prefix's bucket, named by internal/mq") +} + +func TestUnboundEnv_KnowsTheCoordNATSVariables(t *testing.T) { + t.Parallel() + assert.Empty(t, unboundEnv([]string{"WH_COORD_NATS_BUCKET=x"})) +} + +// Rules 3 and 4 (#613): NATS leases need the NATS connection, and a process +// ingesting a shared queue needs shared leases for its shards; one without +// ingest holds no lease there, so it may keep coord.backend=local. The +// sweeper is not wired on nats, so a process that would run only it is +// refused. +func TestValidate_CoordAgainstMQ(t *testing.T) { + t.Parallel() + for _, tc := range []struct { + name string + mq MQBackend + coord CoordBackend + roles []Role + want string + }{ + {"embedded, local", MQEmbedded, CoordLocal, AllRoles(), ""}, + {"nats, nats", MQNATS, CoordNATS, AllRoles(), ""}, + {"rule 3: nats leases on embedded", MQEmbedded, CoordNATS, AllRoles(), "coord.backend=nats with mq.backend=embedded"}, + {"rule 4: every role on a local lease", MQNATS, CoordLocal, AllRoles(), "coord.backend=local with mq.backend=nats in a process running ingest"}, + {"rule 4: an ingest-only process", MQNATS, CoordLocal, []Role{RoleIngest}, "set coord.backend=nats"}, + {"rule 4 spares a process without ingest", MQNATS, CoordLocal, []Role{RoleAPI}, "-"}, + {"a sweeper-only process runs nothing on nats", MQNATS, CoordNATS, []Role{RoleSweeper}, "would run nothing"}, + {"a sweeper-only process runs nothing, whatever the coordinator", MQNATS, CoordLocal, []Role{RoleSweeper}, "would run nothing"}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + cfg := natsBackends() + cfg.MQ.Backend, cfg.Coord.Backend, cfg.Roles = tc.mq, tc.coord, tc.roles + err := cfg.Validate() + if tc.want == "" { + require.NoError(t, err) + return + } + if tc.want == "-" { // passes rules 3 and 4; rule 5 may still refuse + if err != nil { + assert.NotContains(t, err.Error(), "coord.backend") + } + return + } + require.Error(t, err) + assert.Contains(t, err.Error(), tc.want) + }) + } +} + +func TestValidate_CoordNATSBucket(t *testing.T) { + t.Parallel() + for bucket, ok := range map[string]bool{"": true, "wh_coord": true, "Prod-Leases_2": true, "wh.coord": false, "wh coord": false, "a>": false} { + cfg := natsBackends() + cfg.Coord.NATS.Bucket = bucket + err := cfg.Validate() + if ok { + assert.NoError(t, err, "bucket %q", bucket) + continue + } + require.Error(t, err, "bucket %q", bucket) + assert.Contains(t, err.Error(), "coord.nats.bucket (WH_COORD_NATS_BUCKET)") + } +} + +// The block is read only under coord.backend=nats; otherwise boot says so. +func TestWarnings_CoordNATSIgnored(t *testing.T) { + t.Parallel() + cfg := defaultBackends() + cfg.Coord.NATS.Bucket = "wh.bad" + require.NoError(t, cfg.Validate()) + assert.Equal(t, []string{"coord.nats is set but coord.backend=local: the block is ignored"}, cfg.Warnings()) +} diff --git a/internal/config/defaults_test.go b/internal/config/defaults_test.go index ddcba538e..28268b6a8 100644 --- a/internal/config/defaults_test.go +++ b/internal/config/defaults_test.go @@ -47,6 +47,14 @@ var zeroCases = []zeroCase{ {"cache.redis.max_value_bytes", "WH_CACHE_REDIS_MAX_VALUE_BYTES", 0, 1 << 20, "2048", 2048, func(c *Config) any { return c.Cache.Redis.MaxValueBytes }}, {"cache.redis.compress_min_bytes", "WH_CACHE_REDIS_COMPRESS_MIN_BYTES", 0, 1 << 10, "2048", 2048, func(c *Config) any { return c.Cache.Redis.CompressMinBytes }}, {"cache.redis.version_ttl", "WH_CACHE_REDIS_VERSION_TTL", time.Duration(0), 168 * time.Hour, "1h", time.Hour, func(c *Config) any { return c.Cache.Redis.VersionTTL }}, + // So is the mq.nats block, validated only under backend=nats. + {"mq.nats.subject_prefix", "WH_MQ_NATS_SUBJECT_PREFIX", "", "wh", "acme", "acme", func(c *Config) any { return c.MQ.NATS.SubjectPrefix }}, + {"mq.nats.partitions", "WH_MQ_NATS_PARTITIONS", 0, 1, "4", 4, func(c *Config) any { return c.MQ.NATS.Partitions }}, + {"mq.nats.shards", "WH_MQ_NATS_SHARDS", 0, 32, "8", 8, func(c *Config) any { return c.MQ.NATS.Shards }}, + {"mq.nats.ingest_consumer", "WH_MQ_NATS_INGEST_CONSUMER", "", "wh-ingest", "ingest", "ingest", func(c *Config) any { return c.MQ.NATS.IngestConsumer }}, + {"mq.nats.connect_timeout", "WH_MQ_NATS_CONNECT_TIMEOUT", time.Duration(0), 5 * time.Second, "2s", 2 * time.Second, func(c *Config) any { return c.MQ.NATS.ConnectTimeout }}, + {"mq.nats.publish_timeout", "WH_MQ_NATS_PUBLISH_TIMEOUT", time.Duration(0), 5 * time.Second, "2s", 2 * time.Second, func(c *Config) any { return c.MQ.NATS.PublishTimeout }}, + {"mq.nats.topology_wait", "WH_MQ_NATS_TOPOLOGY_WAIT", time.Duration(0), time.Minute, "2s", 2 * time.Second, func(c *Config) any { return c.MQ.NATS.TopologyWait }}, } // refusedZeros are the non-zero defaults whose zero Validate refuses: written @@ -304,7 +312,9 @@ func TestDocs_DefaultsMatchCode(t *testing.T) { func parseDocDefault(t *testing.T, key, cell string, like any) any { t.Helper() cell = strings.TrimSpace(cell) - if cell == "*(empty)*" || cell == "*(required)*" { + // *(empty)*, *(none)*, *(required)*, and a default derived at boot + // (`_coord`) all read as the field's zero. + if strings.HasPrefix(cell, "*(") && strings.HasSuffix(cell, ")*") || strings.Contains(cell, "<") { cell = "" } else { cell = strings.Trim(cell, "`") diff --git a/internal/config/mq_nats.go b/internal/config/mq_nats.go new file mode 100644 index 000000000..b1f0a44e4 --- /dev/null +++ b/internal/config/mq_nats.go @@ -0,0 +1,197 @@ +package config + +import ( + "errors" + "fmt" + "reflect" + "regexp" + "strings" + "time" +) + +// The mq.nats and coord.nats blocks and their rules, apart from backends.go +// and config.go as cache_redis.go is: the e2e stack never selects +// mq.backend: nats, so its coverage gate leaves this file to the unit and +// integration suites (.testcoverage.yml). + +// MQNATSConfig is how to reach the operator's NATS and what topology to +// expect there (mq.NATSConfig, which internal/app builds from it). Secrets +// are file paths only: nothing inline. +type MQNATSConfig struct { + URLs []string `yaml:"urls" env:"WH_MQ_NATS_URLS"` + // Name is the connection name the server reports; empty is + // wavehouse-. + Name string `yaml:"name" env:"WH_MQ_NATS_NAME"` + // CredsFile, NKeySeedFile and User are exclusive: one way to + // authenticate, or none. + CredsFile string `yaml:"creds_file" env:"WH_MQ_NATS_CREDS_FILE"` + NKeySeedFile string `yaml:"nkey_seed_file" env:"WH_MQ_NATS_NKEY_SEED_FILE"` + User string `yaml:"user" env:"WH_MQ_NATS_USER"` + PasswordFile string `yaml:"password_file" env:"WH_MQ_NATS_PASSWORD_FILE"` + TLS MQNATSTLS `yaml:"tls"` + // JSDomain is the JetStream domain, for a leafnode or hub-and-spoke + // deployment. + JSDomain string `yaml:"js_domain" env:"WH_MQ_NATS_JS_DOMAIN"` + SubjectPrefix string `yaml:"subject_prefix" env:"WH_MQ_NATS_SUBJECT_PREFIX"` + Partitions int `yaml:"partitions" env:"WH_MQ_NATS_PARTITIONS"` + // Shards is how many shards each partition has, one durable each. + Shards int `yaml:"shards" env:"WH_MQ_NATS_SHARDS"` + // IngestConsumer names the shard durables: -. + IngestConsumer string `yaml:"ingest_consumer" env:"WH_MQ_NATS_INGEST_CONSUMER"` + // HistoryStream is looked up by name; empty is _HISTORY, + // the name the generated manifests give it. + HistoryStream string `yaml:"history_stream" env:"WH_MQ_NATS_HISTORY_STREAM"` + ConnectTimeout time.Duration `yaml:"connect_timeout" env:"WH_MQ_NATS_CONNECT_TIMEOUT"` + PublishTimeout time.Duration `yaml:"publish_timeout" env:"WH_MQ_NATS_PUBLISH_TIMEOUT"` + TopologyWait time.Duration `yaml:"topology_wait" env:"WH_MQ_NATS_TOPOLOGY_WAIT"` +} + +// MQNATSTLS is the client side of TLS to the NATS servers. +type MQNATSTLS struct { + CAFile string `yaml:"ca_file" env:"WH_MQ_NATS_TLS_CA_FILE"` + CertFile string `yaml:"cert_file" env:"WH_MQ_NATS_TLS_CERT_FILE"` + KeyFile string `yaml:"key_file" env:"WH_MQ_NATS_TLS_KEY_FILE"` + ServerName string `yaml:"server_name" env:"WH_MQ_NATS_TLS_SERVER_NAME"` + HandshakeFirst bool `yaml:"handshake_first" env:"WH_MQ_NATS_TLS_HANDSHAKE_FIRST"` +} + +// defaultMQNATS is the mq.nats part of defaults() +// (TestLoad_MQNATSDefaults pins what Load returns to it). +func defaultMQNATS() MQNATSConfig { + return MQNATSConfig{ + SubjectPrefix: "wh", Partitions: 1, Shards: 32, IngestConsumer: "wh-ingest", + ConnectTimeout: 5 * time.Second, PublishTimeout: 5 * time.Second, TopologyWait: time.Minute, + } +} + +// MaxNATSShards is the largest mq.nats.shards boot accepts: internal/mq's +// own bound, which config cannot import (internal/app pins the two together). +const MaxNATSShards = 256 + +// natsSubjectPrefix is internal/mq's grammar for the prefix: one subject +// token. +var natsSubjectPrefix = regexp.MustCompile(`^[a-z0-9_-]+$`) + +func (n MQNATSConfig) validate() error { + if len(n.URLs) == 0 { + return errors.New("mq.nats.urls (WH_MQ_NATS_URLS) is required with mq.backend=nats") + } + for _, u := range n.URLs { + if u == "" { + return fmt.Errorf("mq.nats.urls (WH_MQ_NATS_URLS) %q has an empty entry", strings.Join(n.URLs, ",")) + } + // A user, password or token in the URL is an inline secret, and would + // also sidestep the one-way-to-authenticate check below. + if strings.Contains(u, "@") { + return errors.New("mq.nats.urls (WH_MQ_NATS_URLS) must not carry credentials (an '@' in a URL): use password_file, nkey_seed_file or creds_file") + } + } + if !natsSubjectPrefix.MatchString(n.SubjectPrefix) { + return fmt.Errorf("mq.nats.subject_prefix (WH_MQ_NATS_SUBJECT_PREFIX) %q must be one token of [a-z0-9_-]", n.SubjectPrefix) + } + if n.Partitions < 1 { + return fmt.Errorf("mq.nats.partitions (WH_MQ_NATS_PARTITIONS) must be at least 1, got %d", n.Partitions) + } + if n.Shards < 1 || n.Shards > MaxNATSShards { + return fmt.Errorf("mq.nats.shards (WH_MQ_NATS_SHARDS) must be from 1 to %d, got %d", MaxNATSShards, n.Shards) + } + if n.IngestConsumer == "" { + return errors.New("mq.nats.ingest_consumer (WH_MQ_NATS_INGEST_CONSUMER) must not be empty") + } + auth := 0 + for _, set := range []string{n.CredsFile, n.NKeySeedFile, n.User} { + if set != "" { + auth++ + } + } + if auth > 1 { + return errors.New("mq.nats: set at most one of creds_file, nkey_seed_file and user") + } + if n.PasswordFile != "" && n.User == "" { + return errors.New("mq.nats.password_file needs mq.nats.user") + } + if (n.TLS.CertFile == "") != (n.TLS.KeyFile == "") { + return errors.New("mq.nats.tls: cert_file and key_file come as a pair") + } + for _, d := range []struct { + key string + v time.Duration + }{ + {"connect_timeout (WH_MQ_NATS_CONNECT_TIMEOUT)", n.ConnectTimeout}, + {"publish_timeout (WH_MQ_NATS_PUBLISH_TIMEOUT)", n.PublishTimeout}, + {"topology_wait (WH_MQ_NATS_TOPOLOGY_WAIT)", n.TopologyWait}, + } { + if d.v <= 0 { + return fmt.Errorf("mq.nats.%s must be positive, got %s", d.key, d.v) + } + } + return nil +} + +// isSet reports whether the block says anything beyond its defaults (or the +// zero value a Config built without Load carries). +func (n MQNATSConfig) isSet() bool { + return !reflect.DeepEqual(n, MQNATSConfig{}) && !reflect.DeepEqual(n, defaultMQNATS()) +} + +// trimURLs drops the spaces a comma-separated WH_MQ_NATS_URLS leaves. +func (n *MQNATSConfig) trimURLs() { + for i, u := range n.URLs { + n.URLs[i] = strings.TrimSpace(u) + } +} + +// natsWarnings are Warnings' lines about the nats blocks, for every role. +func (c *Config) natsWarnings() []string { + var out []string + if c.MQ.Backend != MQNATS && c.MQ.NATS.isSet() { + out = append(out, fmt.Sprintf("mq.nats is set but mq.backend=%s: the block is ignored", c.MQ.Backend)) + } + if c.MQ.Backend == MQNATS { + // WARN although it is by design and fires on every nats boot: the + // key is required in every tenant's config.json, so an operator + // setting a budget there must hear it does nothing (#613). + out = append(out, "mq.max_bytes_gb (settings directory) is not applied with mq.backend=nats: a tenant's queue is bounded by its partition stream's limits, which are the operator's") + } + if c.Coord.Backend != CoordNATS && c.Coord.NATS != (CoordNATSConfig{}) { + out = append(out, fmt.Sprintf("coord.nats is set but coord.backend=%s: the block is ignored", c.Coord.Backend)) + } + return out +} + +// validateNATSTopology is validateTopology's pair of rules for the nats +// backends. +func (c *Config) validateNATSTopology() error { + if c.Coord.Backend == CoordNATS && c.MQ.Backend != MQNATS { + return fmt.Errorf("coord.backend=nats with mq.backend=%s: the NATS leases ride mq.nats's connection — set mq.backend=nats, or coord.backend=local", c.MQ.Backend) + } + // Under nats the sweeper is not wired (retention is the operator's), so + // a process that runs only it would run nothing. + if c.MQ.Backend == MQNATS && len(c.Roles) == 1 && c.Roles[0] == RoleSweeper { + return fmt.Errorf("roles sweeper with mq.backend=nats: this process would run nothing, since the streams' own retention replaces the sweeper there — remove this process") + } + // Ingest processes on a shared queue share its shards through leases, so + // each needs the shared coordinator. + if c.MQ.Backend == MQNATS && c.Coord.Backend == CoordLocal && c.Has(RoleIngest) { + return fmt.Errorf("coord.backend=local with mq.backend=nats in a process running ingest: ingest processes share the queue's shards through shared leases — set coord.backend=nats") + } + return nil +} + +// CoordNATSConfig names the operator's KV bucket. There is no connection +// block: coord.backend=nats rides mq.nats's connection and credentials. +type CoordNATSConfig struct { + // Bucket is the KV bucket the leases live in; empty is + // _coord, the name the generated manifests give it. + Bucket string `yaml:"bucket" env:"WH_COORD_NATS_BUCKET"` +} + +// natsBucketName is JetStream's grammar for a KV bucket name. +var natsBucketName = regexp.MustCompile(`^[a-zA-Z0-9_-]+$`) + +func (n CoordNATSConfig) validate() error { + if n.Bucket != "" && !natsBucketName.MatchString(n.Bucket) { + return fmt.Errorf("coord.nats.bucket (WH_COORD_NATS_BUCKET) %q must be a KV bucket name of [a-zA-Z0-9_-]", n.Bucket) + } + return nil +} diff --git a/internal/config/mq_nats_test.go b/internal/config/mq_nats_test.go new file mode 100644 index 000000000..15dc61d4f --- /dev/null +++ b/internal/config/mq_nats_test.go @@ -0,0 +1,243 @@ +package config + +import ( + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// natsBackends is a valid mq.backend=nats config. +func natsBackends() Config { + c := defaultBackends() + c.MQ.Backend = MQNATS + c.MQ.NATS = defaultMQNATS() + c.MQ.NATS.URLs = []string{"nats://nats:4222"} + c.Coord.Backend = CoordNATS + return c +} + +func TestLoad_MQNATSDefaults(t *testing.T) { + t.Parallel() + cfg, err := Load("nonexistent.yaml") + require.NoError(t, err) + assert.Equal(t, defaultMQNATS(), cfg.MQ.NATS) + assert.False(t, cfg.MQ.NATS.isSet()) + assert.Empty(t, cfg.Warnings()) +} + +func TestLoad_MQNATSFromEnv(t *testing.T) { + for k, v := range map[string]string{ //nolint:gosec // G101: a secret's file path, not the secret + "WH_MQ_BACKEND": "nats", + "WH_COORD_BACKEND": "nats", + "WH_MQ_NATS_URLS": "nats://a:4222, nats://b:4222", + "WH_MQ_NATS_NAME": "wh-api-0", + "WH_MQ_NATS_USER": "wavehouse", + "WH_MQ_NATS_PASSWORD_FILE": "/var/run/secrets/nats/password", + "WH_MQ_NATS_TLS_CA_FILE": "/ca.pem", + "WH_MQ_NATS_TLS_CERT_FILE": "/cert.pem", + "WH_MQ_NATS_TLS_KEY_FILE": "/key.pem", + "WH_MQ_NATS_TLS_SERVER_NAME": "nats.internal", + "WH_MQ_NATS_TLS_HANDSHAKE_FIRST": "true", + "WH_MQ_NATS_JS_DOMAIN": "hub", + "WH_MQ_NATS_SUBJECT_PREFIX": "whprod", + "WH_MQ_NATS_PARTITIONS": "4", + "WH_MQ_NATS_SHARDS": "8", + "WH_MQ_NATS_INGEST_CONSUMER": "ingest", + "WH_MQ_NATS_HISTORY_STREAM": "HIST", + "WH_MQ_NATS_CONNECT_TIMEOUT": "2s", + "WH_MQ_NATS_PUBLISH_TIMEOUT": "3s", + "WH_MQ_NATS_TOPOLOGY_WAIT": "2m", + } { + t.Setenv(k, v) + } + cfg, err := Load("nonexistent.yaml") + require.NoError(t, err) + assert.Equal(t, MQNATSConfig{ //nolint:gosec // G101: a secret's file path, not the secret + URLs: []string{"nats://a:4222", "nats://b:4222"}, Name: "wh-api-0", + User: "wavehouse", PasswordFile: "/var/run/secrets/nats/password", + TLS: MQNATSTLS{CAFile: "/ca.pem", CertFile: "/cert.pem", KeyFile: "/key.pem", ServerName: "nats.internal", HandshakeFirst: true}, + JSDomain: "hub", + SubjectPrefix: "whprod", Partitions: 4, Shards: 8, IngestConsumer: "ingest", HistoryStream: "HIST", + ConnectTimeout: 2 * time.Second, PublishTimeout: 3 * time.Second, TopologyWait: 2 * time.Minute, + }, cfg.MQ.NATS) + assert.True(t, cfg.Distributed()) +} + +func TestLoad_MQNATSFromYAML(t *testing.T) { + t.Parallel() + path := filepath.Join(t.TempDir(), "config.yaml") + require.NoError(t, os.WriteFile(path, []byte(` +mq: + backend: nats + nats: + urls: ["nats://a:4222", "nats://b:4222"] + creds_file: /var/run/secrets/nats/wavehouse.creds + tls: + ca_file: /ca.pem + partitions: 4 + publish_timeout: 2s +coord: + backend: nats +`), 0o600)) + cfg, err := Load(path) + require.NoError(t, err) + want := defaultMQNATS() + want.URLs = []string{"nats://a:4222", "nats://b:4222"} + want.CredsFile = "/var/run/secrets/nats/wavehouse.creds" + want.TLS.CAFile = "/ca.pem" + want.Partitions = 4 + want.PublishTimeout = 2 * time.Second + assert.Equal(t, want, cfg.MQ.NATS) +} + +// Secrets are file paths only: an inline one is an unknown key or an unbound +// variable, never read. +func TestLoad_MQNATSRefusesInlineSecrets(t *testing.T) { + t.Parallel() + path := filepath.Join(t.TempDir(), "config.yaml") + require.NoError(t, os.WriteFile(path, []byte(` +mq: + backend: nats + nats: + urls: ["nats://a:4222"] + user: wavehouse + password: hunter2 + token: abc + tls: + key: inline +`), 0o600)) + _, err := Load(path) + require.Error(t, err) + assert.Contains(t, err.Error(), "mq.nats.password, mq.nats.tls.key, mq.nats.token") + + assert.Equal(t, []string{"WH_MQ_NATS_PASSWORD", "WH_MQ_NATS_TOKEN"}, unboundEnv([]string{ + "WH_MQ_NATS_PASSWORD=hunter2", "WH_MQ_NATS_TOKEN=abc", "WH_MQ_NATS_PASSWORD_FILE=/p", + })) +} + +func TestUnboundEnv_KnowsTheMQNATSVariables(t *testing.T) { + t.Parallel() + assert.Empty(t, unboundEnv([]string{ + "WH_MQ_NATS_URLS=x", "WH_MQ_NATS_NAME=x", "WH_MQ_NATS_CREDS_FILE=x", "WH_MQ_NATS_NKEY_SEED_FILE=x", + "WH_MQ_NATS_USER=x", "WH_MQ_NATS_PASSWORD_FILE=x", "WH_MQ_NATS_TLS_CA_FILE=x", "WH_MQ_NATS_TLS_CERT_FILE=x", + "WH_MQ_NATS_TLS_KEY_FILE=x", "WH_MQ_NATS_TLS_SERVER_NAME=x", "WH_MQ_NATS_TLS_HANDSHAKE_FIRST=x", + "WH_MQ_NATS_JS_DOMAIN=x", "WH_MQ_NATS_SUBJECT_PREFIX=x", "WH_MQ_NATS_PARTITIONS=x", + "WH_MQ_NATS_INGEST_CONSUMER=x", "WH_MQ_NATS_HISTORY_STREAM=x", "WH_MQ_NATS_CONNECT_TIMEOUT=x", + "WH_MQ_NATS_PUBLISH_TIMEOUT=x", "WH_MQ_NATS_TOPOLOGY_WAIT=x", + })) +} + +func TestValidate_MQNATS(t *testing.T) { + t.Parallel() + cases := []struct { + name string + set func(*MQNATSConfig) + want string + }{ + {"valid", func(*MQNATSConfig) {}, ""}, + {"user and password file", func(n *MQNATSConfig) { n.User, n.PasswordFile = "wavehouse", "/p" }, ""}, + {"user alone", func(n *MQNATSConfig) { n.User = "wavehouse" }, ""}, + {"mutual tls", func(n *MQNATSConfig) { n.TLS.CertFile, n.TLS.KeyFile = "/c", "/k" }, ""}, + {"no urls", func(n *MQNATSConfig) { n.URLs = nil }, "mq.nats.urls (WH_MQ_NATS_URLS) is required with mq.backend=nats"}, + {"password in a url", func(n *MQNATSConfig) { n.URLs = []string{"nats://wavehouse:hunter2@nats:4222"} }, "must not carry credentials"}, + {"token in a url", func(n *MQNATSConfig) { n.URLs = []string{"nats://nats:4222", "tls://s3cr3t@nats:4222"} }, "must not carry credentials"}, + {"empty url", func(n *MQNATSConfig) { n.URLs = []string{"nats://a:4222", ""} }, "has an empty entry"}, + {"prefix with a dot", func(n *MQNATSConfig) { n.SubjectPrefix = "wh.prod" }, `mq.nats.subject_prefix (WH_MQ_NATS_SUBJECT_PREFIX) "wh.prod" must be one token`}, + {"prefix upper case", func(n *MQNATSConfig) { n.SubjectPrefix = "WH" }, "must be one token"}, + {"empty prefix", func(n *MQNATSConfig) { n.SubjectPrefix = "" }, "must be one token"}, + {"no partitions", func(n *MQNATSConfig) { n.Partitions = 0 }, "mq.nats.partitions (WH_MQ_NATS_PARTITIONS) must be at least 1, got 0"}, + {"no shards", func(n *MQNATSConfig) { n.Shards = 0 }, "mq.nats.shards (WH_MQ_NATS_SHARDS) must be from 1 to 256, got 0"}, + {"too many shards", func(n *MQNATSConfig) { n.Shards = 257 }, "must be from 1 to 256, got 257"}, + {"no ingest consumer", func(n *MQNATSConfig) { n.IngestConsumer = "" }, "mq.nats.ingest_consumer"}, + {"creds and user", func(n *MQNATSConfig) { n.CredsFile, n.User = "/c", "u" }, "set at most one of creds_file, nkey_seed_file and user"}, + {"creds and nkey", func(n *MQNATSConfig) { n.CredsFile, n.NKeySeedFile = "/c", "/n" }, "set at most one"}, + {"password without user", func(n *MQNATSConfig) { n.PasswordFile = "/p" }, "mq.nats.password_file needs mq.nats.user"}, + {"cert without key", func(n *MQNATSConfig) { n.TLS.CertFile = "/c" }, "cert_file and key_file come as a pair"}, + {"key without cert", func(n *MQNATSConfig) { n.TLS.KeyFile = "/k" }, "come as a pair"}, + {"zero connect timeout", func(n *MQNATSConfig) { n.ConnectTimeout = 0 }, "mq.nats.connect_timeout (WH_MQ_NATS_CONNECT_TIMEOUT) must be positive"}, + {"negative publish timeout", func(n *MQNATSConfig) { n.PublishTimeout = -time.Second }, "mq.nats.publish_timeout (WH_MQ_NATS_PUBLISH_TIMEOUT) must be positive"}, + {"zero topology wait", func(n *MQNATSConfig) { n.TopologyWait = 0 }, "mq.nats.topology_wait"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + cfg := natsBackends() + tc.set(&cfg.MQ.NATS) + err := cfg.Validate() + if tc.want == "" { + require.NoError(t, err) + return + } + require.Error(t, err) + assert.Contains(t, err.Error(), tc.want) + }) + } +} + +// The block is checked only when it is selected: under embedded it is not +// read, so it cannot refuse boot, and boot says it is ignored. +func TestValidate_MQNATSIgnoredUnderEmbedded(t *testing.T) { + t.Parallel() + cfg := defaultBackends() + cfg.MQ.NATS = defaultMQNATS() + cfg.MQ.NATS.Partitions = 0 + require.NoError(t, cfg.Validate()) + assert.Equal(t, []string{"mq.nats is set but mq.backend=embedded: the block is ignored"}, cfg.Warnings()) +} + +// On a shared queue every role split boots except the one the local cache +// cannot serve (rule 5; cache.backend: redis lifts it), and a sweeper-only +// process, which has nothing to run there. +func TestValidate_SplitsBootOnNATS(t *testing.T) { + t.Parallel() + for _, tc := range []struct { + roles []Role + want string + }{ + {AllRoles(), ""}, + {[]Role{RoleAPI, RoleIngest}, ""}, + {[]Role{RoleSweeper}, "would run nothing"}, + {[]Role{RoleAPI}, "roles api with cache.backend=local"}, + {[]Role{RoleIngest, RoleSweeper}, "roles ingest,sweeper with cache.backend=local"}, + } { + cfg := natsBackends() + cfg.Roles = tc.roles + err := cfg.Validate() + if tc.want == "" { + assert.NoError(t, err, "roles %v", tc.roles) + continue + } + require.Error(t, err, "roles %v", tc.roles) + assert.Contains(t, err.Error(), tc.want) + } +} + +func TestWarnings_MQNATS(t *testing.T) { + t.Parallel() + const ( + maxBytes = "mq.max_bytes_gb (settings directory) is not applied with mq.backend=nats" + cache = "cache.backend=local" + dedupe = "dedupe.backend=pebble" + ) + warnings := func(roles ...Role) []string { + cfg := natsBackends() + cfg.Roles = roles + require.NoError(t, cfg.Validate()) + var got []string + for _, w := range cfg.Warnings() { + for _, key := range []string{maxBytes, cache, dedupe} { + if strings.HasPrefix(w, key) { + got = append(got, key) + } + } + } + require.Len(t, got, len(cfg.Warnings()), "every warning is one of the known ones") + return got + } + assert.Equal(t, []string{maxBytes, cache, dedupe}, warnings(AllRoles()...)) +} diff --git a/internal/config/roles_test.go b/internal/config/roles_test.go index 914815bf9..b4c92d078 100644 --- a/internal/config/roles_test.go +++ b/internal/config/roles_test.go @@ -39,7 +39,7 @@ func TestLoad_RolesFromEnv(t *testing.T) { } // One role parses to one entry — refused here only because the embedded MQ -// cannot be split, which is the message a split gets until a shared MQ lands. +// cannot be split; mq.backend=nats can (mq_nats_test.go). func TestLoad_OneRoleFromEnvIsRefusedOnTheEmbeddedMQ(t *testing.T) { t.Setenv("WH_ROLES", "ingest") _, err := Load("nonexistent.yaml") @@ -92,7 +92,7 @@ func TestValidate_Roles(t *testing.T) { } // Rules 2 and 5 of the #613 design. The embedded MQ refuses every split. A -// shared queue, which no backend offers yet and so is set directly, lets a +// shared queue, set directly as a stand-in for any shared backend, lets a // process run any subset — except api without ingest or ingest without api // over a local cache: the worker's invalidation would miss the API's cache. A // sweeper-only process holds no cache, so it passes. diff --git a/internal/coord/coord.go b/internal/coord/coord.go index 8a080b967..411c6f610 100644 --- a/internal/coord/coord.go +++ b/internal/coord/coord.go @@ -1,5 +1,5 @@ // Package coord holds leases for work that must run in one process at a -// time — the sweeper today, partition claims later. A lease is taken with +// time — the sweeper, and the ingest processes' shard claim membership. A lease is taken with // TryAcquire and held as a Term until it is resigned, its coordinator is // closed, or the backend reports it lost; RunElected drives a leader loop // over one. Local is the in-process implementation; a distributed one lives @@ -41,6 +41,16 @@ type Coordinator interface { Close(ctx context.Context) error } +// Observer is implemented by a Coordinator that can tell whether a lease is +// held without campaigning for it: how a set of leases works out how many +// holders are live (the ingest processes count their peers this way). +type Observer interface { + // Held reports whether the named lease is live under any holder, this + // coordinator included, judged as TryAcquire would judge it: false for + // a lease TryAcquire could take now. + Held(ctx context.Context, name string) (bool, error) +} + // Term is one holding of a lease. type Term interface { Name() string diff --git a/internal/coord/coordtest/coordtest.go b/internal/coord/coordtest/coordtest.go index 46a8fd111..d24c45621 100644 --- a/internal/coord/coordtest/coordtest.go +++ b/internal/coord/coordtest/coordtest.go @@ -124,6 +124,30 @@ func Conformance(t *testing.T, newPair Factory, opts ...Option) { require.ErrorIs(t, err, coord.ErrHeld, "the term outlives the context it was taken under") }) + t.Run("held without campaigning", func(t *testing.T) { + a, b := pair(t) + oa, ok := a.(coord.Observer) + if !ok { + t.Skip("this backend does not observe leases") + } + ob := b.(coord.Observer) + held := func(o coord.Observer, name string) bool { + h, err := o.Held(t.Context(), name) + require.NoError(t, err) + return h + } + assert.False(t, held(oa, "lease"), "never taken") + term := acquire(t, a, "lease") + assert.True(t, held(oa, "lease"), "by its holder") + assert.True(t, held(ob, "lease"), "by another") + assert.True(t, held(ob, "lease"), "observing twice takes nothing") + assertOpen(t, term) + require.NoError(t, term.Resign(t.Context())) + assert.False(t, held(oa, "lease"), "resigned") + assert.False(t, held(ob, "lease"), "resigned, seen by another") + acquire(t, b, "lease") + }) + t.Run("loss closes Done", func(t *testing.T) { if o.lose == nil { t.Skip("this backend never loses a live term") diff --git a/internal/coord/local.go b/internal/coord/local.go index ca44dfbba..802a411d2 100644 --- a/internal/coord/local.go +++ b/internal/coord/local.go @@ -60,6 +60,17 @@ func (l *Local) TryAcquire(ctx context.Context, name string) (Term, error) { return t, nil } +// Held implements Observer. +func (l *Local) Held(ctx context.Context, name string) (bool, error) { + if err := ctx.Err(); err != nil { + return false, err + } + l.table.mu.Lock() + defer l.table.mu.Unlock() + _, ok := l.table.held[name] + return ok, nil +} + // Close implements Coordinator. func (l *Local) Close(context.Context) error { l.mu.Lock() diff --git a/internal/ingest/assign.go b/internal/ingest/assign.go new file mode 100644 index 000000000..0c59e38e3 --- /dev/null +++ b/internal/ingest/assign.go @@ -0,0 +1,74 @@ +package ingest + +import ( + "cmp" + "hash/fnv" + "slices" + "strconv" +) + +// assignUnits gives each unit to one of the live membership slots, by +// rendezvous hashing capped at an even share: each unit ranks the slots by a +// score of its own, the units are taken best score first, and each goes to +// its highest-ranked slot still under ⌈units/slots⌉. The result depends only +// on its inputs' contents, not their order, so every process that sees the +// same slots and units derives the same owners. A slot joining or leaving +// moves close to the fewest units it can, and no slot owns more than the cap. +func assignUnits(units []string, slots []int) map[string]int { + out := make(map[string]int, len(units)) + if len(slots) == 0 { + return out + } + slots = slices.Sorted(slices.Values(slots)) + type ranked struct { + unit string + order []int // slots, best score first + best uint64 + } + rs := make([]ranked, len(units)) + for i, u := range units { + scores := make(map[int]uint64, len(slots)) + for _, s := range slots { + scores[s] = unitScore(u, s) + } + order := slices.Clone(slots) + slices.SortFunc(order, func(a, b int) int { + if c := cmp.Compare(scores[b], scores[a]); c != 0 { + return c + } + return cmp.Compare(a, b) + }) + rs[i] = ranked{unit: u, order: order, best: scores[order[0]]} + } + slices.SortFunc(rs, func(a, b ranked) int { + if c := cmp.Compare(b.best, a.best); c != 0 { + return c + } + return cmp.Compare(a.unit, b.unit) + }) + limit := (len(units) + len(slots) - 1) / len(slots) + load := make(map[int]int, len(slots)) + for _, r := range rs { + for _, s := range r.order { + if load[s] < limit { + out[r.unit] = s + load[s]++ + break + } + } + } + return out +} + +// unitScore is slot's rendezvous score for unit: FNV-1a 64 of the two, +// through splitmix64's finalizer so that nearby slots score independently. +func unitScore(unit string, slot int) uint64 { + h := fnv.New64a() + _, _ = h.Write([]byte(unit)) + _, _ = h.Write([]byte{0}) + _, _ = h.Write([]byte(strconv.Itoa(slot))) + z := h.Sum64() + z = (z ^ (z >> 30)) * 0xbf58476d1ce4e5b9 + z = (z ^ (z >> 27)) * 0x94d049bb133111eb + return z ^ (z >> 31) +} diff --git a/internal/ingest/assign_test.go b/internal/ingest/assign_test.go new file mode 100644 index 000000000..a05a0e072 --- /dev/null +++ b/internal/ingest/assign_test.go @@ -0,0 +1,119 @@ +package ingest + +import ( + "fmt" + "math/rand/v2" + "slices" + "testing" + + "github.com/stretchr/testify/assert" +) + +func unitNames(n int) []string { + out := make([]string, n) + for i := range out { + out[i] = fmt.Sprintf("WH_INGEST_%d/wh-ingest-%d", i/8, i%8) + } + return out +} + +func slotsUpTo(m int) []int { + out := make([]int, m) + for i := range out { + out[i] = i + } + return out +} + +func loads(assignment map[string]int) map[int]int { + out := map[int]int{} + for _, s := range assignment { + out[s]++ + } + return out +} + +// Every process derives the same owners from the same members and units, +// in whatever order it lists them, and every unit gets one. +func TestAssignUnits_Deterministic(t *testing.T) { + t.Parallel() + units, slots := unitNames(32), []int{0, 3, 4, 9} + want := assignUnits(units, slots) + assert.Len(t, want, 32) + rng := rand.New(rand.NewPCG(1, 2)) //nolint:gosec // G404: a shuffle, not a secret + for range 20 { + u, s := slices.Clone(units), slices.Clone(slots) + rng.Shuffle(len(u), func(i, j int) { u[i], u[j] = u[j], u[i] }) + rng.Shuffle(len(s), func(i, j int) { s[i], s[j] = s[j], s[i] }) + assert.Equal(t, want, assignUnits(u, s)) + } + assert.Empty(t, assignUnits(units, nil), "no members, no owners") +} + +// No member owns more than an even share, rounded up, and every unit is owned. +func TestAssignUnits_Cap(t *testing.T) { + t.Parallel() + for _, u := range []int{16, 32, 64} { + for m := 1; m <= 12; m++ { + got := assignUnits(unitNames(u), slotsUpTo(m)) + assert.Len(t, got, u) + limit := (u + m - 1) / m + for s, n := range loads(got) { + assert.LessOrEqual(t, n, limit, "units %d, members %d, slot %d", u, m, s) + } + } + } +} + +// A member joining moves close to the fewest units it can while the even +// share's cap holds: at most 1.5x the share the newcomer takes, up to six +// members. Where the cap steps down (⌈32/10⌉=4 to ⌈32/11⌉=3) units re-flow to +// fit it, so the bound for any count is range assignment's, half the units, +// which it never exceeds. A member leaving moves about its own units, at most +// twice as many. +func TestAssignUnits_Movement(t *testing.T) { + t.Parallel() + const u = 32 + units := unitNames(u) + for w := 1; w < u; w++ { + before, after := assignUnits(units, slotsUpTo(w)), assignUnits(units, slotsUpTo(w+1)) + moved := 0 + for _, unit := range units { + if before[unit] != after[unit] { + moved++ + } + } + ideal := float64(u) / float64(w+1) + t.Logf("%2d → %2d members: %2d of %d units moved (ideal %.1f)", w, w+1, moved, u, ideal) + assert.LessOrEqual(t, moved, u/2, "%d → %d members", w, w+1) + if w <= 6 { + assert.LessOrEqual(t, float64(moved), 1.5*ideal+1, "%d → %d members", w, w+1) + } + } + five := assignUnits(units, []int{0, 1, 2, 3, 4}) + four := assignUnits(units, []int{0, 1, 3, 4}) + own, moved := loads(five)[2], 0 + for _, unit := range units { + if five[unit] != four[unit] { + moved++ + } + } + t.Logf("slot 2 of 5 leaves: %d units moved, %d of them its own", moved, own) + assert.LessOrEqual(t, moved, 2*own) +} + +// The mapping is pinned, so a change to it is deliberate: every process of +// a rolling upgrade must compute the same owners. +func TestAssignUnits_Golden(t *testing.T) { + t.Parallel() + got := assignUnits(unitNames(32), []int{0, 1, 2}) + var owners []int + for _, u := range unitNames(32) { + owners = append(owners, got[u]) + } + assert.Equal(t, goldenOwners, owners) +} + +// goldenOwners was derived apart from this code, by the same algorithm in +// another language (FNV-1a 64, splitmix64's finalizer, capped rendezvous). +var goldenOwners = []int{1, 0, 0, 0, 1, 1, 1, 2, 2, 2, 0, 0, 0, 2, 2, 2, 0, 1, 0, 1, 0, 2, 0, 1, 1, 1, 2, 0, 2, 1, 1, 2} diff --git a/internal/ingest/claims.go b/internal/ingest/claims.go new file mode 100644 index 000000000..1cadb378d --- /dev/null +++ b/internal/ingest/claims.go @@ -0,0 +1,761 @@ +package ingest + +import ( + "context" + "errors" + "fmt" + "log/slog" + "maps" + "slices" + "strconv" + "sync" + "sync/atomic" + "time" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/metric" + + "github.com/Wave-RF/WaveHouse/internal/coord" + "github.com/Wave-RF/WaveHouse/internal/mq" +) + +// Shard claims: over a sharded queue (mq.Sharded), each unit — one shard of +// one partition — is consumed by one ingest process, so every table, which +// lives in one unit, has one process batching its inserts. The broker makes +// that exclusive (one pinned puller receives at a time); the claims make it +// balanced. +// +// Membership. Each process holds one lease, "ingest.m" for the lowest free +// j below the number of configured units (at most maxMembers), and every tick +// reads which slots are live (coord.Observer). A process without a slot +// idles. +// +// Assignment. Every process computes the same owner for every unit from the +// live slots (assignUnits: rendezvous hashing capped at an even share), and +// consumes the units assigned to its own slot. Two processes whose views of +// the members differ may both pull a unit, and the broker delivers it to one +// of them. Views can differ for up to a lease: each process starts a +// silent member's expiry clock when it first sees it. +// +// Handover. A unit no longer assigned here stops fetching while keeping the +// broker's pin, waits until the rows it delivered are settled (bounded by +// Handover), then releases the pin, so the next owner receives at once and +// only rows newer than the ones written here. A clean stop halts every unit +// first (Halt), so the worker can flush what it holds, and releases them +// after. +// +// Takeover. A unit whose owner in the previous tick's view is no longer a +// member is reset to its ack floor before it is consumed (ResetOrphaned), so +// the dead owner's unacked rows come back at once instead of after ack_wait. +// The broker refuses the reset while the unit is pinned or was recently +// active, so the membership view alone never resets a live owner's rows; +// the unit then waits, unbound, and is tried again next tick, up to +// orphanWait. On a process's first tick, +// which has no previous view, only when it is the one live member: then +// whatever holds rows unpinned is gone. That covers the only process +// restarting after a clean stop, which resigned its lease. After a crash +// the dead run's lease still counts as live for a lease duration, so the +// restarted process takes the units assigned to its own slot without a +// reset (their rows come back after ack_wait), and the dead slot's units by +// the takeover above once its lease has lapsed. Never after pulling: a +// reset also redelivers the caller's own unacked rows. +// +// Memory. Each unit may hold delivered and unsettled rows up to its share of +// MaxHeld: an even share over the units this process is assigned, and never +// less than minUnitHeld (or MaxHeld, if smaller), so one hot table still +// fills whole batches. At its share a unit fetches only what keeps its pin +// (the broker's: a row every few seconds, at most one ack_wait's worth past +// the share), and the broker keeps the rest; a stuck unit +// never takes another unit's share. The process holds about max(MaxHeld, +// units × minUnitHeld) at most. +const ( + memberLeasePrefix = "ingest.m" + + defaultUnownedEvery = 15 * time.Second + // minUnitHeld is two of the worker's batches: one being written while + // the next fills. + minUnitHeld = 2 * defaultMaxBatch + // settlePoll is how often a handover checks its rows have settled. + settlePoll = 50 * time.Millisecond + // bindEscalate is how long a configured unit may fail to bind, for a + // reason that may pass, before each failure logs an error. + bindEscalate = time.Minute + // tickTimeout bounds one tick's requests. + tickTimeout = 10 * time.Second + // releaseWait bounds a lease resign or a unit's release. + releaseWait = 5 * time.Second + // maxMembers caps the membership slots, and so the ingest processes that + // get work, whatever the unit count: every tick reads every slot. + maxMembers = 64 + // membershipReaders is how many slots a tick reads at once. + membershipReaders = 16 + // orphanWait bounds how long a unit taken over waits for its dead + // owner's pin to lapse before it is bound anyway: past it, whoever holds + // the pin is alive, and the broker's pin decides. + orphanWait = 30 * time.Second +) + +func memberLease(j int) string { return memberLeasePrefix + strconv.Itoa(j) } + +// ClaimConfig tunes ClaimShards. +type ClaimConfig struct { + // Every is how often membership is read and units claimed or given up; + // default coord.RetryPeriod. + Every time.Duration + // Handover bounds how long a unit given up waits for its delivered rows + // to settle before it is released; default the worker's ack_wait, after + // which the broker redelivers them anyway. + Handover time.Duration + // MaxHeld is the process's budget of rows delivered and not yet settled, + // shared out per unit (see Memory above); default the worker's + // maxAckPending. + MaxHeld int + // UnownedEvery is how often the lowest live member reads how many units + // have rows and no owner; default 15s. + UnownedEvery time.Duration + // SlotExpiry frees a delivered row's MaxHeld slot if the worker never + // settles it: by then the broker redelivers the row anyway. Default the + // worker's ack_wait. + SlotExpiry time.Duration +} + +// shardedQueue is a queue ClaimShards can share out. +type shardedQueue interface { + Queue + mq.Sharded +} + +// ClaimShards wraps q so that its consumer takes only the units this process +// is assigned through c, which every ingest process must share and which +// must implement coord.Observer. q must implement mq.Sharded. +func ClaimShards(q Queue, c coord.Coordinator, cfg ClaimConfig) (Queue, error) { + sq, ok := q.(shardedQueue) + if !ok { + return nil, fmt.Errorf("shard claims need a sharded queue, got %T", q) + } + obs, ok := c.(coord.Observer) + if !ok { + return nil, fmt.Errorf("shard claims need a coordinator that observes leases, got %T", c) + } + if cfg.Every <= 0 { + cfg.Every = coord.RetryPeriod + } + if cfg.Handover <= 0 { + cfg.Handover = ackWait + } + if cfg.MaxHeld <= 0 { + cfg.MaxHeld = maxAckPending + } + if cfg.UnownedEvery <= 0 { + cfg.UnownedEvery = defaultUnownedEvery + } + if cfg.SlotExpiry <= 0 { + cfg.SlotExpiry = ackWait + } + return &claimingQueue{Queue: q, sq: sq, c: c, obs: obs, cfg: cfg}, nil +} + +type claimingQueue struct { + Queue + sq shardedQueue + c coord.Coordinator + obs coord.Observer + cfg ClaimConfig +} + +// CreateConsumer checks every configured unit's durable now, as a consumer +// of all of them would, so a missing one still refuses boot; units are bound +// one by one as they are claimed. +func (q *claimingQueue) CreateConsumer(ctx context.Context, cfg mq.ConsumerConfig) (mq.Consumer, error) { + if cfg.Units != nil { + return nil, errors.New("a claiming consumer chooses its own units") + } + configured, _ := q.sq.IngestUnits() + if len(configured) == 0 { + return nil, errors.New("shard claims need at least one unit") + } + check := cfg + check.Units = configured + if _, err := q.sq.CreateConsumer(ctx, check); err != nil { + return nil, err + } + return &claimingConsumer{q: q, ctx: ctx, cfg: cfg}, nil +} + +type claimingConsumer struct { + q *claimingQueue + ctx context.Context + cfg mq.ConsumerConfig + // loop is the running claims, once Consume has started them. + loop atomic.Pointer[claimLoop] +} + +// Consume starts claiming. stop ends it, releasing every unit held and +// resigning the membership lease; failed reports, exactly once, a configured +// unit's delivery ending on its own or a unit that could not be bound +// because its durable or stream is gone or no longer fits. +func (c *claimingConsumer) Consume(handler func(*mq.Message), prefetch int) (func(), <-chan error, error) { + configured, _ := c.q.sq.IngestUnits() + loopCtx, cancel := context.WithCancel(context.WithoutCancel(c.ctx)) + l := &claimLoop{ + q: c.q, ctx: loopCtx, cancel: cancel, cfg: c.cfg, handler: handler, prefetch: prefetch, + slots: min(len(configured), maxMembers), configured: map[string]bool{}, + owned: map[string]*claim{}, releasing: map[string]*claim{}, orphaned: map[string]time.Time{}, + unbound: map[string]time.Time{}, + failed: make(chan error, 1), halting: make(chan struct{}), quit: make(chan struct{}), done: make(chan struct{}), + } + l.share.Store(int64(c.q.cfg.MaxHeld)) + for _, u := range configured { + l.configured[u] = true + } + var err error + if l.gauges, err = registerClaimGauges(l); err != nil { + slog.Warn("ingest: shard claim gauges not registered", "error", err) + } + c.loop.Store(l) + go l.run() + return l.stop, l.failed, nil +} + +// Halt implements mq.Halter: no unit is taken or given up any more, and every +// unit stops fetching while keeping its pin; it returns once what they +// fetched has reached the handler. stop, after the worker has flushed, +// releases them. +func (c *claimingConsumer) Halt() { + if l := c.loop.Load(); l != nil { + l.halt() + } +} + +// claimLoop is one process's claims. member, owned, prev, unbound and +// lastUnowned belong to the run goroutine (and to halt and stop once it has +// exited); releasing is shared with the handovers. +type claimLoop struct { + q *claimingQueue + ctx context.Context + cancel context.CancelFunc + cfg mq.ConsumerConfig + handler func(*mq.Message) + prefetch int + slots int + configured map[string]bool + + member coord.Term + memberSnap atomic.Pointer[coord.Term] // member, for readers off the run goroutine + slot int + owned map[string]*claim + prev map[string]int // the previous tick's owners, nil before the first + // orphaned holds the units taken over from a dead owner that still wait + // for a reset, and since when. + orphaned map[string]time.Time + // unbound holds the configured units whose bind keeps failing, and since + // when. + unbound map[string]time.Time + lastUnowned time.Time + + mu sync.Mutex + releasing map[string]*claim + handovers sync.WaitGroup + + // share is each unit's cap on rows delivered and unsettled, as of the + // last tick; held counts them across the units. + share, held atomic.Int64 + + ownedGauge, membersGauge, unownedGauge atomic.Int64 + rankZero atomic.Bool + gauges metric.Registration + + failed chan error + reported, stopped atomic.Bool + // halting ends the ticks (halt); quit also cuts handovers short (stop). + halting, quit, done chan struct{} + haltOnce, stopOnce sync.Once +} + +// claim is one unit this process holds. +type claim struct { + unit string + cons mq.Consumer + stopOnce sync.Once + stopFn func() + // inflight counts rows delivered and not yet settled. + inflight atomic.Int64 + // halted closes once the unit has stopped fetching. + halted chan struct{} +} + +// stop halts the unit's fetching, keeping its pin, and returns once what it +// fetched has reached the handler, where the consumer can tell (mq.Halter). +func (cl *claim) stop() { + cl.stopOnce.Do(func() { + if h, ok := cl.cons.(mq.Halter); ok { + h.Halt() + } + if cl.stopFn != nil { + cl.stopFn() + } + close(cl.halted) + }) +} + +func (l *claimLoop) run() { + defer close(l.done) + t := time.NewTicker(l.q.cfg.Every) + defer t.Stop() + for { + l.tick() + select { + case <-l.halting: + return + case <-t.C: + } + } +} + +// tick reads membership, then gives up or takes units to match this +// process's assignment. A failed read skips the rest of the tick: acting on +// a partial view of the members would move units for nothing. +func (l *claimLoop) tick() { + ctx, cancel := context.WithTimeout(l.ctx, tickTimeout) + defer cancel() + defer func() { l.ownedGauge.Store(int64(len(l.owned))) }() + defer func() { m := l.member; l.memberSnap.Store(&m) }() + l.reap() + if l.member == nil { + l.joinMembers(ctx) + } + live, err := l.readMembers(ctx) + if err != nil { + l.warn("ingest: could not read the shard claim membership", err) + return + } + l.membersGauge.Store(int64(len(live))) + // Apart, so processes whose lists of extras differ for a while (each + // refreshes it on its own timer) still agree on the configured units. + configured, extra := l.q.sq.IngestUnits() + assignment := assignUnits(configured, live) + maps.Copy(assignment, assignUnits(extra, live)) + mine := l.member != nil && slices.Contains(live, l.slot) + targets := map[string]bool{} + if mine { + for u, s := range assignment { + if s == l.slot { + targets[u] = true + } + } + } + if len(targets) > 0 { + l.share.Store(int64(unitShare(l.q.cfg.MaxHeld, len(targets)))) + } + l.giveUp(targets) + l.take(ctx, targets, len(targets), live) + l.prev = assignment + l.rankZero.Store(mine && len(live) > 0 && live[0] == l.slot) + l.readUnowned() +} + +// memberTerm is the membership term as of the last tick, nil for none. +func (l *claimLoop) memberTerm() coord.Term { + if t := l.memberSnap.Load(); t != nil { + return *t + } + return nil +} + +// readMembers reads which membership slots are live, a few at a time. +func (l *claimLoop) readMembers(ctx context.Context) ([]int, error) { + held := make([]bool, l.slots) + errs := make([]error, l.slots) + sem := make(chan struct{}, membershipReaders) + var wg sync.WaitGroup + for j := range l.slots { + sem <- struct{}{} + wg.Go(func() { + defer func() { <-sem }() + held[j], errs[j] = l.q.obs.Held(ctx, memberLease(j)) + }) + } + wg.Wait() + if err := errors.Join(errs...); err != nil { + return nil, err + } + var live []int + for j, h := range held { + if h { + live = append(live, j) + } + } + return live, nil +} + +// reap drops the membership term if it ended under this process; the same +// tick joins again, and gives up whatever the new slot is not assigned. +func (l *claimLoop) reap() { + if l.member == nil { + return + } + select { + case <-l.member.Done(): + slog.Warn("ingest: lost the shard claim membership lease; giving up every shard", "slot", l.slot, "error", l.member.Err()) + claimEvents.Add(context.Background(), 1, eventAttr("lost")) + l.member = nil + default: + } +} + +// joinMembers takes the lowest free membership slot. With every slot taken, +// or on an error, the process stays a member of none this tick. +func (l *claimLoop) joinMembers(ctx context.Context) { + for j := range l.slots { + term, err := l.q.c.TryAcquire(ctx, memberLease(j)) + switch { + case err == nil: + l.member, l.slot = term, j + slog.Info("ingest: joined the shard claim membership", "slot", j) + return + case errors.Is(err, coord.ErrHeld): + case errors.Is(err, coord.ErrClosed): + return + default: + l.warn("ingest: could not join the shard claim membership", err) + return + } + } +} + +// giveUp hands on every unit held that is not a target, and forgets the +// takeovers of units that are no longer targets. +func (l *claimLoop) giveUp(targets map[string]bool) { + for u := range l.orphaned { + if !targets[u] { + delete(l.orphaned, u) + } + } + for _, u := range slices.Sorted(maps.Keys(l.owned)) { + if targets[u] { + continue + } + cl := l.owned[u] + delete(l.owned, u) + l.mu.Lock() + l.releasing[u] = cl + l.mu.Unlock() + l.handovers.Go(func() { l.handOver(cl) }) + } +} + +// take binds every target not held yet. A unit whose previous owner is no +// longer a member is reset first, so its unsettled rows come back at once. +func (l *claimLoop) take(ctx context.Context, targets map[string]bool, count int, live []int) { + for _, u := range slices.Sorted(maps.Keys(targets)) { + if _, ok := l.owned[u]; ok || l.isReleasing(u) { + continue + } + owner, known := l.prev[u] + dead := known && owner != l.slot && !slices.Contains(live, owner) + alone := l.prev == nil && len(live) == 1 && live[0] == l.slot + if dead || alone { + if _, waiting := l.orphaned[u]; !waiting { + l.orphaned[u] = time.Now() + } + } + if since, ok := l.orphaned[u]; ok { + switch reset, err := l.q.sq.ResetOrphaned(ctx, u); { + case errors.Is(err, mq.ErrUnitHeld) && time.Since(since) < orphanWait: + continue // its pin has not lapsed yet, or it was active too recently + case errors.Is(err, mq.ErrUnitHeld): + case err != nil: + l.warn("ingest: could not take over a shard's unsettled rows; they come back after ack_wait", err) + case reset: + claimEvents.Add(context.Background(), 1, eventAttr("reset")) + slog.Info("ingest: took over a shard's unsettled rows", "unit", u, "waited", time.Since(since).Round(time.Millisecond)) + } + delete(l.orphaned, u) + } + if err := l.open(u, count); err != nil { + if !l.configured[u] { + slog.Info("ingest: could not bind a shard durable outside the configured ones", "unit", u, "error", err) + continue + } + if errors.Is(err, mq.ErrConsumerNotFound) || errors.Is(err, mq.ErrConsumerMismatch) { + // Its durable or stream is gone, or the durable no longer + // fits: the same end as a delivery the broker ends because + // the durable was deleted under it. + l.fail(fmt.Errorf("shard %s: %w: %w", u, mq.ErrDeliveryEnded, err)) + return + } + // Anything else (a request the server did not answer while a + // consumer leader moves) is tried again next tick; the unit + // shows as unowned meanwhile, and a failure that lasts logs an + // error. + claimEvents.Add(context.Background(), 1, eventAttr("bind_failed")) + since, ok := l.unbound[u] + if !ok { + since = time.Now() + l.unbound[u] = since + } + err = fmt.Errorf("shard %s: %w", u, err) + if unbound := time.Since(since); unbound >= bindEscalate && l.ctx.Err() == nil { + slog.Error("ingest: a shard has not been bound for a while; its rows wait", "unbound_for", unbound.Round(time.Second), "error", err) + } else { + l.warn("ingest: could not bind a shard; trying again next tick", err) + } + continue + } + delete(l.unbound, u) + } +} + +func (l *claimLoop) isReleasing(u string) bool { + l.mu.Lock() + defer l.mu.Unlock() + _, ok := l.releasing[u] + return ok +} + +// unitShare is each of count units' cap on rows delivered and unsettled. +func unitShare(maxHeld, count int) int { + return max(min(maxHeld, minUnitHeld), maxHeld/max(1, count)) +} + +// open binds unit u and starts delivering it, under the unit's share of +// MaxHeld, counting each row until it settles. +func (l *claimLoop) open(u string, count int) error { + cfg := l.cfg + cfg.Units = []string{u} + cfg.MaxHeld = func() int { return int(l.share.Load()) } + cons, err := l.q.sq.CreateConsumer(l.ctx, cfg) + if err != nil { + return err + } + cl := &claim{unit: u, cons: cons, halted: make(chan struct{})} + share := l.prefetch + if share > 0 { + share = max(1, share/max(1, count)) + } + stop, failed, err := cons.Consume(func(m *mq.Message) { + cl.inflight.Add(1) + l.held.Add(1) + // The row leaves this process's hands at its first settle attempt, + // or, if the worker never makes one (it leaves some rows for the + // broker to redeliver), once the broker would redeliver it anyway. + letGo := sync.OnceFunc(func() { + cl.inflight.Add(-1) + l.held.Add(-1) + }) + expire := time.AfterFunc(l.q.cfg.SlotExpiry, letGo) + m.OnSettled(func() { expire.Stop(); letGo() }) + l.handler(m) + }, share) + if err != nil { + return err + } + cl.stopFn = stop + l.owned[u] = cl + l.watch(cl, failed) + claimEvents.Add(context.Background(), 1, eventAttr("taken")) + slog.Info("ingest: took a shard", "unit", u, "slot", l.slot) + return nil +} + +// watch reports a configured unit's delivery ending on its own; an extra's +// ending is the operator deleting it once drained. +func (l *claimLoop) watch(cl *claim, failed <-chan error) { + go func() { + select { + case err := <-failed: + if !l.configured[cl.unit] { + slog.Info("ingest: stopped draining a shard outside the configured ones", "unit", cl.unit, "reason", err) + return + } + l.fail(fmt.Errorf("shard %s: %w", cl.unit, err)) + case <-cl.halted: + case <-l.quit: + } + }() +} + +// handOver stops fetching cl's unit, keeping its pin, waits for what it +// delivered to settle, bounded by Handover (or cut short by stop), and +// releases it. +func (l *claimLoop) handOver(cl *claim) { + began := time.Now() + cl.stop() + deadline := time.After(l.q.cfg.Handover) + poll := time.NewTicker(settlePoll) + defer poll.Stop() +wait: + for cl.inflight.Load() > 0 { + select { + case <-poll.C: + case <-deadline: + claimEvents.Add(context.Background(), 1, eventAttr("handover_timeout")) + slog.Warn("ingest: handing a shard on before its rows settled", "unit", cl.unit, "unsettled", cl.inflight.Load()) + break wait + case <-l.quit: + break wait + } + } + l.release(cl) + l.mu.Lock() + delete(l.releasing, cl.unit) + l.mu.Unlock() + handoverSeconds.Record(context.Background(), time.Since(began).Seconds()) + claimEvents.Add(context.Background(), 1, eventAttr("released")) + slog.Info("ingest: handed a shard on", "unit", cl.unit, "took", time.Since(began).Round(time.Millisecond)) +} + +// release gives cl's unit up at the broker, best effort: a pin not released +// lapses on its own. +func (l *claimLoop) release(cl *claim) { + r, ok := cl.cons.(mq.Releaser) + if !ok { + return + } + ctx, cancel := context.WithTimeout(context.Background(), releaseWait) + defer cancel() + if err := r.Release(ctx); err != nil { + slog.Warn("ingest: could not release a shard; its pin lapses on its own", "unit", cl.unit, "error", err) + } +} + +// readUnowned has the lowest live member count the units with rows and no +// owner, every UnownedEvery; the others report nothing. +func (l *claimLoop) readUnowned() { + if !l.rankZero.Load() || time.Since(l.lastUnowned) < l.q.cfg.UnownedEvery { + return + } + // A context of its own, not what the tick's reads left. + ctx, cancel := context.WithTimeout(l.ctx, tickTimeout) + defer cancel() + n, err := l.q.sq.Unowned(ctx) + if err != nil { + l.warn("ingest: could not count the shards without an owner", err) + return + } + l.lastUnowned = time.Now() + l.unownedGauge.Store(int64(n)) +} + +func (l *claimLoop) warn(msg string, err error) { + if l.ctx.Err() == nil { + slog.Warn(msg, "error", err) + } +} + +func (l *claimLoop) fail(err error) { + if l.stopped.Load() || !l.reported.CompareAndSwap(false, true) { + return + } + l.failed <- err +} + +// halt ends the ticks and halts every unit held or being handed over, at +// once, returning when none will deliver again. +func (l *claimLoop) halt() { + l.haltOnce.Do(func() { + close(l.halting) + <-l.done + var halts sync.WaitGroup + for _, cl := range l.owned { + halts.Go(cl.stop) + } + l.mu.Lock() + for _, cl := range l.releasing { + halts.Go(cl.stop) + } + l.mu.Unlock() + halts.Wait() + }) +} + +// stop ends the claims: every unit is halted, then released, and the +// membership lease resigned, at once. The worker halts the claims and +// flushes what it held before calling it, so each unit's next owner receives +// only rows newer than the ones written here. +func (l *claimLoop) stop() { + l.stopOnce.Do(func() { + l.stopped.Store(true) + l.cancel() + l.halt() + close(l.quit) + l.handovers.Wait() + // At once, so a broker that no longer answers costs one release's + // wait, not one per unit. + var releases sync.WaitGroup + for _, cl := range l.owned { + releases.Go(func() { l.release(cl) }) + claimEvents.Add(context.Background(), 1, eventAttr("released")) + } + releases.Wait() + clear(l.owned) + l.ownedGauge.Store(0) + if l.member != nil { + ctx, cancel := context.WithTimeout(context.Background(), releaseWait) + if err := l.member.Resign(ctx); err != nil { + slog.Warn("ingest: resign failed; the lease runs out on its own", "lease", l.member.Name(), "error", err) + } + cancel() + } + if l.gauges != nil { + _ = l.gauges.Unregister() + } + }) +} + +var ( + claimEvents, _ = otel.Meter("wavehouse-ingest").Int64Counter( + "wavehouse_ingest_shard_events_total", + metric.WithDescription("Shard claim events in this process: taken, released (handed on or stopped), reset (a dead owner's unsettled rows redelivered at takeover), handover_timeout (released before its rows settled), lost (the membership lease ended under it), bind_failed (a shard could not be bound; tried again next tick)"), + ) + handoverSeconds, _ = otel.Meter("wavehouse-ingest").Float64Histogram( + "wavehouse_ingest_shard_handover_seconds", + metric.WithDescription("Time from giving a shard up to releasing it: the drain of what it had delivered"), + metric.WithUnit("s"), + ) +) + +// claimMeter is the meter the claim gauges register on (tests substitute +// their own rather than the global provider, which binds only once). +var claimMeter = func() metric.Meter { return otel.Meter("wavehouse-ingest") } + +func eventAttr(event string) metric.AddOption { + return metric.WithAttributes(attribute.String("event", event)) +} + +// registerClaimGauges registers the gauges on the meter provider in place +// now: an observable made before the provider was set cannot be registered +// with one made after. +func registerClaimGauges(l *claimLoop) (metric.Registration, error) { + meter := claimMeter() + owned, err := meter.Int64ObservableGauge("wavehouse_ingest_shards_owned", + metric.WithDescription("Shards this process holds and consumes")) + if err != nil { + return nil, err + } + members, err := meter.Int64ObservableGauge("wavehouse_ingest_shard_members", + metric.WithDescription("Ingest processes this process last counted sharing the shards")) + if err != nil { + return nil, err + } + unowned, err := meter.Int64ObservableGauge("wavehouse_ingest_shards_unowned", + metric.WithDescription("Shards with rows waiting that no process holds, as the lowest-ranked member last counted them; the other members report nothing")) + if err != nil { + return nil, err + } + held, err := meter.Int64ObservableGauge("wavehouse_ingest_rows_held", + metric.WithDescription("Rows this process holds delivered and not yet acked or nacked, across its shards")) + if err != nil { + return nil, err + } + return meter.RegisterCallback(func(_ context.Context, o metric.Observer) error { + o.ObserveInt64(owned, l.ownedGauge.Load()) + o.ObserveInt64(members, l.membersGauge.Load()) + o.ObserveInt64(held, l.held.Load()) + if l.rankZero.Load() { + o.ObserveInt64(unowned, l.unownedGauge.Load()) + } + return nil + }, owned, members, unowned, held) +} diff --git a/internal/ingest/claims_test.go b/internal/ingest/claims_test.go new file mode 100644 index 000000000..8c0d9ef4f --- /dev/null +++ b/internal/ingest/claims_test.go @@ -0,0 +1,833 @@ +package ingest + +import ( + "context" + "errors" + "fmt" + "slices" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel/metric" + sdkmetric "go.opentelemetry.io/otel/sdk/metric" + "go.opentelemetry.io/otel/sdk/metric/metricdata" + + "github.com/Wave-RF/WaveHouse/internal/coord" + "github.com/Wave-RF/WaveHouse/internal/mq" +) + +// fakeShards is a sharded queue every test process shares: it records, per +// unit, which process consumes it and what each process did with it. +type fakeShards struct { + units, extra []string + + mu sync.Mutex + consumers map[string]map[string]*fakeUnitConsumer // unit → process → consumer + events []string // " " + unowned []string // the processes that asked + pinned map[string]bool // units a gone owner's pin still holds +} + +func newFakeShards(n int, extra ...string) *fakeShards { + f := &fakeShards{consumers: map[string]map[string]*fakeUnitConsumer{}, extra: extra} + for i := range n { + f.units = append(f.units, fmt.Sprintf("S/u-%02d", i)) + } + return f +} + +func (f *fakeShards) record(proc, verb, unit string) { + f.mu.Lock() + defer f.mu.Unlock() + f.events = append(f.events, proc+" "+verb+" "+unit) +} + +func (f *fakeShards) eventLog() []string { + f.mu.Lock() + defer f.mu.Unlock() + return slices.Clone(f.events) +} + +// owners maps each unit to the processes consuming it now. +func (f *fakeShards) owners() map[string][]string { + f.mu.Lock() + defer f.mu.Unlock() + out := map[string][]string{} + for u, by := range f.consumers { + for proc, c := range by { + if c.running.Load() { + out[u] = append(out[u], proc) + } + } + } + return out +} + +// split reports whether every unit has exactly one consumer, among procs, +// none of them above an even share. +func (f *fakeShards) split(procs ...string) bool { + owners := f.owners() + units := slices.Concat(f.units, f.extra) + limit := (len(units) + len(procs) - 1) / len(procs) + load := map[string]int{} + for _, u := range units { + if len(owners[u]) != 1 || !slices.Contains(procs, owners[u][0]) { + return false + } + load[owners[u][0]]++ + } + for _, n := range load { + if n > limit { + return false + } + } + return true +} + +// deliver hands one row to unit u's running consumer, if any, and returns +// whether one took it. It runs the handler on a goroutine of its own, as a +// delivery goroutine would; settled fires when the row is settled. +func (f *fakeShards) deliver(u string, settled func()) bool { + f.mu.Lock() + var c *fakeUnitConsumer + for _, cand := range f.consumers[u] { + if cand.running.Load() { + c = cand + } + } + f.mu.Unlock() + if c == nil { + return false + } + ack := func() error { + if settled != nil { + settled() + } + return nil + } + m := mq.NewMessage(context.Background(), mq.Topic{Tenant: "acme", Table: u}, []byte(u), time.Now(), + func(context.Context) error { return ack() }, ack, ack) + go c.handler(m) + return true +} + +// deliverFailing is deliver with every settle answered by err. +func (f *fakeShards) deliverFailing(u string, err error) bool { + f.mu.Lock() + var c *fakeUnitConsumer + for _, cand := range f.consumers[u] { + if cand.running.Load() { + c = cand + } + } + f.mu.Unlock() + if c == nil { + return false + } + fail := func() error { return err } + m := mq.NewMessage(context.Background(), mq.Topic{Tenant: "acme", Table: u}, []byte(u), time.Now(), + func(context.Context) error { return fail() }, fail, fail) + go c.handler(m) + return true +} + +// fakeProc is one process's view of the shared fake. +type fakeProc struct { + f *fakeShards + name string + // failCreate, when set, fails binding that unit. + failCreate atomic.Pointer[string] + // failWith, when set, is the error binding failCreate's unit returns. + failWith error + // extra, when set, is this process's own list of extras, as a process + // that last refreshed it at another time than the others would have. + extra []string +} + +func (p *fakeProc) IngestUnits() ([]string, []string) { + if p.extra != nil { + return p.f.units, p.extra + } + return p.f.units, p.f.extra +} + +func (p *fakeProc) ResetOrphaned(_ context.Context, unit string) (bool, error) { + p.f.mu.Lock() + held := p.f.pinned[unit] + p.f.mu.Unlock() + if held { + return false, mq.ErrUnitHeld + } + p.f.record(p.name, "reset", unit) + return true, nil +} + +func (p *fakeProc) Unowned(context.Context) (int, error) { + p.f.mu.Lock() + defer p.f.mu.Unlock() + p.f.unowned = append(p.f.unowned, p.name) + return 0, nil +} + +func (p *fakeProc) DeadLetter(context.Context, *mq.Message, ...mq.PublishOpt) error { return nil } + +func (p *fakeProc) CreateConsumer(_ context.Context, cfg mq.ConsumerConfig) (mq.Consumer, error) { + if len(cfg.Units) != 1 { + return &fakeUnitConsumer{}, nil // the boot check of every unit + } + u := cfg.Units[0] + if bad := p.failCreate.Load(); bad != nil && *bad == u { + if p.failWith != nil { + return nil, p.failWith + } + return nil, errors.New("no such durable") + } + p.f.record(p.name, "create", u) + return &fakeUnitConsumer{p: p, unit: u, failed: make(chan error, 1), maxHeld: cfg.MaxHeld}, nil +} + +type fakeUnitConsumer struct { + p *fakeProc + unit string + handler func(*mq.Message) + failed chan error + running atomic.Bool + maxHeld func() int +} + +func (c *fakeUnitConsumer) Consume(handler func(*mq.Message), _ int) (func(), <-chan error, error) { + c.handler = handler + c.running.Store(true) + c.p.f.mu.Lock() + if c.p.f.consumers[c.unit] == nil { + c.p.f.consumers[c.unit] = map[string]*fakeUnitConsumer{} + } + c.p.f.consumers[c.unit][c.p.name] = c + c.p.f.mu.Unlock() + return func() { + if c.running.CompareAndSwap(true, false) { + c.p.f.record(c.p.name, "stop", c.unit) + } + }, c.failed, nil +} + +func (c *fakeUnitConsumer) Release(context.Context) error { + c.p.f.record(c.p.name, "release", c.unit) + return nil +} + +// claimProc is one process's claims over the fake. +type claimProc struct { + *fakeProc + stop func() + failed <-chan error + loop *claimLoop +} + +func fastClaims(cfg ClaimConfig) ClaimConfig { + if cfg.Every == 0 { + cfg.Every = 20 * time.Millisecond + } + if cfg.Handover == 0 { + cfg.Handover = time.Second + } + if cfg.UnownedEvery == 0 { + cfg.UnownedEvery = time.Millisecond + } + return cfg +} + +func startClaims(t *testing.T, f *fakeShards, c *coord.Local, name string, cfg ClaimConfig, handler func(*mq.Message)) *claimProc { + t.Helper() + return startClaimsAs(t, &fakeProc{f: f, name: name}, c, cfg, handler) +} + +func startClaimsAs(t *testing.T, p *fakeProc, c *coord.Local, cfg ClaimConfig, handler func(*mq.Message)) *claimProc { + t.Helper() + q, err := ClaimShards(p, c, fastClaims(cfg)) + require.NoError(t, err) + cons, err := q.CreateConsumer(t.Context(), mq.ConsumerConfig{Durable: BufferConsumerName}) + require.NoError(t, err) + if handler == nil { + handler = func(m *mq.Message) { _ = m.Ack() } + } + stop, failed, err := cons.Consume(handler, 100) + require.NoError(t, err) + t.Cleanup(stop) + return &claimProc{fakeProc: p, stop: stop, failed: failed, loop: cons.(*claimingConsumer).loop.Load()} +} + +func noFailure(t *testing.T, procs ...*claimProc) { + t.Helper() + for _, p := range procs { + select { + case err := <-p.failed: + t.Fatalf("%s failed: %v", p.name, err) + default: + } + } +} + +func TestClaimShards_Refuses(t *testing.T) { + t.Parallel() + f := newFakeShards(2) + _, err := ClaimShards(&fakeProc{f: f}, noObserver{coord.NewLocal()}, ClaimConfig{}) + require.ErrorContains(t, err, "observes leases") + _, err = ClaimShards(plainQueue{}, coord.NewLocal(), ClaimConfig{}) + require.ErrorContains(t, err, "sharded queue") + q, err := ClaimShards(&fakeProc{f: f}, coord.NewLocal(), ClaimConfig{}) + require.NoError(t, err) + _, err = q.CreateConsumer(t.Context(), mq.ConsumerConfig{Units: []string{"S/u-00"}}) + require.ErrorContains(t, err, "chooses its own units") +} + +type noObserver struct{ coord.Coordinator } + +type plainQueue struct{} + +func (plainQueue) CreateConsumer(context.Context, mq.ConsumerConfig) (mq.Consumer, error) { + return nil, nil //nolint:nilnil // never called +} +func (plainQueue) DeadLetter(context.Context, *mq.Message, ...mq.PublishOpt) error { return nil } + +// Three processes split the units with none shared, none left over, and no +// process above an even share. +func TestClaims_SplitWithoutOverlap(t *testing.T) { + t.Parallel() + f, c := newFakeShards(12), coord.NewLocal() + a := startClaims(t, f, c, "a", ClaimConfig{}, nil) + b := startClaims(t, f, c.Peer(), "b", ClaimConfig{}, nil) + d := startClaims(t, f, c.Peer(), "c", ClaimConfig{}, nil) + require.Eventually(t, func() bool { return f.split("a", "b", "c") }, 5*time.Second, 10*time.Millisecond, "%v", f.owners()) + noFailure(t, a, b, d) +} + +// A process joining takes its share, handed over by the others; one that +// stops releases its units at once, and the others take them. +func TestClaims_JoinAndStopHandOver(t *testing.T) { + t.Parallel() + f, c := newFakeShards(12), coord.NewLocal() + a := startClaims(t, f, c, "a", ClaimConfig{}, nil) + require.Eventually(t, func() bool { return f.split("a") }, 5*time.Second, 10*time.Millisecond) + b := startClaims(t, f, c.Peer(), "b", ClaimConfig{}, nil) + require.Eventually(t, func() bool { return f.split("a", "b") }, 5*time.Second, 10*time.Millisecond, "%v", f.owners()) + // Every unit b took, a gave up (stopped, then released): a handover, + // not two owners. b does not wait for the release; the pin keeps a unit + // exclusive meanwhile. + require.Eventually(t, func() bool { + log := f.eventLog() + for _, e := range log { + if unit, ok := strings.CutPrefix(e, "b create "); ok && !slices.Contains(log, "a release "+unit) { + return false + } + } + return true + }, 5*time.Second, 10*time.Millisecond, "a released every unit b took: %v", f.eventLog()) + b.stop() + require.Eventually(t, func() bool { return f.split("a") }, 5*time.Second, 10*time.Millisecond) + for _, e := range f.eventLog() { + if strings.HasPrefix(e, "b create ") { + assert.Contains(t, f.eventLog(), "b release "+strings.TrimPrefix(e, "b create ")) + } + } + noFailure(t, a) +} + +// A unit given up waits for the rows it delivered to settle before it is +// released, bounded by Handover. +func TestClaims_HandoverWaitsForDeliveredRows(t *testing.T) { + t.Parallel() + f, c := newFakeShards(4), coord.NewLocal() + var pending sync.WaitGroup + release := make(chan struct{}) + a := startClaims(t, f, c, "a", ClaimConfig{Handover: 5 * time.Second}, func(m *mq.Message) { + go func() { <-release; _ = m.Ack(); pending.Done() }() + }) + require.Eventually(t, func() bool { return f.split("a") }, 5*time.Second, 10*time.Millisecond) + for _, u := range f.units { + pending.Add(1) + require.True(t, f.deliver(u, nil)) + } + require.Eventually(t, func() bool { return a.loop.held.Load() == 4 }, 5*time.Second, 10*time.Millisecond) + startClaims(t, f, c.Peer(), "b", ClaimConfig{}, nil) + require.Eventually(t, func() bool { + return slices.ContainsFunc(f.eventLog(), func(e string) bool { return strings.HasPrefix(e, "a stop ") }) + }, + 5*time.Second, 10*time.Millisecond, "a gives some units up") + time.Sleep(300 * time.Millisecond) + assert.False(t, slices.ContainsFunc(f.eventLog(), func(e string) bool { return strings.HasPrefix(e, "a release ") }), + "no release while its rows are unsettled: %v", f.eventLog()) + close(release) + pending.Wait() + require.Eventually(t, func() bool { return f.split("a", "b") }, 5*time.Second, 10*time.Millisecond) + assert.Zero(t, a.loop.held.Load(), "every settled row gave its slot back") +} + +// A process that loses its membership lease gives every unit up. +func TestClaims_LostMembershipGivesEveryUnitUp(t *testing.T) { + t.Parallel() + f, c := newFakeShards(4), coord.NewLocal() + a := startClaims(t, f, c, "a", ClaimConfig{}, nil) + require.Eventually(t, func() bool { return f.split("a") }, 5*time.Second, 10*time.Millisecond) + // The lease ends under a (the in-process backend never expires one). + require.NoError(t, c.Close(t.Context())) + require.Eventually(t, func() bool { return len(f.owners()) == 0 }, 5*time.Second, 10*time.Millisecond, "%v", f.owners()) + noFailure(t, a) +} + +// More processes than units: the extra ones hold nothing, and fail nothing. +func TestClaims_MoreProcessesThanUnits(t *testing.T) { + t.Parallel() + f, c := newFakeShards(2), coord.NewLocal() + a := startClaims(t, f, c, "a", ClaimConfig{}, nil) + b := startClaims(t, f, c.Peer(), "b", ClaimConfig{}, nil) + d := startClaims(t, f, c.Peer(), "c", ClaimConfig{}, nil) + require.Eventually(t, func() bool { + o := f.owners() + return len(o) == 2 && len(o["S/u-00"]) == 1 && len(o["S/u-01"]) == 1 + }, 5*time.Second, 10*time.Millisecond) + time.Sleep(100 * time.Millisecond) + held := 0 + for _, p := range []*claimProc{a, b, d} { + held += int(p.loop.ownedGauge.Load()) + } + assert.Equal(t, 2, held) + noFailure(t, a, b, d) +} + +// crash stops p's claims the way a killed process ends: no release, and its +// membership lease gone (the in-process backend never expires one). +func crash(t *testing.T, p *claimProc) { + t.Helper() + l := p.loop + l.stopOnce.Do(func() { // stop is then a no-op + l.stopped.Store(true) + l.cancel() + close(l.halting) + close(l.quit) + <-l.done + require.NoError(t, l.member.Resign(context.Background())) + for _, cl := range l.owned { + cl.stop() // its fake consumers stop being chosen; nothing is released + } + }) +} + +// A unit whose owner died is reset before its new owner binds it, so the +// dead owner's unsettled rows come back at once; a handover to a joining +// process resets nothing. +func TestClaims_TakeoverResetsBeforeBinding(t *testing.T) { + t.Parallel() + f, c := newFakeShards(8), coord.NewLocal() + a := startClaims(t, f, c, "a", ClaimConfig{}, nil) + require.Eventually(t, func() bool { return f.split("a") }, 5*time.Second, 10*time.Millisecond) + b := startClaims(t, f, c.Peer(), "b", ClaimConfig{}, nil) + require.Eventually(t, func() bool { return f.split("a", "b") }, 5*time.Second, 10*time.Millisecond) + assert.False(t, slices.ContainsFunc(f.eventLog(), func(e string) bool { return strings.HasPrefix(e, "b reset ") }), + "a handover is no takeover: %v", f.eventLog()) + + var aUnits []string + for u, owners := range f.owners() { + if owners[0] == "a" { + aUnits = append(aUnits, u) + } + } + crash(t, a) + require.Eventually(t, func() bool { + for _, u := range aUnits { + if !slices.Contains(f.eventLog(), "b create "+u) || !slices.Contains(f.eventLog(), "b reset "+u) { + return false + } + } + return true + }, 5*time.Second, 10*time.Millisecond, "%v", f.eventLog()) + log := f.eventLog() + for _, u := range aUnits { + reset := slices.Index(log, "b reset "+u) + bound := -1 + for i, e := range log { + if e == "b create "+u { + bound = i + } + } + assert.Less(t, reset, bound, "%s reset before bound", u) + } + noFailure(t, b) +} + +// A configured unit that cannot be bound, or whose delivery ends, fails the +// claims; an extra one's only ends that unit. +func TestClaims_EndedDelivery(t *testing.T) { + t.Parallel() + t.Run("extra", func(t *testing.T) { + t.Parallel() + f := newFakeShards(2, "X/u-09") + p := startClaims(t, f, coord.NewLocal(), "a", ClaimConfig{}, nil) + require.Eventually(t, func() bool { return f.split("a") }, 5*time.Second, 10*time.Millisecond) + f.mu.Lock() + f.consumers["X/u-09"]["a"].failed <- errors.New("deleted") + f.mu.Unlock() + time.Sleep(100 * time.Millisecond) + noFailure(t, p) + }) + t.Run("configured", func(t *testing.T) { + t.Parallel() + f := newFakeShards(2) + p := startClaims(t, f, coord.NewLocal(), "a", ClaimConfig{}, nil) + require.Eventually(t, func() bool { return f.split("a") }, 5*time.Second, 10*time.Millisecond) + f.mu.Lock() + f.consumers["S/u-01"]["a"].failed <- errors.New("deleted") + f.mu.Unlock() + select { + case err := <-p.failed: + assert.ErrorContains(t, err, "S/u-01") + case <-time.After(5 * time.Second): + t.Fatal("a configured unit's delivery ending did not fail the claims") + } + }) + t.Run("unbound, then bound", func(t *testing.T) { + t.Parallel() + f := newFakeShards(2) + bad := "S/u-01" + p := &fakeProc{f: f, name: "a"} + p.failCreate.Store(&bad) + q, err := ClaimShards(p, coord.NewLocal(), fastClaims(ClaimConfig{})) + require.NoError(t, err) + cons, err := q.CreateConsumer(t.Context(), mq.ConsumerConfig{}) + require.NoError(t, err) + stop, failed, err := cons.Consume(func(*mq.Message) {}, 10) + require.NoError(t, err) + t.Cleanup(stop) + require.Eventually(t, func() bool { return len(f.owners()["S/u-00"]) == 1 }, 5*time.Second, 10*time.Millisecond) + time.Sleep(100 * time.Millisecond) // a few ticks of failed binds + select { + case err := <-failed: + t.Fatalf("a bind that failed on something other than a missing durable failed the claims: %v", err) + default: + } + assert.Empty(t, f.owners()[bad]) + p.failCreate.Store(nil) + require.Eventually(t, func() bool { return f.split("a") }, 5*time.Second, 10*time.Millisecond, "bound on a later tick") + }) +} + +// Each unit's cap on rows delivered and unsettled is an even share of +// MaxHeld over the units the process is assigned, never below two batches, +// so one stuck unit cannot take the others' room; it follows the assignment. +func TestClaims_UnitShareOfMaxHeld(t *testing.T) { + t.Parallel() + assert.Equal(t, 5000, unitShare(10_000, 2)) + assert.Equal(t, minUnitHeld, unitShare(10_000, 32), "never below two batches") + assert.Equal(t, 64, unitShare(64, 32), "nor above MaxHeld") + + f, c := newFakeShards(8), coord.NewLocal() + a := startClaims(t, f, c, "a", ClaimConfig{MaxHeld: 8000}, nil) + require.Eventually(t, func() bool { return f.split("a") }, 5*time.Second, 10*time.Millisecond) + caps := func(proc string) map[int]int { + f.mu.Lock() + defer f.mu.Unlock() + out := map[int]int{} + for _, by := range f.consumers { + if cons, ok := by[proc]; ok && cons.running.Load() { + out[cons.maxHeld()]++ + } + } + return out + } + assert.Equal(t, map[int]int{1000: 8}, caps("a"), "8000 over 8 units") + startClaims(t, f, c.Peer(), "b", ClaimConfig{MaxHeld: 8000}, nil) + require.Eventually(t, func() bool { return f.split("a", "b") }, 5*time.Second, 10*time.Millisecond) + require.Eventually(t, func() bool { return a.loop.share.Load() == 2000 }, 5*time.Second, 10*time.Millisecond) + assert.Equal(t, map[int]int{2000: 4}, caps("a"), "the units kept read the new share") +} + +// Only the lowest live member counts the units without an owner. +func TestClaims_UnownedCountedByTheLowestMember(t *testing.T) { + t.Parallel() + f, c := newFakeShards(4), coord.NewLocal() + a := startClaims(t, f, c, "a", ClaimConfig{}, nil) + require.Eventually(t, func() bool { return f.split("a") }, 5*time.Second, 10*time.Millisecond) + startClaims(t, f, c.Peer(), "b", ClaimConfig{}, nil) + require.Eventually(t, func() bool { return f.split("a", "b") }, 5*time.Second, 10*time.Millisecond) + f.mu.Lock() + f.unowned = nil + f.mu.Unlock() + time.Sleep(200 * time.Millisecond) + f.mu.Lock() + defer f.mu.Unlock() + assert.NotEmpty(t, f.unowned) + for _, who := range f.unowned { + assert.Equal(t, "a", who) + } + assert.True(t, a.loop.rankZero.Load()) +} + +// A unit taken over while the dead owner's pin still holds waits, unbound, +// until the pin lapses; then it is reset and bound. +func TestClaims_TakeoverWaitsForTheDeadOwnersPin(t *testing.T) { + t.Parallel() + f, c := newFakeShards(2), coord.NewLocal() + a := startClaims(t, f, c, "a", ClaimConfig{}, nil) + require.Eventually(t, func() bool { return f.split("a") }, 5*time.Second, 10*time.Millisecond) + b := startClaims(t, f, c.Peer(), "b", ClaimConfig{}, nil) + require.Eventually(t, func() bool { return f.split("a", "b") }, 5*time.Second, 10*time.Millisecond) + var aUnit string + for u, owners := range f.owners() { + if owners[0] == "a" { + aUnit = u + } + } + f.mu.Lock() + f.pinned = map[string]bool{aUnit: true} + f.mu.Unlock() + crash(t, a) + time.Sleep(300 * time.Millisecond) + assert.NotContains(t, f.eventLog(), "b create "+aUnit, "not bound while the pin holds") + f.mu.Lock() + f.pinned = nil + f.mu.Unlock() + require.Eventually(t, func() bool { return slices.Contains(f.eventLog(), "b create "+aUnit) }, 5*time.Second, 10*time.Millisecond) + assert.Contains(t, f.eventLog(), "b reset "+aUnit) + noFailure(t, b) +} + +// The gauges register on the meter provider in place when the claims start, +// which the process sets after this package's instruments are made. +func TestClaims_GaugesReport(t *testing.T) { //nolint:paralleltest // substitutes the package's claim meter + reader := sdkmetric.NewManualReader() + prev := claimMeter + claimMeter = func() metric.Meter { return sdkmetric.NewMeterProvider(sdkmetric.WithReader(reader)).Meter("test") } + t.Cleanup(func() { claimMeter = prev }) + + f := newFakeShards(4) + startClaims(t, f, coord.NewLocal(), "a", ClaimConfig{}, nil) + require.Eventually(t, func() bool { return f.split("a") }, 5*time.Second, 10*time.Millisecond) + got := map[string]int64{} + require.Eventually(t, func() bool { + var rm metricdata.ResourceMetrics + require.NoError(t, reader.Collect(t.Context(), &rm)) + for _, sm := range rm.ScopeMetrics { + for _, m := range sm.Metrics { + if g, ok := m.Data.(metricdata.Gauge[int64]); ok && len(g.DataPoints) > 0 { + got[m.Name] = g.DataPoints[0].Value + } + } + } + return got["wavehouse_ingest_shards_owned"] == 4 + }, 5*time.Second, 20*time.Millisecond, "%v", got) + assert.Equal(t, int64(1), got["wavehouse_ingest_shard_members"]) + assert.Contains(t, got, "wavehouse_ingest_shards_unowned", "the lowest member reports it") + assert.Contains(t, got, "wavehouse_ingest_rows_held") +} + +// A row the worker lets go of without the broker confirming it (an ack that +// failed) is no longer counted as held: the broker redelivers it as a new +// row. +func TestClaims_HeldRowsFreedOnAFailedSettle(t *testing.T) { + t.Parallel() + f := newFakeShards(2) + fail := errors.New("no answer") + var received atomic.Int64 + p := startClaims(t, f, coord.NewLocal(), "a", ClaimConfig{}, func(m *mq.Message) { + received.Add(1) + _ = m.Ack() + }) + require.Eventually(t, func() bool { return f.split("a") }, 5*time.Second, 10*time.Millisecond) + for i := range 10 { + require.True(t, f.deliverFailing(f.units[i%2], fail)) + } + require.Eventually(t, func() bool { return received.Load() == 10 }, 5*time.Second, 10*time.Millisecond) + require.Eventually(t, func() bool { return p.loop.held.Load() == 0 }, 5*time.Second, 10*time.Millisecond) +} + +// A row the worker never settles (it leaves some for the broker to +// redeliver) stops counting as held once the broker would redeliver it +// anyway, so a handover does not wait on it past that. +func TestClaims_HeldRowsExpire(t *testing.T) { + t.Parallel() + f := newFakeShards(1) + var received atomic.Int64 + p := startClaims(t, f, coord.NewLocal(), "a", ClaimConfig{SlotExpiry: 200 * time.Millisecond}, func(*mq.Message) { + received.Add(1) // never settled + }) + require.Eventually(t, func() bool { return f.split("a") }, 5*time.Second, 10*time.Millisecond) + for range 4 { + require.True(t, f.deliver(f.units[0], nil)) + } + require.Eventually(t, func() bool { return received.Load() == 4 && p.loop.held.Load() == 4 }, 5*time.Second, 10*time.Millisecond) + require.Eventually(t, func() bool { return p.loop.held.Load() == 0 }, 5*time.Second, 10*time.Millisecond) +} + +// A process whose membership lease ends under it takes a fresh one on its +// next tick and keeps consuming. +func TestClaims_RejoinsAfterLosingItsLease(t *testing.T) { + t.Parallel() + f, c := newFakeShards(4), coord.NewLocal() + a := startClaims(t, f, c, "a", ClaimConfig{}, nil) + var first coord.Term + require.Eventually(t, func() bool { + first = a.loop.memberTerm() + return f.split("a") && first != nil + }, 5*time.Second, 10*time.Millisecond) + // The lease ends under a, as a backend reporting it lost would. + require.NoError(t, first.Resign(t.Context())) + require.Eventually(t, func() bool { m := a.loop.memberTerm(); return m != nil && m != first }, 5*time.Second, 10*time.Millisecond, "a joined again") + assert.Greater(t, a.loop.memberTerm().Token(), first.Token()) + require.Eventually(t, func() bool { return f.split("a") }, 5*time.Second, 10*time.Millisecond) + noFailure(t, a) +} + +// The only live process, on its first tick, resets the units nobody holds: +// a restart of the only ingest process gets its predecessor's unsettled +// rows back at once. With another member live, it does not. +func TestClaims_FirstTickResetsOnlyWhenAlone(t *testing.T) { + t.Parallel() + f := newFakeShards(3) + startClaims(t, f, coord.NewLocal(), "a", ClaimConfig{}, nil) + require.Eventually(t, func() bool { return f.split("a") }, 5*time.Second, 10*time.Millisecond) + for _, u := range f.units { + assert.Contains(t, f.eventLog(), "a reset "+u) + } + + g, c := newFakeShards(3), coord.NewLocal() + startClaims(t, g, c, "b", ClaimConfig{}, nil) + require.Eventually(t, func() bool { return g.split("b") }, 5*time.Second, 10*time.Millisecond) + d := startClaims(t, g, c.Peer(), "d", ClaimConfig{}, nil) + require.Eventually(t, func() bool { return g.split("b", "d") }, 5*time.Second, 10*time.Millisecond) + assert.False(t, slices.ContainsFunc(g.eventLog(), func(e string) bool { return strings.HasPrefix(e, "d reset ") }), + "a process joining a live member resets nothing: %v", g.eventLog()) + noFailure(t, d) +} + +// stop cuts a handover short: it does not wait out Handover for rows the +// worker will never settle. +func TestClaims_StopCutsHandoversShort(t *testing.T) { + t.Parallel() + f, c := newFakeShards(4), coord.NewLocal() + var received atomic.Int64 + a := startClaims(t, f, c, "a", ClaimConfig{Handover: time.Minute}, func(*mq.Message) { received.Add(1) }) + require.Eventually(t, func() bool { return f.split("a") }, 5*time.Second, 10*time.Millisecond) + for i := range 4 { + require.True(t, f.deliver(f.units[i%4], nil)) + } + require.Eventually(t, func() bool { return received.Load() == 4 }, 5*time.Second, 10*time.Millisecond) + startClaims(t, f, c.Peer(), "b", ClaimConfig{}, nil) // a gives half its units up, unsettled + require.Eventually(t, func() bool { + return slices.ContainsFunc(f.eventLog(), func(e string) bool { return strings.HasPrefix(e, "a stop ") }) + }, + 5*time.Second, 10*time.Millisecond) + stopped := make(chan struct{}) + go func() { a.stop(); close(stopped) }() + select { + case <-stopped: + case <-time.After(10 * time.Second): + t.Fatal("stop waited out the handover") + } +} + +// Membership slots are capped: a queue of many units still reads at most +// maxMembers leases a tick. +func TestClaims_MembershipSlotsCapped(t *testing.T) { + t.Parallel() + f := newFakeShards(3 * maxMembers) + p := startClaims(t, f, coord.NewLocal(), "a", ClaimConfig{}, nil) + require.Eventually(t, func() bool { return f.split("a") }, 10*time.Second, 10*time.Millisecond) + assert.Equal(t, maxMembers, p.loop.slots) +} + +// A configured unit whose durable (or its stream) is gone, or no longer fits +// the worker's config, when it is bound ends delivery, the same end as the +// broker ending a durable deleted under it: no later tick can bind it. +func TestClaims_MissingDurableEndsDelivery(t *testing.T) { + t.Parallel() + for _, cause := range []error{mq.ErrConsumerNotFound, mq.ErrConsumerMismatch} { + t.Run(cause.Error(), func(t *testing.T) { + t.Parallel() + f := newFakeShards(2) + bad := "S/u-01" + p := &fakeProc{f: f, name: "a", failWith: fmt.Errorf("consumer %s: %w", bad, cause)} + p.failCreate.Store(&bad) + q, err := ClaimShards(p, coord.NewLocal(), fastClaims(ClaimConfig{})) + require.NoError(t, err) + cons, err := q.CreateConsumer(t.Context(), mq.ConsumerConfig{}) + require.NoError(t, err) + stop, failed, err := cons.Consume(func(*mq.Message) {}, 10) + require.NoError(t, err) + t.Cleanup(stop) + select { + case err := <-failed: + assert.ErrorIs(t, err, mq.ErrDeliveryEnded) + assert.ErrorIs(t, err, cause) + case <-time.After(5 * time.Second): + t.Fatal("an unbindable durable did not end delivery") + } + }) + } +} + +// Processes whose lists of extras differ (each refreshes it on its own +// timer) still agree on every configured unit: each has exactly one owner, +// and nothing is handed back and forth. +func TestClaims_DifferingExtrasLeaveConfiguredUnitsAlone(t *testing.T) { + t.Parallel() + f, c := newFakeShards(32), coord.NewLocal() + procs := []*claimProc{startClaimsAs(t, &fakeProc{f: f, name: "p0", extra: []string{"X/a", "X/b", "X/c"}}, c, ClaimConfig{}, nil)} + for i := 1; i < 5; i++ { + procs = append(procs, startClaims(t, f, c.Peer(), fmt.Sprintf("p%d", i), ClaimConfig{}, nil)) + } + oneEach := func() bool { + owners := f.owners() + for _, u := range f.units { + if len(owners[u]) != 1 { + return false + } + } + return true + } + // Settled once every process has joined and holds a share, not at the + // first moment each unit has one owner while members still arrive. + settledSplit := func() bool { + if !oneEach() { + return false + } + holders := map[string]bool{} + for _, u := range f.units { + holders[f.owners()[u][0]] = true + } + return len(holders) == len(procs) + } + require.Eventually(t, settledSplit, 5*time.Second, 10*time.Millisecond, "%v", f.owners()) + before := f.eventLog() + time.Sleep(300 * time.Millisecond) // several ticks + assert.True(t, oneEach(), "%v", f.owners()) + // A handover's release follows its stop once the rows settle, so one + // stopped before settling may be released after. + stopped := map[string]bool{} + for _, e := range before { + if proc, unit, ok := strings.Cut(e, " stop "); ok { + stopped[proc+" release "+unit] = true + } + } + for _, e := range f.eventLog()[len(before):] { + if stopped[e] { + delete(stopped, e) + continue + } + assert.NotContains(t, e, " S/u-", "a configured unit moved once settled: %s", e) + } + noFailure(t, procs...) +} + +// The owned gauge reads the tick's own result, not the count it began with. +func TestClaims_OwnedGaugeCurrentAfterOneTick(t *testing.T) { + t.Parallel() + f := newFakeShards(4) + p := startClaims(t, f, coord.NewLocal(), "a", ClaimConfig{Every: time.Hour}, nil) + require.Eventually(t, func() bool { return f.split("a") }, 5*time.Second, 10*time.Millisecond, "the first tick takes every unit") + require.Eventually(t, func() bool { return p.loop.ownedGauge.Load() == 4 }, time.Second, 10*time.Millisecond, "after the first tick alone") +} diff --git a/internal/ingest/worker.go b/internal/ingest/worker.go index 5ec26ac6a..523eb1c85 100644 --- a/internal/ingest/worker.go +++ b/internal/ingest/worker.go @@ -246,23 +246,29 @@ func waitOrDeadline(ctx context.Context, wg *sync.WaitGroup) error { // so a low-volume table can never strand another table's rows behind a shared // timer. It is the ONLY goroutine that watches ctx; tableLoops stop via // channel-close, which gives a deterministic drain with no abandoned messages. +// +// Stopping keeps each table's rows in order across the processes that share a +// sharded queue: the consumer is halted first (mq.Halter), keeping its hold on +// its units, and what it still delivers is routed like any other row; then +// every table flushes; only then does the consumer's stop give the units up. +// A delivery that arrives once this loop reads no more is NAKed at once, +// before that stop, so it comes back ahead of anything newer rather than +// after ack_wait. func (w *IngestWorker) dispatchLoop(ctx context.Context, cons mq.Consumer) { defer w.wg.Done() msgChan := make(chan *mq.Message, w.maxBatch*2) + abandon := make(chan struct{}) // Pull consumer with a push-like callback (the client prefetches pullMaxMessages, // shared by the tenants' queues). It runs on one delivery goroutine per tenant, // so the handoff is a channel send, safe from all of them at once. Hand off to // msgChan only, so a consume goroutine never blocks on flush work. - // The handoff also watches ctx: stop (deferred below) does not wait for a - // delivery already in the handler, so once this loop has stopped draining - // msgChan a full channel would otherwise pin a delivery goroutine - // forever. A message dropped here is unacked and simply redelivered. stop, deliveryEnded, err := cons.Consume(func(msg *mq.Message) { select { case msgChan <- msg: - case <-ctx.Done(): + case <-abandon: + nakAbandoned(msg) } }, pullMaxMessages) if err != nil { @@ -270,7 +276,24 @@ func (w *IngestWorker) dispatchLoop(ctx context.Context, cons mq.Consumer) { w.failed <- fmt.Errorf("ingest worker: start consumer: %w", err) return } - defer stop() + nakLeft := func() { + for { + select { + case m := <-msgChan: + nakAbandoned(m) + default: + return + } + } + } + defer func() { + close(abandon) + nakLeft() + stop() + // A consumer that cannot halt may still have been handing one + // over; it has no units to give up, so after its stop is as good. + nakLeft() + }() // flushCtx carries values (trace) but is never cancelled: a started flush must // finish so data already in ClickHouse gets acked rather than redelivered. It @@ -302,52 +325,89 @@ func (w *IngestWorker) dispatchLoop(ctx context.Context, cons mq.Consumer) { w.ackWg.Wait() } + // route hands a message to its table's loop. A tableLoop only blocks in + // its final drain, which starts after the last route, so the send waits + // at most for the loop to take the rows ahead of it. + route := func(m *mq.Message) { + pm, ok := w.parseMsg(flushCtx, m) + if !ok { + return // unreadable envelope: parked on the DLQ (or acked-and-dropped) in parseMsg + } + key := batchKey{tenant: pm.tenant, table: pm.tableName} + ch, exists := tableChans[key] + if !exists { + ch = make(chan parsedMsg, w.maxBatch) + tableChans[key] = ch + + // TODO(#263): tableLoops are spawned per distinct tenant table and + // never reaped — they live for the process lifetime. Safe while + // tenants and table names are bounded (a settings folder per + // tenant, schema-validated tables, in-process publishers only, + // DontListen:true). Add idle-reaping + route/teardown coordination + // before remote/untrusted publishers can create unbounded cardinality. + table, in := pm.tableName, ch + tableWg.Go(func() { w.tableLoop(flushCtx, table, in) }) + } + ch <- pm + } + for { select { case <-ctx.Done(): + haltRouting(cons, msgChan, route) shutdown() return case err := <-deliveryEnded: // The MQ gave up on the consumer: no message will arrive again, so // waiting on msgChan would stall ingestion silently. Flush and ack // what is already in hand — those rows are delivered and the - // publish path is not what broke — then fail loud. Messages still - // in msgChan are unacked and redelivered to the next consumer. + // publish path is not what broke — then fail loud. slog.ErrorContext(ctx, "ingest consumer delivery ended; ingestion has stopped", "error", err) + haltRouting(cons, msgChan, route) shutdown() w.failed <- fmt.Errorf("ingest worker: %w", err) return case m := <-msgChan: - pm, ok := w.parseMsg(flushCtx, m) - if !ok { - continue // unreadable envelope: parked on the DLQ (or acked-and-dropped) in parseMsg - } - key := batchKey{tenant: pm.tenant, table: pm.tableName} - ch, exists := tableChans[key] - if !exists { - ch = make(chan parsedMsg, w.maxBatch) - tableChans[key] = ch - - // TODO(#263): tableLoops are spawned per distinct tenant table and - // never reaped — they live for the process lifetime. Safe while - // tenants and table names are bounded (a settings folder per - // tenant, schema-validated tables, in-process publishers only, - // DontListen:true). Add idle-reaping + route/teardown coordination - // before remote/untrusted publishers can create unbounded cardinality. - table, in := pm.tableName, ch - tableWg.Go(func() { w.tableLoop(flushCtx, table, in) }) - } - // Route to the table's loop, but stay responsive to shutdown if its - // channel is full (a busy tableLoop must not wedge teardown). A pm - // dropped here is unacked and simply redelivered. + route(m) + } + } +} + +// haltRouting halts cons, if it can halt ahead of its stop, routing what it +// still delivers until it has, then routes what msgChan still holds. +func haltRouting(cons mq.Consumer, msgChan <-chan *mq.Message, route func(*mq.Message)) { + if h, ok := cons.(mq.Halter); ok { + halted := make(chan struct{}) + go func() { + h.Halt() + close(halted) + }() + halting: + for { select { - case ch <- pm: - case <-ctx.Done(): - shutdown() - return + case m := <-msgChan: + route(m) + case <-halted: + break halting } } } + for { + select { + case m := <-msgChan: + route(m) + default: + return + } + } +} + +// nakAbandoned hands back a delivery the worker will not write: redelivered +// at once, to whichever process owns its unit next. +func nakAbandoned(m *mq.Message) { + if err := m.Nak(); err != nil { + slog.Debug("ingest: could not hand back an undelivered row; it comes back after the ack wait", "error", err) + } } // tableBatcher accumulates one table's rows and flushes them to ClickHouse. diff --git a/internal/ingest/worker_test.go b/internal/ingest/worker_test.go index 42251f0da..26a6f7479 100644 --- a/internal/ingest/worker_test.go +++ b/internal/ingest/worker_test.go @@ -2471,3 +2471,123 @@ func TestTableBatcher_Add_HandsRowsBackWhileTheProbeIsOut(t *testing.T) { assert.True(t, m.Naked.Load()) assert.GreaterOrEqual(t, time.Duration(m.NakDelay.Load()), retryBase/2) } + +// scriptedConsumer hands the worker's handler to the test and records, in +// order, what became of each row and when the consumer was stopped. +type scriptedConsumer struct { + mu sync.Mutex + handler func(*mq.Message) + events []string +} + +func (c *scriptedConsumer) Consume(h func(*mq.Message), _ int) (func(), <-chan error, error) { + c.mu.Lock() + c.handler = h + c.mu.Unlock() + return func() { c.record("stop") }, make(chan error), nil +} + +func (c *scriptedConsumer) record(e string) { + c.mu.Lock() + defer c.mu.Unlock() + c.events = append(c.events, e) +} + +func (c *scriptedConsumer) log() []string { + c.mu.Lock() + defer c.mu.Unlock() + return slices.Clone(c.events) +} + +func (c *scriptedConsumer) deliver(t *testing.T, id string) { + t.Helper() + msg := mq.NewMessage(context.Background(), mq.Topic{Tenant: tenant.Default, Table: "events"}, + makeEnvelope(t, "events", "", map[string]any{"id": id}), time.Now(), + func(context.Context) error { c.record("ack " + id); return nil }, + func() error { c.record("ack " + id); return nil }, + func() error { c.record("nak " + id); return nil }) + c.mu.Lock() + h := c.handler + c.mu.Unlock() + h(msg) +} + +// haltingConsumer is a scriptedConsumer that halts ahead of its stop, and +// delivers one more row while halting, as a fetch in flight would. +type haltingConsumer struct { + *scriptedConsumer + t *testing.T +} + +func (c haltingConsumer) Halt() { + c.record("halt") + c.deliver(c.t, "late") +} + +// scriptedWorker runs a worker over cons whose ClickHouse records each +// insert on cons and then waits for gate, if set. +func scriptedWorker(t *testing.T, cons mq.Consumer, rec func(string), gate func()) (cancel func(), done <-chan struct{}) { + t.Helper() + ch := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(r.Body) + rec(fmt.Sprintf("insert %d", strings.Count(string(body), "\n"))) + if gate != nil { + gate() + } + w.WriteHeader(http.StatusOK) + })) + t.Cleanup(ch.Close) + w := &IngestWorker{ + clients: chconn.NewHTTPClients(ingestHTTPClient), + cache: &testutil.MockCache{}, + target: func(tenant.ID) chconn.Target { return chconn.Target{URL: ch.URL, Database: "db"} }, + // Flushed only at the stop. + maxBatch: defaultMaxBatch, + maxWait: time.Hour, + } + ctx, stop := context.WithCancel(context.Background()) + finished := make(chan struct{}) + w.wg.Add(1) + go func() { w.dispatchLoop(ctx, cons); close(finished) }() + t.Cleanup(func() { stop(); <-finished }) + return stop, finished +} + +// Stopping halts the consumer before the final flush, writes what it still +// delivered while halting, and stops it — giving its units up — only after. +func TestIngestWorker_StopHaltsBeforeTheFlushAndStopsAfter(t *testing.T) { + t.Parallel() + cons := haltingConsumer{scriptedConsumer: &scriptedConsumer{}, t: t} + cancel, done := scriptedWorker(t, cons, cons.record, nil) + require.Eventually(t, func() bool { cons.mu.Lock(); defer cons.mu.Unlock(); return cons.handler != nil }, 5*time.Second, time.Millisecond) + cons.deliver(t, "a") + cons.deliver(t, "b") + cancel() + <-done + log := cons.log() + require.Len(t, log, 6, "%v", log) + require.Equal(t, "halt", log[0], "%v", log) + assert.Equal(t, "insert 3", log[1], "the row delivered while halting is written with the rest: %v", log) + assert.ElementsMatch(t, []string{"ack a", "ack b", "ack late"}, log[2:5]) + assert.Equal(t, []string{"stop"}, log[5:], "stopped only after the flush: %v", log) +} + +// A consumer that cannot halt may still deliver while the worker flushes: +// those rows are NAKed, in the order they arrived, before its stop. +func TestIngestWorker_StopNaksWhatArrivesAfterItStopsReading(t *testing.T) { + t.Parallel() + cons := &scriptedConsumer{} + flushing, release := make(chan struct{}), make(chan struct{}) + var once sync.Once + cancel, done := scriptedWorker(t, cons, cons.record, func() { once.Do(func() { close(flushing) }); <-release }) + require.Eventually(t, func() bool { cons.mu.Lock(); defer cons.mu.Unlock(); return cons.handler != nil }, 5*time.Second, time.Millisecond) + cons.deliver(t, "a") + cancel() + <-flushing + for _, id := range []string{"x", "y", "z"} { + cons.deliver(t, id) + } + close(release) + <-done + assert.Equal(t, []string{"insert 1", "ack a", "nak x", "nak y", "nak z", "stop"}, cons.log()) +} diff --git a/internal/mq/embedded.go b/internal/mq/embedded.go index 72cb40625..6ee93f072 100644 --- a/internal/mq/embedded.go +++ b/internal/mq/embedded.go @@ -108,7 +108,7 @@ type tenantQueue struct { ingestCap int64 } -// EmbeddedNATS is the one implementation of every mq interface. +// EmbeddedNATS implements every mq interface. var _ Broker = (*EmbeddedNATS)(nil) const ( @@ -734,8 +734,12 @@ func (e *EmbeddedNATS) Subscribe(ctx context.Context, consumerName string, handl // every tenant's queue, and joins each queue opened later — one it cannot // join is that tenant's failure (SetMaxBytes), not the consumer's. ctx // becomes every delivered Message.Ctx (see ConsumerManager); it does not -// stop delivery — Consumer.Consume's stop does. +// stop delivery — Consumer.Consume's stop does. It has no units: a config +// naming any is ErrUnitsUnsupported. func (e *EmbeddedNATS) CreateConsumer(ctx context.Context, cfg ConsumerConfig) (Consumer, error) { + if cfg.Units != nil { + return nil, fmt.Errorf("create consumer: %w: the embedded queue is one per tenant", ErrUnitsUnsupported) + } c := &workerConsumer{ fanIn: e.newFanIn(ctx, jetstream.ConsumerConfig{ Durable: cfg.Durable, diff --git a/internal/mq/embedded_failed_test.go b/internal/mq/embedded_failed_test.go index 9ab730919..ef74cc470 100644 --- a/internal/mq/embedded_failed_test.go +++ b/internal/mq/embedded_failed_test.go @@ -11,6 +11,7 @@ import ( // A durable deleted on several tenants' queues ends each delivery; a caller // that drained the first report must not see the next. func TestEmbeddedNATS_Consume_ReportsOnceHoweverManyDeliveriesEnd(t *testing.T) { + t.Parallel() e := newTestEmbedded(t, "acme", "globex") ctx := t.Context() cons, err := e.CreateConsumer(ctx, ConsumerConfig{Durable: "doomed", MaxAckPending: 10}) diff --git a/internal/mq/external.go b/internal/mq/external.go new file mode 100644 index 000000000..778cdac0d --- /dev/null +++ b/internal/mq/external.go @@ -0,0 +1,1079 @@ +package mq + +import ( + "context" + "crypto/tls" + "errors" + "fmt" + "log/slog" + "net/url" + "os" + "slices" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/Wave-RF/WaveHouse/internal/observability" + "github.com/Wave-RF/WaveHouse/internal/tenant" + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + "github.com/nats-io/nuid" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/metric" +) + +// NATSConfig is how ExternalNATS reaches an operator-owned NATS cluster and +// what topology it expects there. Secrets are file paths only. +type NATSConfig struct { + URLs []string + // Name is the connection name the server reports; default + // wavehouse-. + Name string + // CredsFile (a user JWT and nkey seed) and NKeySeedFile are exclusive + // with each other and with User. + CredsFile string + NKeySeedFile string + User string + PasswordFile string + TLS NATSTLS + // JSDomain is the JetStream domain, for a leafnode or hub-and-spoke + // deployment. + JSDomain string + // Topology is what the operator must have created (see NATSTopology). + // Its PublishTimeout also bounds each publish attempt. + Topology NATSTopology + // ConnectTimeout bounds one dial; default 5s. + ConnectTimeout time.Duration + // TopologyWait is how long boot waits for the operator's topology; + // default 60s. + TopologyWait time.Duration + + // recheckEvery and historyEvery override the periodic checks' intervals + // (tests), and activeWithin how recently a unit's durable must have + // delivered or had an ack to count as someone's (default its pinned_ttl). + recheckEvery, historyEvery, activeWithin time.Duration +} + +// NATSTLS is the client side of TLS to the NATS servers. +type NATSTLS struct { + CAFile string + CertFile string + KeyFile string + ServerName string + HandshakeFirst bool +} + +const ( + defaultNATSConnectTimeout = 5 * time.Second + defaultNATSTopologyWait = 60 * time.Second + // topologyRecheck is how often the topology is checked again after boot, + // and historyPoll how often the history and the partitions are read for + // the history's gauges: one stream-info call each. + topologyRecheck = 5 * time.Minute + historyPoll = 30 * time.Second + recheckTimeout = 30 * time.Second + // publishRetries is how many times a publish that got no answer is sent + // again, with the same Nats-Msg-Id, so the stream stores it once. + publishRetries = 2 + publishRetryWait = 250 * time.Millisecond + natsDrainTimeout = 5 * time.Second + // hubInactiveThreshold and replayInactiveThreshold are how long the + // server keeps the history consumers of a pod that went away. A replay's + // also has to outlast sending one fetched batch to a slow SSE client, + // since no pull is waiting meanwhile; a finished replay deletes its own. + hubInactiveThreshold = time.Minute + replayInactiveThreshold = time.Minute + // replayPullWait bounds one pull of a replay whose remaining events the + // server has already counted, and replayBatch is how many one pull asks + // for: a replay is a round trip per batch, not per event. + replayPullWait = 2 * time.Second + replayBatch = 256 + // workerDurable is the ingest worker's durable name + // (ingest.BufferConsumerName), which maps to the operator's durable. + workerDurable = "buffer-consumer" + // jsErrStreamNotMatch is the server's answer to a publish whose subject + // is held by a stream other than the one it expected. + jsErrStreamNotMatch jetstream.ErrorCode = 10060 +) + +// ExternalNATS is the Broker over an operator-owned NATS cluster (see +// NATSTopology): every tenant shares N work-queue ingest partitions, a +// history stream they republish every row to, and one dead-letter stream. It never +// creates, changes, purges or deletes a stream or a durable. The only +// JetStream objects it creates are auto-expiring consumers on the history +// stream: one per Subscribe (the hub bridge) and one per replay. +type ExternalNATS struct { + topo NATSTopology + nc *nats.Conn + js jetstream.JetStream + // apiPrefix is the JetStream API's subject prefix, the domain's if set. + apiPrefix string + // activeWithin overrides each durable's pinned_ttl as the window + // recentlyActive reads (tests). + activeWithin time.Duration + + // partitions is partition p's stream name, dlq the dead-letter stream's, + // both found by subject at boot; units is every partition's shard + // durables, partition by partition. extras is the units outside them + // still draining, as last read (at boot and every re-check). + partitions []string + dlq string + units []natsUnit + extras atomic.Pointer[[]natsUnit] + + budgets sync.Map // tenant.ID → int64 + budgetNote sync.Once + // warnedGap holds, per tenant CheckReplayWindows warned about, the gap + // window it warned for. + warnedGap sync.Map // tenant.ID → time.Duration + // historyMaxAge is the history stream's max_age as last read, and + // replayWindows the gap windows CheckReplayWindows was last given, checked + // again when the poll reads a different max_age. + historyMaxAge atomic.Int64 + replayWindows atomic.Pointer[map[tenant.ID]time.Duration] + + // perms is every publish the server refused the connection, which the + // topology check probes the user's shard durable permissions against. + perms *natsPermissionWatch + connected atomic.Bool + topologyOK atomic.Bool + // closing is set by Close, whose own disconnect is not worth a warning. + closing atomic.Bool + // historyBehind is how far the history's newest row trails the + // partitions' newest, as last read (nanoseconds). + historyBehind atomic.Int64 + gauges metric.Registration + + mu sync.Mutex + nextID int + stoppers map[int]func() + + // stopping ends the watch loop's checks at Close, which waits for + // loopDone; connClosed is closed by the client's closed callback. + stopping context.Context + stop context.CancelFunc + loopDone, connClosed chan struct{} + closeOnce sync.Once +} + +var ( + _ Broker = (*ExternalNATS)(nil) + _ Sharded = (*ExternalNATS)(nil) +) + +// NewNATS connects to the cluster, waits up to cfg.TopologyWait for the +// operator's topology to pass verifyNATSTopology, and returns the broker. +// Recommended findings are logged; a required one still missing when the +// wait runs out is a *TopologyError listing every finding. A cluster not +// reached in that time is ErrUnavailable. The topology is checked again every +// five minutes, reported on wavehouse_mq_topology_ok and in the log, and +// never repaired. +func NewNATS(ctx context.Context, cfg NATSConfig) (*ExternalNATS, error) { + topo := cfg.Topology.withDefaults() + if err := topo.validate(); err != nil { + return nil, err + } + if len(cfg.URLs) == 0 { + return nil, errors.New("nats: no server URLs") + } + e := &ExternalNATS{ + topo: topo, + apiPrefix: jetstream.DefaultAPIPrefix, + activeWithin: cfg.activeWithin, + perms: newNATSPermissionWatch(), + stoppers: map[int]func(){}, + loopDone: make(chan struct{}), + connClosed: make(chan struct{}), + } + opts, err := e.connectOptions(cfg) + if err != nil { + return nil, err + } + e.nc, err = nats.Connect(strings.Join(cfg.URLs, ","), opts...) + if err != nil { + return nil, fmt.Errorf("%w: connect: %w", ErrUnavailable, err) + } + e.connected.Store(e.nc.IsConnected()) + if cfg.JSDomain != "" { + e.apiPrefix = "$JS." + cfg.JSDomain + ".API." + e.js, err = jetstream.NewWithDomain(e.nc, cfg.JSDomain) + } else { + e.js, err = jetstream.New(e.nc) + } + if err != nil { + e.nc.Close() + return nil, fmt.Errorf("jetstream: %w", err) + } + + wait := cfg.TopologyWait + if wait == 0 { + wait = defaultNATSTopologyWait + } + // Each check's requests end with the wait, not the client's API timeout. + bootCtx, cancel := context.WithTimeout(ctx, wait+time.Second) + defer cancel() + findings, err := awaitNATSTopology(bootCtx, e.js, topo, e.perms, wait) + if err == nil { + err = e.resolveStreams(bootCtx) + } + if err != nil { + if !e.nc.IsConnected() { + err = fmt.Errorf("%w: not connected to %s: %w", ErrUnavailable, redactURLs(cfg.URLs), errors.Join(e.nc.LastError(), err)) + } + // A refused request only times out; name what the server refused, + // unless the findings already do. + if denied := e.perms.list(); len(denied) > 0 && !errors.Is(err, ErrTopology) { + err = fmt.Errorf("%w (the server refused publishes to %s: regenerate the user's permissions with wavehouse mq permissions)", err, strings.Join(denied, ", ")) + } + e.nc.Close() + return nil, err + } + for _, f := range findings { + slog.Warn("mq: nats topology: "+f.String(), "component", "nats") + } + e.topologyOK.Store(true) + if err := e.readHistory(ctx); err != nil { + e.nc.Close() + return nil, err + } + if e.gauges, err = e.registerGauges(); err != nil { + e.nc.Close() + return nil, fmt.Errorf("register mq gauges: %w", err) + } + e.stopping, e.stop = context.WithCancel(context.Background()) //nolint:gosec // G118: Close calls it + go e.watch(orDefault(cfg.recheckEvery, topologyRecheck), orDefault(cfg.historyEvery, historyPoll)) + return e, nil +} + +// redactURLs lists urls with any password in them masked. +func redactURLs(urls []string) string { + out := make([]string, len(urls)) + for i, raw := range urls { + if u, err := url.Parse(raw); err == nil { + raw = u.Redacted() + } + out[i] = raw + } + return strings.Join(out, ",") +} + +func orDefault(d, def time.Duration) time.Duration { + if d > 0 { + return d + } + return def +} + +// connectOptions is the connection's auth, TLS and reconnect behavior. It +// reconnects forever: only Close ends the connection, or the server ending +// it for good (auth revoked), which fails every consumer through failed. +func (e *ExternalNATS) connectOptions(cfg NATSConfig) ([]nats.Option, error) { + name := cfg.Name + if name == "" { + host, _ := os.Hostname() + name = "wavehouse-" + host + } + timeout := cfg.ConnectTimeout + if timeout == 0 { + timeout = defaultNATSConnectTimeout + } + opts := []nats.Option{ + nats.Name(name), + nats.Timeout(timeout), + nats.RetryOnFailedConnect(true), + nats.MaxReconnects(-1), + nats.ReconnectWait(2 * time.Second), + nats.ReconnectJitter(500*time.Millisecond, 2*time.Second), + nats.PingInterval(20 * time.Second), + nats.MaxPingsOutstanding(3), + nats.CustomInboxPrefix(natsInboxPrefix(e.topo.Prefix)), + nats.DrainTimeout(natsDrainTimeout), + nats.ConnectHandler(func(*nats.Conn) { e.connected.Store(true) }), + nats.DisconnectErrHandler(func(_ *nats.Conn, err error) { + e.connected.Store(false) + if !e.closing.Load() { + slog.Warn("mq: disconnected from nats; reconnecting", "component", "nats", "error", err) + } + }), + nats.ReconnectHandler(func(nc *nats.Conn) { + e.connected.Store(true) + slog.Info("mq: reconnected to nats", "component", "nats", "url", nc.ConnectedUrlRedacted()) + }), + nats.ClosedHandler(func(*nats.Conn) { + e.connected.Store(false) + close(e.connClosed) + }), + nats.ErrorHandler(func(_ *nats.Conn, sub *nats.Subscription, err error) { + // A refused consumer request is a topology fault the next check + // names; the others (a refused publish) surface where they fail. + if subj, first, ok := e.perms.record(err); ok && isConsumerAPI(subj) { + e.topologyOK.Store(false) + if !first { + return + } + slog.Error("mq: nats refused a consumer request: the wavehouse user's permissions do not match the topology; regenerate them with wavehouse mq permissions", + "component", "nats", "subject", subj) + return + } + subj := "" + if sub != nil { + subj = sub.Subject + } + slog.Warn("mq: nats error", "component", "nats", "subject", subj, "error", err) + }), + } + + auth := 0 + if cfg.CredsFile != "" { + auth++ + opts = append(opts, nats.UserCredentials(cfg.CredsFile)) + } + if cfg.NKeySeedFile != "" { + auth++ + opt, err := nats.NkeyOptionFromSeed(cfg.NKeySeedFile) + if err != nil { + return nil, fmt.Errorf("nats nkey seed: %w", err) + } + opts = append(opts, opt) + } + if cfg.User != "" { + auth++ + password := "" + if cfg.PasswordFile != "" { + raw, err := os.ReadFile(cfg.PasswordFile) + if err != nil { + return nil, fmt.Errorf("nats password: %w", err) + } + password = strings.TrimRight(string(raw), "\r\n") + } + opts = append(opts, nats.UserInfo(cfg.User, password)) + } + if auth > 1 { + return nil, errors.New("nats: set one of creds file, nkey seed file, or user") + } + + t := cfg.TLS + if (t.CertFile == "") != (t.KeyFile == "") { + return nil, errors.New("nats tls: cert and key files come as a pair") + } + if t.CAFile != "" { + opts = append(opts, nats.RootCAs(t.CAFile)) + } + if t.CertFile != "" { + opts = append(opts, nats.ClientCert(t.CertFile, t.KeyFile)) + } + if t.ServerName != "" { + opts = append(opts, nats.Secure(&tls.Config{ServerName: t.ServerName, MinVersion: tls.VersionTLS12})) + } + if t.HandshakeFirst { + opts = append(opts, nats.TLSHandshakeFirst()) + } + return opts, nil +} + +// resolveStreams finds each partition's stream and the dead-letter stream by +// subject, once the verifier has found exactly one of each. +func (e *ExternalNATS) resolveStreams(ctx context.Context) error { + e.partitions = make([]string, e.topo.Partitions) + for p := range e.partitions { + name, err := e.js.StreamNameBySubject(ctx, fmt.Sprintf("%s.ingest.%d.x", e.topo.Prefix, p)) + if err != nil { + return fmt.Errorf("find partition %d: %w", p, err) + } + e.partitions[p] = name + } + name, err := e.js.StreamNameBySubject(ctx, e.topo.Prefix+".dlq.x") + if err != nil { + return fmt.Errorf("find dead-letter stream: %w", err) + } + e.dlq = name + e.units = e.units[:0] + for _, stream := range e.partitions { + for s := range e.topo.Shards { + e.units = append(e.units, natsUnit{stream: stream, durable: natsShardDurable(e.topo.IngestConsumer, s)}) + } + } + return e.readExtras(ctx) +} + +// readExtras reads the shard durables outside the configured units. +func (e *ExternalNATS) readExtras(ctx context.Context) error { + extras, err := findExtraUnits(ctx, e.js, e.topo, e.partitions) + if err != nil { + return e.apiError(err) + } + extras = slices.DeleteFunc(extras, func(u natsUnit) bool { return u.durable == "" }) + e.extras.Store(&extras) + return nil +} + +// watch re-checks the topology and polls the history until Close. +func (e *ExternalNATS) watch(recheck, poll time.Duration) { + defer close(e.loopDone) + topology := time.NewTicker(recheck) + defer topology.Stop() + history := time.NewTicker(poll) + defer history.Stop() + for { + select { + case <-e.stopping.Done(): + return + case <-topology.C: + e.recheck() + case <-history.C: + ctx, cancel := context.WithTimeout(e.stopping, recheckTimeout) + if err := e.readHistory(ctx); err != nil { + slog.Warn("mq: read the nats history stream", "component", "nats", "error", err) + } + cancel() + } + } +} + +// recheck verifies the topology again, reporting the outcome on the gauge +// and in the log. It skips a check while disconnected, which has a gauge of +// its own: the server version reads as empty then, a false fault. +func (e *ExternalNATS) recheck() { + if !e.nc.IsConnected() { + return + } + ctx, cancel := context.WithTimeout(e.stopping, recheckTimeout) + defer cancel() + findings, err := verifyNATSTopology(ctx, e.js, e.topo, e.perms) + if !e.nc.IsConnected() { + return + } + if err != nil { + slog.Warn("mq: nats topology re-check could not run", "component", "nats", "error", err) + return + } + var faults []string + for _, f := range findings { + if f.Severity == FindingRequired { + faults = append(faults, f.String()) + } + } + if err := e.readExtras(ctx); err != nil { + slog.Warn("mq: could not list the shard durables outside the configured ones", "component", "nats", "error", err) + } + was := e.topologyOK.Swap(len(faults) == 0) + switch { + case len(faults) > 0: + slog.Error("mq: nats topology no longer matches what WaveHouse needs", "component", "nats", "findings", faults) + case !was: + slog.Info("mq: nats topology matches again", "component", "nats") + } +} + +// readHistory reads the history stream's max_age, and how far its newest row +// trails the newest row any partition stored. Republish is best effort, so +// the history misses what the partitions stored while it refused rows (full, +// or electing a leader); the gap shows until the next row reaches it. The +// partitions are read first, so a row stored while the history is read +// counts as newer only if the history really lacks it. +func (e *ExternalNATS) readHistory(ctx context.Context) error { + var newest time.Time + for _, name := range e.partitions { + p, err := e.js.Stream(ctx, name) + if err != nil { + return fmt.Errorf("partition stream %s: %w", name, err) + } + if last := p.CachedInfo().State.LastTime; last.After(newest) { + newest = last + } + } + s, err := e.js.Stream(ctx, e.topo.HistoryStream) + if err != nil { + return fmt.Errorf("history stream %s: %w", e.topo.HistoryStream, err) + } + info := s.CachedInfo() + if prev := e.historyMaxAge.Swap(int64(info.Config.MaxAge)); prev != int64(info.Config.MaxAge) { + if w := e.replayWindows.Load(); w != nil { + e.CheckReplayWindows(*w) + } + } + // An empty history has no last row: it has seen nothing since it was + // created, so rows the partitions stored before that are not its gap. + since := info.State.LastTime + if since.IsZero() { + since = info.Created + } + e.historyBehind.Store(int64(max(0, newest.Sub(since)))) + return nil +} + +// registerGauges reports the connection, the topology check, and how far the +// history trails the partitions. +func (e *ExternalNATS) registerGauges() (metric.Registration, error) { + meter := otel.Meter("wavehouse-mq") + connected, err := meter.Int64ObservableGauge("wavehouse_mq_connected", + metric.WithDescription("1 while connected to the external NATS cluster, else 0")) + if err != nil { + return nil, err + } + topologyOK, err := meter.Int64ObservableGauge("wavehouse_mq_topology_ok", + metric.WithDescription("1 while the external NATS topology passed its last check, else 0")) + if err != nil { + return nil, err + } + behind, err := meter.Float64ObservableGauge("wavehouse_mq_history_behind_seconds", + metric.WithDescription("How far the history stream's newest row trails the newest row an ingest partition stored; one that stays up or grows means the history is not taking rows, which SSE replay and the live hub then miss")) + if err != nil { + return nil, err + } + return meter.RegisterCallback(func(_ context.Context, o metric.Observer) error { + o.ObserveInt64(connected, boolGauge(e.connected.Load())) + o.ObserveInt64(topologyOK, boolGauge(e.topologyOK.Load())) + o.ObserveFloat64(behind, time.Duration(e.historyBehind.Load()).Seconds()) + return nil + }, connected, topologyOK, behind) +} + +func boolGauge(b bool) int64 { + if b { + return 1 + } + return 0 +} + +// track registers stop to run at Close, returning its unregistration. +func (e *ExternalNATS) track(stop func()) (untrack func()) { + e.mu.Lock() + defer e.mu.Unlock() + id := e.nextID + e.nextID++ + e.stoppers[id] = stop + return func() { + e.mu.Lock() + defer e.mu.Unlock() + delete(e.stoppers, id) + } +} + +// Publish stores data on topic's subject in its table's partition and shard, bounded +// by the topology's PublishTimeout per attempt. A publish that gets no answer +// is sent again up to twice with the same Nats-Msg-Id — the caller's +// WithIdempotencyKey when given — which the partition's duplicate window +// stores once. A partition at max_bytes, or a topic at its +// max_msgs_per_subject, is ErrQueueFull; no answer, a lost connection, or a +// partition stream that is gone is ErrUnavailable. It never creates anything. +func (e *ExternalNATS) Publish(ctx context.Context, topic Topic, data []byte, opts ...PublishOpt) error { + subj, p, err := natsIngestSubject(e.topo.Prefix, e.topo.Partitions, e.topo.Shards, topic) + if err != nil { + return err + } + return e.publish(ctx, subj, e.partitions[p], false, data, opts) +} + +// DuplicateWindow is the shortest duplicate_window among the partitions: how +// long an idempotency key is remembered everywhere a tenant may publish. +func (e *ExternalNATS) DuplicateWindow(ctx context.Context) (time.Duration, error) { + var shortest time.Duration + for _, name := range e.partitions { + s, err := e.js.Stream(ctx, name) + if err != nil { + return 0, fmt.Errorf("stream %s: %w", name, err) + } + if d := s.CachedInfo().Config.Duplicates; shortest == 0 || d < shortest { + shortest = d + } + } + return shortest, nil +} + +// DeadLetter parks msg's data on the shared dead-letter stream under its +// topic, with a fresh Nats-Msg-Id. It does not ack msg. +func (e *ExternalNATS) DeadLetter(ctx context.Context, msg *Message, opts ...PublishOpt) error { + return e.publish(ctx, e.topo.Prefix+".dlq."+msg.topicKey, e.dlq, true, msg.Data, opts) +} + +// publish stores data on subj, which stream should hold. With expect the +// server refuses a publish another stream holds. Without it the row is +// stored wherever the subject leads, and a mismatch the ack names is +// reported as a topology fault and ErrUnavailable, the row left on the other +// stream. An ingest publish goes without: the partition republishes each row +// with its headers, and the history would refuse a copy expecting the +// partition. +func (e *ExternalNATS) publish(ctx context.Context, subj, stream string, expect bool, data []byte, opts []PublishOpt) error { + msg := nats.NewMsg(subj) + msg.Data = data + headers := Headers{} + for _, opt := range opts { + opt(headers) + } + observability.InjectHeaders(ctx, headers) + msg.Header = nats.Header(headers) + // WithMsgID overwrites the header, so a caller's idempotency key must be + // the id itself; otherwise a fresh one keeps this publish's retries one. + id := headers.Get(idempotencyHeader) + if id == "" { + id = nuid.Next() + } + pubOpts := []jetstream.PublishOpt{ + jetstream.WithMsgID(id), + jetstream.WithRetryAttempts(0), + } + if expect { + pubOpts = append(pubOpts, jetstream.WithExpectStream(stream)) + } + + var err error + for attempt := 0; ; attempt++ { + actx, cancel := context.WithTimeout(ctx, e.topo.PublishTimeout) + var ack *jetstream.PubAck + ack, err = e.js.PublishMsg(actx, msg, pubOpts...) + cancel() + if err == nil && ack.Stream != stream { + e.lostTopology("the subject is held by another stream than " + stream) + return fmt.Errorf("%w: %s was stored by stream %s, not %s", ErrUnavailable, subj, ack.Stream, stream) + } + if err == nil { + return nil + } + if ctx.Err() != nil { + return fmt.Errorf("publish: %w", ctx.Err()) + } + if !noAnswer(err) || attempt == publishRetries { + break + } + select { + case <-ctx.Done(): + return fmt.Errorf("publish: %w", ctx.Err()) + case <-time.After(publishRetryWait): + } + } + return e.publishError(stream, err) +} + +// noAnswer reports a publish that got no answer: it may or may not have been +// stored, so it is sent again with the same id. +func noAnswer(err error) bool { + return errors.Is(err, jetstream.ErrNoStreamResponse) || errors.Is(err, nats.ErrNoResponders) || + errors.Is(err, context.DeadlineExceeded) || errors.Is(err, nats.ErrTimeout) +} + +// publishError maps a failed publish to the sentinel the API answers. +func (e *ExternalNATS) publishError(stream string, err error) error { + // The server names a full store only in the error's text. + if msg := err.Error(); strings.Contains(msg, "maximum bytes exceeded") || strings.Contains(msg, "maximum messages per subject exceeded") { + return fmt.Errorf("%w: %w", ErrQueueFull, err) + } + var apiErr *jetstream.APIError + if errors.As(err, &apiErr) && apiErr.ErrorCode == jsErrStreamNotMatch { + e.lostTopology("the subject is held by another stream than " + stream) + return fmt.Errorf("%w: %w", ErrUnavailable, err) + } + if errors.Is(err, jetstream.ErrNoStreamResponse) && e.nc.IsConnected() { + ctx, cancel := context.WithTimeout(context.Background(), e.topo.PublishTimeout) + defer cancel() + if _, serr := e.js.Stream(ctx, stream); errors.Is(serr, jetstream.ErrStreamNotFound) { + e.lostTopology("stream " + stream + " does not exist") + return fmt.Errorf("%w: stream %s does not exist: %w", ErrUnavailable, stream, err) + } + } + if noAnswer(err) || errors.Is(err, nats.ErrConnectionClosed) || errors.Is(err, nats.ErrConnectionDraining) || + errors.Is(err, nats.ErrReconnectBufExceeded) || errors.Is(err, nats.ErrDisconnected) { + return fmt.Errorf("%w: %w", ErrUnavailable, err) + } + return err +} + +// lostTopology records a topology fault found between checks. +func (e *ExternalNATS) lostTopology(problem string) { + e.topologyOK.Store(false) + slog.Error("mq: nats topology no longer matches what WaveHouse needs", "component", "nats", "problem", problem) +} + +// wrapMsg adapts a delivered message: its topic key is the subject with the +// prefix and partition stripped. +func (e *ExternalNATS) wrapMsg(ctx context.Context, m jetstream.Msg, acks bool) *Message { + key, ok := natsTopicKey(e.topo.Prefix, m.Subject()) + if !ok { + key = m.Subject() + } + if !acks { + return newMessage(ctx, key, m.Data(), time.Now(), nil, nil, nil) + } + return newMessage(ctx, key, m.Data(), time.Now(), m.DoubleAck, m.Ack, m.Nak, WithNakDelay(m.NakWithDelay)) +} + +// Subscribe delivers every ingest event stored on the history stream from +// now on to handler, on one goroutine, until ctx is done or Close. It reads +// through an ordered ack-less consumer of its own, which skips nothing across +// reconnects and expires once this process is gone: every pod's hub needs +// every event, which one shared durable would split between them. So +// consumerName names nothing here, and a handler error has no redelivery to +// ask for: it is logged. +func (e *ExternalNATS) Subscribe(ctx context.Context, consumerName string, handler func(msg *Message) error) error { + cons, err := e.js.OrderedConsumer(ctx, e.topo.HistoryStream, jetstream.OrderedConsumerConfig{ + FilterSubjects: []string{natsHistorySubjects(e.topo.Prefix)}, + DeliverPolicy: jetstream.DeliverNewPolicy, + InactiveThreshold: hubInactiveThreshold, + }) + if err != nil { + return fmt.Errorf("history consumer: %w", e.apiError(err)) + } + cc, err := cons.Consume(func(m jetstream.Msg) { + msg := e.wrapMsg(observability.ExtractHeaders(ctx, m.Headers()), m, false) + if err := handler(msg); err != nil { + slog.Warn("mq: subscriber could not handle an event", "component", "nats", "consumer", consumerName, "topic", msg.TopicKey(), "error", err) + } + }, jetstream.ConsumeErrHandler(func(_ jetstream.ConsumeContext, err error) { + slog.Warn("mq: history consumer reported an error", "component", "nats", "consumer", consumerName, "error", err) + })) + if err != nil { + return fmt.Errorf("consume history: %w", err) + } + untrack := e.track(cc.Stop) + go func() { + select { + case <-ctx.Done(): + case <-cc.Closed(): + } + untrack() + cc.Stop() + }() + return nil +} + +// apiError marks a JetStream request that got no answer as ErrUnavailable. +func (e *ExternalNATS) apiError(err error) error { + if noAnswer(err) || errors.Is(err, nats.ErrConnectionClosed) { + return fmt.Errorf("%w: %w", ErrUnavailable, err) + } + return err +} + +// durable maps the durable a caller names to the operator's shard durables' +// prefix: the ingest worker's name, or the operator's own. +func (e *ExternalNATS) durable(name string) (string, bool) { + if name == workerDurable || name == e.topo.IngestConsumer { + return e.topo.IngestConsumer, true + } + return "", false +} + +// IngestUnits implements Sharded: every partition's shard durables, then the +// ones outside them still draining. A unit's name is /. +func (e *ExternalNATS) IngestUnits() (configured, extra []string) { + for _, u := range e.units { + configured = append(configured, u.id()) + } + if xs := e.extras.Load(); xs != nil { + for _, u := range *xs { + extra = append(extra, u.id()) + } + } + return configured, extra +} + +// unit resolves a unit's name, and whether it is an extra. +func (e *ExternalNATS) unit(name string) (natsUnit, bool, bool) { + for _, u := range e.units { + if u.id() == name { + return u, false, true + } + } + if xs := e.extras.Load(); xs != nil { + for _, u := range *xs { + if u.id() == name { + return u, true, true + } + } + } + return natsUnit{}, false, false +} + +// ResetOrphaned implements Sharded. A pinned client holds the unit, and one +// that delivered or had an ack within the durable's pinned_ttl may still: +// its pin can be gone for a moment (a consumer leader change releases it) +// while it lives. Either is ErrUnitHeld. Otherwise, with rows awaiting an +// ack, whoever received them is gone: the reset redelivers them from the ack +// floor at once, where the server would wait out ack_wait. Rows already +// acked are not delivered again. +func (e *ExternalNATS) ResetOrphaned(ctx context.Context, name string) (bool, error) { + u, _, ok := e.unit(name) + if !ok { + return false, fmt.Errorf("unit %q: %w", name, ErrUnitsUnsupported) + } + c, err := e.js.Consumer(ctx, u.stream, u.durable) + if err != nil { + return false, fmt.Errorf("consumer %s: %w", name, e.apiError(err)) + } + info := c.CachedInfo() + if pinnedClient(info) != "" { + return false, fmt.Errorf("unit %s: %w", name, ErrUnitHeld) + } + if info.NumAckPending == 0 { + return false, nil + } + if e.recentlyActive(info) { + return false, fmt.Errorf("unit %s: %w: it delivered or had an ack within %s", name, ErrUnitHeld, e.activeWindow(info)) + } + if _, err := e.js.ResetConsumer(ctx, u.stream, u.durable); err != nil { + return false, fmt.Errorf("reset %s: %w", name, e.apiError(err)) + } + return true, nil +} + +// Unowned implements Sharded: the units whose durable has rows pending or +// awaiting an ack while no client holds its pin and none was active on it +// within its pinned_ttl. A live owner keeps its pin by pulling, even with +// nothing more to fetch, so such a unit has nobody writing it. +func (e *ExternalNATS) Unowned(ctx context.Context) (int, error) { + units := slices.Clone(e.units) + if xs := e.extras.Load(); xs != nil { + units = append(units, *xs...) + } + n := 0 + for _, u := range units { + c, err := e.js.Consumer(ctx, u.stream, u.durable) + if errors.Is(err, jetstream.ErrConsumerNotFound) { + continue // the topology check names it + } + if err != nil { + return 0, fmt.Errorf("consumer %s: %w", u.id(), e.apiError(err)) + } + info := c.CachedInfo() + if (info.NumPending > 0 || info.NumAckPending > 0) && pinnedClient(info) == "" && !e.recentlyActive(info) { + n++ + } + } + return n, nil +} + +// activeWindow is how recently a durable must have delivered or had an ack +// to count as someone's: its pinned_ttl, the time the server gives a silent +// pinned client before it unpins it. +func (e *ExternalNATS) activeWindow(info *jetstream.ConsumerInfo) time.Duration { + if e.activeWithin > 0 { + return e.activeWithin + } + return max(info.Config.PinnedTTL, minPinnedTTL) +} + +// recentlyActive reports whether a durable delivered a row, or its ack +// floor moved, within its activeWindow, by the server's own clock. +func (e *ExternalNATS) recentlyActive(info *jetstream.ConsumerInfo) bool { + window := e.activeWindow(info) + for _, last := range []*time.Time{info.Delivered.Last, info.AckFloor.Last} { + if last != nil && info.TimeStamp.Sub(*last) < window { + return true + } + } + return false +} + +// pinnedClient is the pin id the server holds for WaveHouse's priority group +// on a consumer, "" when no client is pinned. +func pinnedClient(info *jetstream.ConsumerInfo) string { + for _, g := range info.PriorityGroups { + if g.Group == natsPriorityGroup { + return g.PinnedClientID + } + } + return "" +} + +// DeadLetterCounts counts tenant id's parked messages on the shared +// dead-letter stream, by a subject filter on its tenant: one call. Per-table +// counts are keyed by deadLetterTables, whose table filter matches every +// scope of that table. A tenant with nothing parked has zero counts; there is +// no queue of its own whose absence could mean anything. +func (e *ExternalNATS) DeadLetterCounts(ctx context.Context, id tenant.ID, table string) (DeadLetterCounts, error) { + if _, err := tenant.Parse(string(id)); err != nil { + return DeadLetterCounts{}, fmt.Errorf("tenant: %w", err) + } + s, err := e.js.Stream(ctx, e.dlq) + if err != nil { + return DeadLetterCounts{}, fmt.Errorf("dead-letter stream %s: %w", e.dlq, e.apiError(err)) + } + prefix := e.topo.Prefix + ".dlq." + info, err := s.Info(ctx, jetstream.WithSubjectFilter(prefix+string(id)+".>")) + if err != nil { + return DeadLetterCounts{}, fmt.Errorf("dead-letter stream %s: %w", e.dlq, e.apiError(err)) + } + var total uint64 + for _, n := range info.State.Subjects { + total += n + } + return DeadLetterCounts{Tables: deadLetterTables(info.State.Subjects, prefix, table), Total: total}, nil +} + +// PurgeAcked removes nothing: the partitions delete each row once it is +// acknowledged, and the history keeps what its max_age allows, both the +// operator's. No sweeper is wired under this broker, so nothing calls it +// there; it is here for the Broker contract. +func (e *ExternalNATS) PurgeAcked(_ context.Context, consumer string, _ map[tenant.ID]time.Time) (bool, error) { + if _, ok := e.durable(consumer); !ok { + return false, fmt.Errorf("consumer %q: %w", consumer, ErrConsumerNotFound) + } + return false, nil +} + +// CheckReplayWindows warns for each tenant whose gap window is longer than +// the history stream keeps: SSE replay serves only the last max_age of it. +// It warns once per tenant and window, so calling it after every settings +// reload repeats nothing until a window changes. No I/O: max_age is read at +// boot and by the periodic poll, which runs the check again with the last +// windows when max_age changes. +func (e *ExternalNATS) CheckReplayWindows(windows map[tenant.ID]time.Duration) { + e.replayWindows.Store(&windows) + maxAge := time.Duration(e.historyMaxAge.Load()) + if maxAge <= 0 { + return + } + for id, window := range windows { + if window <= maxAge { + e.warnedGap.Delete(id) + continue + } + if prev, ok := e.warnedGap.Swap(id, window); ok && prev.(time.Duration) == window { + continue + } + slog.Warn("mq: the nats history keeps less than this tenant's gap window; SSE replay serves only the last max_age", + "component", "nats", "tenant", id, "history_stream", e.topo.HistoryStream, "max_age", maxAge, "gap_window", window) + } +} + +// ReplaySince reads topic's events from the history stream, stored at or +// after since, through an ack-less consumer of its own that expires once +// idle, in batches, until send returns false or the events the server +// counted when the replay began are sent. Anything older than the history's max_age is gone. +// A pull that fails before then is an error; a done ctx returns ctx's error. +func (e *ExternalNATS) ReplaySince(ctx context.Context, topic Topic, since time.Time, send func(data []byte) bool) error { + subj, err := natsHistorySubject(e.topo.Prefix, topic) + if err != nil { + return err + } + cons, err := e.js.CreateConsumer(ctx, e.topo.HistoryStream, jetstream.ConsumerConfig{ + FilterSubject: subj, + DeliverPolicy: jetstream.DeliverByStartTimePolicy, + OptStartTime: &since, + AckPolicy: jetstream.AckNonePolicy, + InactiveThreshold: replayInactiveThreshold, + MemoryStorage: true, + Replicas: 1, + }) + if err != nil { + return fmt.Errorf("replay consumer: %w", e.apiError(err)) + } + defer e.dropConsumer(cons.CachedInfo().Name) + + // Counted once: events published during the replay reach the SSE client + // through the live subscription it registered first, so chasing them here + // would only send duplicates. + remaining := cons.CachedInfo().NumPending + for remaining > 0 { + if err := ctx.Err(); err != nil { + return err + } + batch, err := cons.Fetch(int(min(remaining, replayBatch)), jetstream.FetchMaxWait(replayPullWait)) //nolint:gosec // capped + if err != nil { + return fmt.Errorf("replay fetch: %w", err) + } + got := 0 + for msg := range batch.Messages() { + got++ + remaining-- + if !send(msg.Data()) { + return nil + } + if err := ctx.Err(); err != nil { + return err + } + if e.nc.IsClosed() || e.nc.IsDraining() { + return fmt.Errorf("replay: %w", nats.ErrConnectionClosed) + } + } + if err := batch.Error(); err != nil && !errors.Is(err, nats.ErrTimeout) { + return fmt.Errorf("replay fetch: %w", err) + } + if got == 0 { + // The history dropped what was left (max_age) while connected: + // that is caught up. A pull that raced the connection closing + // can end the same way, and that is not. + if e.nc.IsConnected() { + return nil + } + return fmt.Errorf("replay fetch: %w", nats.ErrConnectionClosed) + } + } + return nil +} + +// dropConsumer deletes a finished replay's consumer rather than leaving it to +// expire, best effort. +func (e *ExternalNATS) dropConsumer(name string) { + if !e.nc.IsConnected() { + return + } + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + _ = e.js.DeleteConsumer(ctx, e.topo.HistoryStream, name) +} + +// SetMaxBytes records tenant id's budget and enforces nothing: the tenants of +// a partition share its max_bytes, and max_msgs_per_subject caps each topic. +// The first call says so in the log. +func (e *ExternalNATS) SetMaxBytes(_ context.Context, id tenant.ID, maxBytes int64) error { + if _, err := tenant.Parse(string(id)); err != nil { + return fmt.Errorf("tenant: %w", err) + } + e.budgetNote.Do(func() { + slog.Info("mq: tenant byte budgets (mq.max_bytes_gb) are recorded, not enforced, on external NATS; a partition's max_bytes is shared by its tenants", + "component", "nats", "partitions", e.topo.Partitions) + }) + e.budgets.Store(id, maxBytes) + return nil +} + +// MaxBytes reports the budget last recorded for id. +func (e *ExternalNATS) MaxBytes(id tenant.ID) int64 { + if v, ok := e.budgets.Load(id); ok { + return v.(int64) + } + return 0 +} + +// Stats reports this process's connection and its inbound message count. +func (e *ExternalNATS) Stats() (observability.MQStats, error) { + s := e.nc.Stats() + return observability.MQStats{ + Connections: boolGauge(e.nc.IsConnected()), + InMsgs: int64(min(s.InMsgs, uint64(1<<62))), //nolint:gosec // capped + }, nil +} + +// Close stops every consumer, so none reports failed, then drains the +// connection so pending acks are flushed. Safe to call more than once. +func (e *ExternalNATS) Close() error { + e.closeOnce.Do(func() { + e.closing.Store(true) + e.stop() + <-e.loopDone + e.mu.Lock() + stops := make([]func(), 0, len(e.stoppers)) + for _, stop := range e.stoppers { + stops = append(stops, stop) + } + clear(e.stoppers) + e.mu.Unlock() + for _, stop := range stops { + stop() + } + if e.gauges != nil { + _ = e.gauges.Unregister() + } + if err := e.nc.Drain(); err != nil { + e.nc.Close() + } + select { + case <-e.connClosed: + case <-time.After(natsDrainTimeout + time.Second): + e.nc.Close() + } + }) + return nil +} diff --git a/internal/mq/external_auth_test.go b/internal/mq/external_auth_test.go new file mode 100644 index 000000000..63572e5b8 --- /dev/null +++ b/internal/mq/external_auth_test.go @@ -0,0 +1,199 @@ +//go:build integration + +package mq + +import ( + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "math/big" + "net" + "os" + "path/filepath" + "testing" + "time" + + "github.com/nats-io/jwt/v2" + natsserver "github.com/nats-io/nats-server/v2/server" + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + "github.com/nats-io/nkeys" + "github.com/stretchr/testify/require" +) + +// authFixture starts a JetStream server from opts, connects the operator's +// stand-in with admin, and applies the shipped topology. +func authFixture(t *testing.T, opts *natsserver.Options, admin ...nats.Option) *natsFixture { + t.Helper() + opts.Host, opts.Port, opts.NoSigs, opts.NoLog = "127.0.0.1", -1, true, true + opts.JetStream, opts.StoreDir = true, t.TempDir() + opts.JetStreamMaxStore, opts.JetStreamMaxMemory = shippedFileStore(t), 1<<50 + s, err := natsserver.NewServer(opts) + require.NoError(t, err) + s.Start() + require.True(t, s.ReadyForConnections(10*time.Second), "nats server not ready") + t.Cleanup(s.Shutdown) + f := &natsFixture{server: s, opts: opts} + url := s.ClientURL() + if opts.TLS { + url = "tls://localhost:" + portOf(f) + } + nc, err := nats.Connect(url, admin...) + require.NoError(t, err) + t.Cleanup(nc.Close) + js, err := jetstream.New(nc) + require.NoError(t, err) + f.admin = js + f.apply(t, shippedTopology(t)) + return f +} + +// connects reports that the broker boots against f with cfg and publishes. +func connects(t *testing.T, url string, cfg NATSConfig) { + t.Helper() + cfg.URLs = []string{url} + cfg.Topology = NATSTopology{Partitions: 4, Shards: 8} + cfg.TopologyWait = 5 * time.Second + e, err := NewNATS(t.Context(), cfg) + require.NoError(t, err) + t.Cleanup(func() { _ = e.Close() }) + require.NoError(t, e.Publish(t.Context(), Topic{Tenant: "acme", Table: "t"}, []byte("x"))) +} + +func TestNewNATS_NKeySeed(t *testing.T) { + t.Parallel() + kp, err := nkeys.CreateUser() + require.NoError(t, err) + pub, err := kp.PublicKey() + require.NoError(t, err) + seed, err := kp.Seed() + require.NoError(t, err) + seedFile := writeSecret(t, string(seed)) + + adminOpt, err := nats.NkeyOptionFromSeed(seedFile) + require.NoError(t, err) + f := authFixture(t, &natsserver.Options{Nkeys: []*natsserver.NkeyUser{{Nkey: pub}}}, adminOpt) + connects(t, f.server.ClientURL(), NATSConfig{NKeySeedFile: seedFile}) + + _, err = NewNATS(t.Context(), NATSConfig{URLs: []string{f.server.ClientURL()}, TopologyWait: 300 * time.Millisecond}) + require.ErrorIs(t, err, ErrUnavailable, "no credentials: never connected") +} + +func TestNewNATS_CredsFile(t *testing.T) { + t.Parallel() + operator, err := nkeys.CreateOperator() + require.NoError(t, err) + opub, err := operator.PublicKey() + require.NoError(t, err) + oc := jwt.NewOperatorClaims(opub) + signed, err := oc.Encode(operator) + require.NoError(t, err) + oc, err = jwt.DecodeOperatorClaims(signed) + require.NoError(t, err) + + account, err := nkeys.CreateAccount() + require.NoError(t, err) + apub, err := account.PublicKey() + require.NoError(t, err) + ac := jwt.NewAccountClaims(apub) + ac.Limits.JetStreamLimits = jwt.JetStreamLimits{MemoryStorage: -1, DiskStorage: -1, Streams: -1, Consumer: -1} + ajwt, err := ac.Encode(operator) + require.NoError(t, err) + resolver := &natsserver.MemAccResolver{} + require.NoError(t, resolver.Store(apub, ajwt)) + // Operator mode wants a system account. + sys, err := nkeys.CreateAccount() + require.NoError(t, err) + spub, err := sys.PublicKey() + require.NoError(t, err) + sjwt, err := jwt.NewAccountClaims(spub).Encode(operator) + require.NoError(t, err) + require.NoError(t, resolver.Store(spub, sjwt)) + + user, err := nkeys.CreateUser() + require.NoError(t, err) + upub, err := user.PublicKey() + require.NoError(t, err) + ujwt, err := jwt.NewUserClaims(upub).Encode(account) + require.NoError(t, err) + seed, err := user.Seed() + require.NoError(t, err) + creds, err := jwt.FormatUserConfig(ujwt, seed) + require.NoError(t, err) + credsFile := writeSecret(t, string(creds)) + + f := authFixture(t, &natsserver.Options{TrustedOperators: []*jwt.OperatorClaims{oc}, AccountResolver: resolver, SystemAccount: spub}, + nats.UserCredentials(credsFile)) + connects(t, f.server.ClientURL(), NATSConfig{CredsFile: credsFile}) +} + +func TestNewNATS_MutualTLS(t *testing.T) { + t.Parallel() + dir := t.TempDir() + ca, caKey := testCA(t, dir) + serverCert, serverKey := testLeaf(t, dir, "server", ca, caKey, x509.ExtKeyUsageServerAuth) + clientCert, clientKey := testLeaf(t, dir, "client", ca, caKey, x509.ExtKeyUsageClientAuth) + caFile := filepath.Join(dir, "ca.pem") + + tc, err := natsserver.GenTLSConfig(&natsserver.TLSConfigOpts{CertFile: serverCert, KeyFile: serverKey, CaFile: caFile, Verify: true}) + require.NoError(t, err) + f := authFixture(t, &natsserver.Options{TLS: true, TLSVerify: true, TLSConfig: tc, TLSTimeout: 5}, + nats.RootCAs(caFile), nats.ClientCert(clientCert, clientKey)) + url := "tls://localhost:" + portOf(f) + connects(t, url, NATSConfig{TLS: NATSTLS{CAFile: caFile, CertFile: clientCert, KeyFile: clientKey, ServerName: "localhost"}}) + + _, err = NewNATS(t.Context(), NATSConfig{URLs: []string{url}, TLS: NATSTLS{CAFile: caFile}, TopologyWait: 300 * time.Millisecond}) + require.ErrorIs(t, err, ErrUnavailable, "no client certificate: never connected") +} + +func portOf(f *natsFixture) string { + _, port, _ := net.SplitHostPort(f.server.Addr().String()) + return port +} + +// testCA writes a self-signed CA to dir/ca.pem. +func testCA(t *testing.T, dir string) (*x509.Certificate, *ecdsa.PrivateKey) { + t.Helper() + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + require.NoError(t, err) + tmpl := &x509.Certificate{ + SerialNumber: big.NewInt(1), Subject: pkix.Name{CommonName: "test ca"}, + NotBefore: time.Now().Add(-time.Hour), NotAfter: time.Now().Add(time.Hour), + IsCA: true, BasicConstraintsValid: true, KeyUsage: x509.KeyUsageCertSign, + } + der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key) + require.NoError(t, err) + writePEM(t, filepath.Join(dir, "ca.pem"), "CERTIFICATE", der) + ca, err := x509.ParseCertificate(der) + require.NoError(t, err) + return ca, key +} + +// testLeaf writes a localhost certificate signed by ca, and its key. +func testLeaf(t *testing.T, dir, name string, ca *x509.Certificate, caKey *ecdsa.PrivateKey, usage x509.ExtKeyUsage) (certFile, keyFile string) { + t.Helper() + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + require.NoError(t, err) + tmpl := &x509.Certificate{ + SerialNumber: big.NewInt(time.Now().UnixNano()), Subject: pkix.Name{CommonName: "localhost"}, + DNSNames: []string{"localhost"}, IPAddresses: []net.IP{net.ParseIP("127.0.0.1")}, + NotBefore: time.Now().Add(-time.Hour), NotAfter: time.Now().Add(time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature, ExtKeyUsage: []x509.ExtKeyUsage{usage}, + } + der, err := x509.CreateCertificate(rand.Reader, tmpl, ca, &key.PublicKey, caKey) + require.NoError(t, err) + keyDER, err := x509.MarshalECPrivateKey(key) + require.NoError(t, err) + certFile, keyFile = filepath.Join(dir, name+".pem"), filepath.Join(dir, name+"-key.pem") + writePEM(t, certFile, "CERTIFICATE", der) + writePEM(t, keyFile, "EC PRIVATE KEY", keyDER) + return certFile, keyFile +} + +func writePEM(t *testing.T, path, kind string, der []byte) { + t.Helper() + require.NoError(t, os.WriteFile(path, pem.EncodeToMemory(&pem.Block{Type: kind, Bytes: der}), 0o600)) +} diff --git a/internal/mq/external_conformance_test.go b/internal/mq/external_conformance_test.go new file mode 100644 index 000000000..76cd940fc --- /dev/null +++ b/internal/mq/external_conformance_test.go @@ -0,0 +1,56 @@ +//go:build integration + +// The external broker's tests run in `make test-integration`, not the unit +// suite: each starts a NATS server with the shipped topology, which the +// internal/mq unit binary's 15s budget cannot absorb (#617). + +package mq_test + +import ( + "sync" + "testing" + + "github.com/Wave-RF/WaveHouse/internal/mq" + "github.com/Wave-RF/WaveHouse/internal/mq/mqtest" + "github.com/Wave-RF/WaveHouse/internal/tenant" + "github.com/stretchr/testify/require" +) + +func TestExternalNATS_Conformance(t *testing.T) { + var ( + mu sync.Mutex + fixtures = map[mq.Broker]*mq.ExternalFixture{} + ) + fixtureOf := func(b mq.Broker) *mq.ExternalFixture { + mu.Lock() + defer mu.Unlock() + return fixtures[b] + } + mqtest.Run(t, mqtest.Harness{ + New: func(t *testing.T) mq.Broker { + f := mq.NewExternalFixture(t) + b := f.Broker(t) + for _, id := range []tenant.ID{mqtest.Acme, mqtest.Globex} { + require.NoError(t, b.SetMaxBytes(t.Context(), id, 64<<20)) + } + mu.Lock() + fixtures[b] = f + mu.Unlock() + return b + }, + // The operator deletes the durable on every partition. + EndDelivery: func(t *testing.T, b mq.Broker) { + fixtureOf(b).DeleteIngestDurable(t) + }, + // The tenant's partition shrunk to a few KiB, then filled. + Fill: func(t *testing.T, b mq.Broker, id tenant.ID) { + fixtureOf(b).FillPartition(t, b, id) + }, + Caps: mqtest.Caps{ + PerTenantBudget: false, + PurgesAcked: false, + UnbudgetedNotFound: false, + ConfiguresDurables: false, + }, + }) +} diff --git a/internal/mq/external_consumer.go b/internal/mq/external_consumer.go new file mode 100644 index 000000000..277940d93 --- /dev/null +++ b/internal/mq/external_consumer.go @@ -0,0 +1,649 @@ +package mq + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "log/slog" + "slices" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" +) + +const ( + // pinIDHeader carries, on a row the server delivered to the pinned + // puller, that puller's pin id. + pinIDHeader = "Nats-Pin-Id" + // releaseTimeout bounds one unit's release. + releaseTimeout = 2 * time.Second + // renewEvery is how often a unit at its cap, or halted, sends the server + // a pull that keeps its pin. The server resets + // the pin's timer on each pull from the pinned client, and the verifier + // holds pinned_ttl to at least twice natsPullExpiry, so the pin cannot + // lapse between two. + renewEvery = natsPullExpiry + // renewWait is how long a renewing pull waits: short, so that most of + // the time no pull of a paused unit is waiting, and a durable deleted + // meanwhile answers the next one with no responders. + renewWait = time.Second + // fetchExpiry is how long one fetch waits for rows. A halt waits for the + // fetch in flight to end rather than cut it short, since a row the server + // already sent to a pull its client dropped would be redelivered only + // after ack_wait, behind newer rows: so this bounds how long a halt, and + // with it a handover, takes. An idle unit costs one pull a second. + fetchExpiry = time.Second + // fetchRetryWait spaces the fetches of a unit whose last one failed. + fetchRetryWait = 250 * time.Millisecond + // noAnswersBeforeCheck is how many pulls in a row may get no responders + // (the durable is gone, or its leader is moving) before the durable is + // looked up. + noAnswersBeforeCheck = 2 + // goneCheckTimeout bounds that lookup. + goneCheckTimeout = 5 * time.Second +) + +// CreateConsumer finds the operator's shard durables — it never creates one +// — and checks each against cfg: its ack_wait must cover cfg.AckWait and its +// max_ack_pending must be set, else ErrConsumerMismatch. cfg.Units picks the +// units (nil: every one, the extras still draining included). A durable name +// that does not map to the operator's, or a configured unit whose durable or +// stream is gone, is ErrConsumerNotFound; an extra whose durable is gone by +// now is skipped. +func (e *ExternalNATS) CreateConsumer(ctx context.Context, cfg ConsumerConfig) (Consumer, error) { + if _, ok := e.durable(cfg.Durable); !ok { + return nil, fmt.Errorf("consumer %q: %w: the ingest durables are %s-", cfg.Durable, ErrConsumerNotFound, e.topo.IngestConsumer) + } + names := cfg.Units + if names == nil { + configured, extra := e.IngestUnits() + names = slices.Concat(configured, extra) + } + c := &externalConsumer{e: e, ctx: ctx, whole: cfg.Units == nil, maxHeld: cfg.MaxHeld, failed: make(chan error, 1)} + for _, name := range names { + u, extra, ok := e.unit(name) + if !ok { + return nil, fmt.Errorf("consumer unit %q: %w", name, ErrUnitsUnsupported) + } + // A handle of its own: the client keeps the pin id per handle. + h, err := e.js.Consumer(ctx, u.stream, u.durable) + gone := errors.Is(err, jetstream.ErrConsumerNotFound) || errors.Is(err, jetstream.ErrStreamNotFound) + switch { + case extra && (gone || errors.Is(err, jetstream.ErrNotPullConsumer)): + continue // drained and deleted since it was listed + case gone: + return nil, fmt.Errorf("consumer %s: %w: %w", name, ErrConsumerNotFound, err) + case errors.Is(err, jetstream.ErrNotPullConsumer): + return nil, fmt.Errorf("consumer %s: %w: %w", name, ErrConsumerMismatch, err) + case err != nil: + return nil, fmt.Errorf("consumer %s: %w", name, e.apiError(err)) + } + have := h.CachedInfo().Config + if have.AckWait < cfg.AckWait { + return nil, fmt.Errorf("consumer %s: %w: ack_wait %s is shorter than the %s asked for", name, ErrConsumerMismatch, have.AckWait, cfg.AckWait) + } + if have.MaxAckPending <= 0 { + return nil, fmt.Errorf("consumer %s: %w: max_ack_pending must be set", name, ErrConsumerMismatch) + } + if extra { + slog.Info("mq: draining a shard durable outside the configured ones", "component", "nats", "unit", name, "pending", h.CachedInfo().NumPending) + } + c.parts = append(c.parts, &consumerPart{unit: u, extra: extra, h: h, ackWait: have.AckWait}) + } + return c, nil +} + +// externalConsumer is the operator's shard durables it was created for. +// +// Each unit has two goroutines. A puller fetches from the server and never +// runs the handler, so it keeps pulling, and with that keeps the unit's pin, +// however long the handler takes. A deliverer hands what was fetched to the +// handler, in order. The puller fetches only as much as the unit's cap +// (MaxHeld, and prefetch for what waits for the handler) leaves room for; at +// the cap it renews the pin every renewEvery with a one-row fetch, up to one +// ack_wait's worth of renewals past the cap, and past that, and once halted, +// with pulls that deliver nothing (max_bytes 1: the server holds any row +// back). +type externalConsumer struct { + e *ExternalNATS + ctx context.Context + // whole is a consumer of every unit, the one a process with no claims + // runs: it takes orphaned units over itself, and releases them on stop. + whole bool + maxHeld func() int + parts []*consumerPart + failed chan error + // reported and stopped keep failed to one error, none after stop. + reported, stopped atomic.Bool +} + +type consumerPart struct { + unit natsUnit + // extra is outside the configured units: its delivery ending is the + // operator deleting it once drained, not a failure. + extra bool + h jetstream.Consumer + // ackWait is the durable's: a row not settled by then is the server's to + // redeliver, so it no longer counts against the cap. + ackWait time.Duration + // pin is the pin id of the last row this process received from the + // unit, nil before the first or once the server said it lost the pin. + pin atomic.Pointer[string] + + share int + maxHeld func() int + + mu sync.Mutex + // queue is fetched and not yet handed to the handler; held is fetched + // and not yet settled, queue included. + queue []jetstream.Msg + held int + + // wake tells the puller a row settled; ready tells the deliverer rows + // are queued. + wake, ready chan struct{} + // halting stops fetching; end stops renewing the pin. fetched closes + // once nothing more will be queued, drained once the deliverer has + // handed everything on. + halting, end, fetched, drained chan struct{} + haltOnce, endOnce sync.Once + // lastPull is when the puller last sent the server a pull (its + // goroutine's only). + lastPull time.Time +} + +func (p *consumerPart) start(share int, maxHeld func() int) { + p.share, p.maxHeld = share, maxHeld + p.wake, p.ready = make(chan struct{}, 1), make(chan struct{}, 1) + p.halting, p.end = make(chan struct{}), make(chan struct{}) + p.fetched, p.drained = make(chan struct{}), make(chan struct{}) +} + +func signal(ch chan struct{}) { + select { + case ch <- struct{}{}: + default: + } +} + +func (p *consumerPart) halt() { p.haltOnce.Do(func() { close(p.halting) }) } +func (p *consumerPart) endRenewals() { p.endOnce.Do(func() { close(p.end) }) } + +func (p *consumerPart) halted() bool { + select { + case <-p.halting: + return true + default: + return false + } +} + +// room is how many rows the unit may fetch now. +func (p *consumerPart) room() int { + p.mu.Lock() + defer p.mu.Unlock() + n := p.share - len(p.queue) + if p.maxHeld != nil { + n = min(n, p.maxHeld()-p.held) + } + return n +} + +// mayOvershoot reports whether a unit at its cap may renew its pin by taking +// one more row: while the rows so taken, each counted until it settles or its +// ack_wait passes, stay under one ack_wait of renewals past the cap, both +// MaxHeld and the prefetch share of rows waiting for the handler. Past +// that it renews without taking a row, which can reorder due redeliveries +// (see renewRequest), rather than hold more. +func (p *consumerPart) mayOvershoot() bool { + p.mu.Lock() + defer p.mu.Unlock() + if p.maxHeld == nil { + return false + } + k := max(1, int(p.ackWait/renewEvery)) + return p.held-p.maxHeld() < k && len(p.queue)-p.share < k +} + +func (p *consumerPart) enqueue(m jetstream.Msg) { + if pin := m.Headers().Get(pinIDHeader); pin != "" { + p.pin.Store(&pin) + } + p.mu.Lock() + p.queue = append(p.queue, m) + p.held++ + p.mu.Unlock() + signal(p.ready) +} + +func (p *consumerPart) settle() { + p.mu.Lock() + p.held-- + p.mu.Unlock() + signal(p.wake) +} + +// next is the next row for the handler, false once fetching has ended and +// every row was handed on. +func (p *consumerPart) next() (jetstream.Msg, bool) { + for { + p.mu.Lock() + if len(p.queue) > 0 { + m := p.queue[0] + p.queue[0] = nil + p.queue = p.queue[1:] + p.mu.Unlock() + return m, true + } + p.mu.Unlock() + select { + case <-p.ready: + case <-p.fetched: + p.mu.Lock() + empty := len(p.queue) == 0 + p.mu.Unlock() + if empty { + return nil, false + } + } + } +} + +// Consume pulls every unit in the priority group, so the server delivers a +// unit to one pinned puller at a time. Prefetch is split between the +// configured units (at least one each; 0 is the client's default per unit), +// and an extra gets a quarter share. A unit's delivery that ends on its own +// — the durable or its stream deleted, the connection closed for good — is +// reported on failed; an extra's is only logged. stop halts every unit +// without waiting (Halt waits); a consumer of every unit then releases each +// once it has handed on what it fetched, while one of chosen units keeps +// renewing their pins until Release, or until a durable's ack_wait and one +// renewal have passed since its halt handed on what it fetched. +func (c *externalConsumer) Consume(handler func(msg *Message), prefetch int) (func(), <-chan error, error) { + own := 0 + for _, part := range c.parts { + if !part.extra { + own++ + } + } + if c.whole { + c.takeOver() + } + for _, part := range c.parts { + share := jetstream.DefaultMaxMessages + if prefetch > 0 { + share = max(1, prefetch/max(1, own)) + } + if part.extra { + share = max(1, share/4) + } + part.start(share, c.maxHeld) + go c.run(part) + go c.deliver(part, handler) + } + untrack := c.e.track(c.shutdown) + return func() { + untrack() + c.stopped.Store(true) + for _, part := range c.parts { + part.halt() + if c.whole { + go func() { + <-part.drained + c.releaseAfterDrain(part) + }() + } + } + }, c.failed, nil +} + +// Halt implements Halter: every unit stops fetching, and Halt returns once +// what was fetched has reached the handler, which may take one fetch's +// expiry. The units keep their pins. +func (c *externalConsumer) Halt() { + c.stopped.Store(true) + for _, part := range c.parts { + part.halt() + } + for _, part := range c.parts { + <-part.drained + } +} + +// shutdown is Close's stop: it halts and ends the renewals without waiting. +func (c *externalConsumer) shutdown() { + c.stopped.Store(true) + for _, part := range c.parts { + part.halt() + part.endRenewals() + } +} + +func (c *externalConsumer) run(part *consumerPart) { + ended := c.pull(part) + close(part.fetched) + if !ended { + c.hold(part) + } +} + +// pull fetches part until it is halted, reporting whether its delivery ended +// for good. +func (c *externalConsumer) pull(part *consumerPart) bool { + noAnswers := 0 + for !part.halted() { + n := part.room() + if n <= 0 { + if !c.renewalDue(part) { + continue + } + if !part.mayOvershoot() { + if c.renew(part, *part.pin.Load()) { + return true + } + continue + } + // Renewed by taking one row past the cap: a pull that held a + // due redelivery back would have the server requeue it behind + // the others, reordering them. + n = 1 + } + part.lastPull = time.Now() + batch, err := part.h.Fetch(n, jetstream.FetchMaxWait(fetchExpiry), jetstream.FetchPriorityGroup(natsPriorityGroup)) + if err == nil { + for m := range batch.Messages() { + part.enqueue(m) + } + err = batch.Error() + } + switch { + case err == nil || errors.Is(err, nats.ErrTimeout): + noAnswers = 0 + case errors.Is(err, jetstream.ErrPinIDMismatch) || errors.Is(err, jetstream.ErrConsumerLeadershipChanged): + // Another puller holds the pin, or the consumer's leader moved: + // the next pull tries again, and nothing is lost. + noAnswers = 0 + slog.Debug("mq: shard delivery paused", "component", "nats", "unit", part.unit.id(), "reason", err) + case errors.Is(err, nats.ErrNoResponders): + if noAnswers++; noAnswers >= noAnswersBeforeCheck && c.gone(part) { + return true + } + c.pause(part, fetchRetryWait) + default: + if c.endedBy(part, err) { + return true + } + level := slog.LevelWarn + if part.extra { + level = slog.LevelInfo + } + slog.Log(context.Background(), level, "mq: consumer reported an error", "component", "nats", "unit", part.unit.id(), "error", err) + c.pause(part, fetchRetryWait) + } + } + return false +} + +// renewalDue, for a unit at its cap, reports whether its pin is due a +// renewal, and otherwise waits for a row to settle, a halt, or the renewal. +func (c *externalConsumer) renewalDue(part *consumerPart) bool { + if part.pin.Load() == nil { + c.pause(part, 0) // never received: no pin to keep + return false + } + if due := time.Until(part.lastPull.Add(renewEvery)); due > 0 { + c.pause(part, due) + return false + } + return true +} + +// pause waits up to d (0: no bound) for a row to settle, a halt, or the +// end of the renewals. +func (c *externalConsumer) pause(part *consumerPart, d time.Duration) { + var timeout <-chan time.Time + if d > 0 { + t := time.NewTimer(d) + defer t.Stop() + timeout = t.C + } + select { + case <-part.wake: + case <-part.halting: + case <-part.end: + case <-timeout: + } +} + +// hold keeps a halted unit's pin, so that no other process receives from it +// while this one writes what it holds, until Release, Close, the pin is lost, +// or the durable's ack_wait has passed since what it fetched reached the +// handler, and one renewal more, so that a caller's own wait of ack_wait from +// then (a handover's) ends before the renewals do. By then the server +// redelivers what this process still holds anyway. +func (c *externalConsumer) hold(part *consumerPart) { + var until time.Time + for { + if until.IsZero() { + select { + case <-part.drained: + until = time.Now().Add(part.ackWait + renewEvery) + default: + } + } + pin := part.pin.Load() + if pin == nil || (!until.IsZero() && !time.Now().Before(until)) { + return + } + t := time.NewTimer(max(0, time.Until(part.lastPull.Add(renewEvery)))) + select { + case <-part.end: + t.Stop() + return + case <-t.C: + } + if c.renew(part, *pin) { + return + } + } +} + +// renewRequest is a pull the server answers with no row: max_bytes 1 is +// below any row's size, so it holds the row back (409) and keeps it where it +// is. It still resets the pin's timer, which the server does on any pull that +// carries the current pin id. +type renewRequest struct { + Expires time.Duration `json:"expires"` + Batch int `json:"batch"` + MaxBytes int `json:"max_bytes"` + Group string `json:"group"` + ID string `json:"id"` +} + +// renew sends part's pin-keeping pull, reporting whether the unit's delivery +// ended for good (its durable deleted). +func (c *externalConsumer) renew(part *consumerPart, pin string) bool { + part.lastPull = time.Now() + body, err := json.Marshal(renewRequest{Expires: renewWait, Batch: 1, MaxBytes: 1, Group: natsPriorityGroup, ID: pin}) + if err != nil { + return false + } + ctx, cancel := context.WithTimeout(context.Background(), renewWait+time.Second) + defer cancel() + subj := c.e.apiPrefix + "CONSUMER.MSG.NEXT." + part.unit.stream + "." + part.unit.durable + resp, err := c.e.nc.RequestWithContext(ctx, subj, body) + switch { + case errors.Is(err, nats.ErrNoResponders): + return c.gone(part) + case err != nil: + if c.endedBy(part, err) { + return true + } + slog.Debug("mq: a shard's pin renewal got no answer", "component", "nats", "unit", part.unit.id(), "error", err) + return false + } + status, descr := resp.Header.Get("Status"), strings.ToLower(resp.Header.Get("Description")) + switch { + case status == "409" && strings.Contains(descr, "consumer deleted"): + c.end(part, jetstream.ErrConsumerDeleted) + return true + case status == "423": + // The pin lapsed or went to another puller: this unit no longer + // holds it, and the next fetch pins afresh if it can. + if cur := part.pin.Load(); cur != nil && *cur == pin { + part.pin.Store(nil) + } + slog.Debug("mq: a shard's pin was lost while it fetched nothing", "component", "nats", "unit", part.unit.id()) + } + // 408 (nothing waiting), 409 over max_bytes (a row held back) or a + // leadership change: the pin's timer was reset, or the next pull re-pins. + return false +} + +// gone looks a durable that answers no pull up: deleted, or its stream +// deleted, ends the unit's delivery. Anything else (its leader moving, the +// JetStream API not answering either) is tried again. +func (c *externalConsumer) gone(part *consumerPart) bool { + ctx, cancel := context.WithTimeout(context.Background(), goneCheckTimeout) + defer cancel() + _, err := part.h.Info(ctx) + if errors.Is(err, jetstream.ErrConsumerNotFound) || errors.Is(err, jetstream.ErrStreamNotFound) { + c.end(part, fmt.Errorf("%w: %w", ErrConsumerNotFound, err)) + return true + } + if err != nil && !c.stopped.Load() { + slog.Warn("mq: a shard's pulls get no answer; trying again", "component", "nats", "unit", part.unit.id(), "error", err) + } + return false +} + +// endedBy reports whether err ends part's delivery for good, and reports it +// if so: the durable deleted under a pull, or the connection closed for good. +func (c *externalConsumer) endedBy(part *consumerPart, err error) bool { + switch { + case errors.Is(err, jetstream.ErrConsumerDeleted) || errors.Is(err, jetstream.ErrBadRequest): + c.end(part, err) + return true + case errors.Is(err, nats.ErrConnectionClosed) || errors.Is(err, nats.ErrConnectionDraining) || c.e.nc.IsClosed(): + if !c.e.closing.Load() { + c.end(part, nats.ErrConnectionClosed) + } + return true + } + return false +} + +// end reports part's delivery ending on its own: a configured unit's on +// failed, an extra's only in the log. +func (c *externalConsumer) end(part *consumerPart, reason error) { + err := fmt.Errorf("%w: %w", ErrDeliveryEnded, reason) + if part.extra { + slog.Info("mq: stopped draining a shard durable outside the configured ones", "component", "nats", "unit", part.unit.id(), "reason", err) + return + } + c.fail(fmt.Errorf("%s: %w", part.unit.id(), err)) +} + +// deliver hands part's fetched rows to handler, in order, each counted +// against the unit's cap until it settles or its ack_wait passes. +func (c *externalConsumer) deliver(part *consumerPart, handler func(*Message)) { + defer close(part.drained) + for { + m, ok := part.next() + if !ok { + return + } + msg := c.e.wrapMsg(c.ctx, m, true) + letGo := sync.OnceFunc(part.settle) + expire := time.AfterFunc(part.ackWait, letGo) + msg.OnSettled(func() { expire.Stop(); letGo() }) + handler(msg) + } +} + +// takeOver resets every unit no one holds that has rows awaiting an ack, +// before this consumer pulls it: whoever received them is gone. +func (c *externalConsumer) takeOver() { + ctx, cancel := context.WithTimeout(c.ctx, recheckTimeout) + defer cancel() + for _, part := range c.parts { + if reset, err := c.e.ResetOrphaned(ctx, part.unit.id()); errors.Is(err, ErrUnitHeld) { + continue // another consumer holds it; the pin decides + } else if err != nil { + slog.Warn("mq: could not take over a shard's unsettled rows; they come back after ack_wait", "component", "nats", "unit", part.unit.id(), "error", err) + } else if reset { + slog.Info("mq: took over a shard's unsettled rows", "component", "nats", "unit", part.unit.id()) + } + } +} + +// releaseAfterDrain releases part once its halt has delivered what it +// fetched. A whole consumer's handler has no settlement to wait for here, +// and an unsettled row comes back to the next owner after ack_wait. +func (c *externalConsumer) releaseAfterDrain(part *consumerPart) { + defer part.endRenewals() + if c.e.nc.IsClosed() { + return + } + ctx, cancel := context.WithTimeout(context.Background(), releaseTimeout) + defer cancel() + if err := c.e.release(ctx, part); err != nil { + slog.Debug("mq: could not release a shard; its pin lapses on its own", "component", "nats", "unit", part.unit.id(), "error", err) + } +} + +// Release implements Releaser: each unit this consumer was last pinned on, +// and whose pin the server still gives it, is unpinned; then the unit stops +// renewing its pin. +func (c *externalConsumer) Release(ctx context.Context) error { + var errs []error + for _, part := range c.parts { + errs = append(errs, c.e.release(ctx, part)) + part.endRenewals() + } + return errors.Join(errs...) +} + +// release unpins part if this process still holds its pin. Checking first +// keeps a process that never received from the unit, or lost the pin since, +// from unpinning the real owner. +// +// The check and the unpin are two requests, and the unpin names no pin id: +// if the pin moved to another client between them, that client is unpinned +// and pins again with its next pull. A halted unit renews its pin until its +// release returns, if that comes within ack_wait of the halt's drain (a +// handover's bound), so the pin cannot lapse in between; only a consumer +// leader change there (which releases the pin itself) opens that window. +func (e *ExternalNATS) release(ctx context.Context, part *consumerPart) error { + pin := part.pin.Load() + if pin == nil { + return nil + } + info, err := part.h.Info(ctx) + if err != nil { + return fmt.Errorf("consumer %s: %w", part.unit.id(), e.apiError(err)) + } + if pinnedClient(info) != *pin { + return nil + } + s, err := e.js.Stream(ctx, part.unit.stream) + if err != nil { + return fmt.Errorf("stream %s: %w", part.unit.stream, e.apiError(err)) + } + if err := s.UnpinConsumer(ctx, part.unit.durable, natsPriorityGroup); err != nil { + return fmt.Errorf("unpin %s: %w", part.unit.id(), e.apiError(err)) + } + return nil +} + +func (c *externalConsumer) fail(err error) { + if c.stopped.Load() || !c.reported.CompareAndSwap(false, true) { + return + } + c.failed <- err +} diff --git a/internal/mq/external_export_test.go b/internal/mq/external_export_test.go new file mode 100644 index 000000000..1c33d454b --- /dev/null +++ b/internal/mq/external_export_test.go @@ -0,0 +1,129 @@ +//go:build integration + +package mq + +import ( + "strconv" + "testing" + "time" + + "github.com/Wave-RF/WaveHouse/internal/tenant" + "github.com/stretchr/testify/require" +) + +// ExternalFixture is the external-NATS fixture as the conformance run (in +// mq_test, since mqtest imports mq) sees it: a server with the shipped +// topology, and the operator's hand on it. +type ExternalFixture struct { + f *natsFixture +} + +// NewExternalFixture starts a server from the shipped Helm values and applies +// the shipped manifests to it. +func NewExternalFixture(t *testing.T) *ExternalFixture { + t.Helper() + f := newNATSFixture(t) + f.apply(t, shippedTopology(t)) + return &ExternalFixture{f: f} +} + +// Broker connects an ExternalNATS as the restricted wavehouse user, closed +// by the test framework. +func (x *ExternalFixture) Broker(t *testing.T) Broker { + t.Helper() + return x.f.broker(t, nil) +} + +// DeleteIngestDurable deletes every shard durable of every partition, as the +// operator could. +func (x *ExternalFixture) DeleteIngestDurable(t *testing.T) { + t.Helper() + for p := range 4 { + for s := range 8 { + require.NoError(t, x.f.admin.DeleteConsumer(t.Context(), shippedPartition(p), natsShardDurable(DefaultNATSIngestConsumer, s))) + } + } +} + +// FillPartition shrinks every partition to a few KiB and publishes id's +// events into each until it refuses even the smallest: a tenant's tables +// spread over the partitions, so any of them then finds its own full. +func (x *ExternalFixture) FillPartition(t *testing.T, b Broker, id tenant.ID) { + t.Helper() + for p := range 4 { + x.f.shrink(t, shippedPartition(p), 4<<10) + topic := topicIn(t, id, p, 4, 8) + for _, size := range []int{1 << 10, 1} { + payload := make([]byte, size) + for i := 0; ; i++ { + require.Less(t, i, 1<<10, "partition %d never filled", p) + err := b.Publish(t.Context(), topic, payload) + if err != nil { + require.ErrorIs(t, err, ErrQueueFull) + break + } + } + } + } +} + +// topicIn is a table of id's whose events go to partition p of n (v shards). +func topicIn(t *testing.T, id tenant.ID, p, n, v int) Topic { + t.Helper() + for i := range 1000 { + topic := Topic{Tenant: id, Table: "t" + strconv.Itoa(i)} + if got, _ := natsRoute(topic, n, v); got == p { + return topic + } + } + t.Fatalf("no table of %s in partition %d of %d", id, p, n) + return Topic{} +} + +// shippedPartition is partition p's stream in the shipped manifests. +func shippedPartition(p int) string { return "WH_INGEST_" + strconv.Itoa(p) } + +// broker connects an ExternalNATS to f as the wavehouse user, with cfg's +// fields over the fixture's, closed by the test framework. +func (f *natsFixture) broker(t *testing.T, edit func(*NATSConfig)) *ExternalNATS { + t.Helper() + cfg := NATSConfig{ + URLs: []string{f.server.ClientURL()}, + User: "wavehouse", + PasswordFile: writeSecret(t, fixturePassword("wavehouse")), + Topology: NATSTopology{Partitions: 4, Shards: 8}, + TopologyWait: 10 * time.Second, + } + if edit != nil { + edit(&cfg) + } + e, err := NewNATS(t.Context(), cfg) + require.NoError(t, err) + t.Cleanup(func() { _ = e.Close() }) + return e +} + +// shrink sets a stream's max_bytes, as the operator could. +func (f *natsFixture) shrink(t *testing.T, stream string, maxBytes int64) { + t.Helper() + s, err := f.admin.Stream(t.Context(), stream) + require.NoError(t, err) + cfg := s.CachedInfo().Config + cfg.MaxBytes = maxBytes + _, err = f.admin.UpdateStream(t.Context(), cfg) + require.NoError(t, err) +} + +// streamMsgs is how many messages a stream holds. +func (f *natsFixture) streamMsgs(t *testing.T, stream string) uint64 { + t.Helper() + s, err := f.admin.Stream(t.Context(), stream) + require.NoError(t, err) + return s.CachedInfo().State.Msgs +} + +// mustPartition is the shipped partition topic's table lives in. +func mustPartition(topic Topic) int { + p, _ := natsRoute(topic, 4, 8) + return p +} diff --git a/internal/mq/external_perms_test.go b/internal/mq/external_perms_test.go new file mode 100644 index 000000000..434d47e35 --- /dev/null +++ b/internal/mq/external_perms_test.go @@ -0,0 +1,289 @@ +//go:build integration + +package mq + +import ( + "context" + "fmt" + "net/url" + "os" + "path/filepath" + "strings" + "testing" + "time" + + natsserver "github.com/nats-io/nats-server/v2/server" + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gopkg.in/yaml.v3" + + "github.com/Wave-RF/WaveHouse/internal/coord" + "github.com/Wave-RF/WaveHouse/internal/mq/natstest" +) + +// Boot refuses a permission set narrower than the shard count, the +// validator's case: the shipped values name 8 shards' durables, and the +// topology has 16. Without the check the broker boots, publishes to shards +// 8-15 succeed, and nothing ever pulls them. +func TestNewNATS_RefusesPermissionsNarrowerThanTheShards(t *testing.T) { + t.Parallel() + f := newNATSFixture(t) + f.apply(t, generatedTopology(t, 4, 16)) + _, err := NewNATS(t.Context(), NATSConfig{ + URLs: []string{f.server.ClientURL()}, User: "wavehouse", PasswordFile: writeSecret(t, fixturePassword("wavehouse")), + Topology: NATSTopology{Partitions: 4, Shards: 16}, TopologyWait: time.Second, + }) + require.ErrorIs(t, err, ErrTopology) + var te *TopologyError + require.ErrorAs(t, err, &te) + assert.Contains(t, err.Error(), "required: consumer WH_INGEST_3/wh-ingest-15: permissions: the connecting user may not publish to $JS.API.CONSUMER.") + assert.Contains(t, err.Error(), "wavehouse mq permissions --shards 16") + assert.NotContains(t, err.Error(), "wh-ingest-7: permissions") + assert.Equal(t, 4*8, countRequired(te.Findings)) +} + +// The same under a JetStream domain, on a server in it: the probe goes out +// as $JS.hub.API.…, and the server refuses it under the plain form it maps +// that to. +func TestNewNATS_RefusesNarrowPermissionsUnderAJSDomain(t *testing.T) { + t.Parallel() + f := newNATSFixtureFrom(t, valuesWithPermissions(t, 8, "hub"), func(o *natsserver.Options) { o.JetStreamDomain = "hub" }) + f.apply(t, generatedTopology(t, 4, 16)) + _, err := NewNATS(t.Context(), NATSConfig{ + URLs: []string{f.server.ClientURL()}, User: "wavehouse", PasswordFile: writeSecret(t, fixturePassword("wavehouse")), + Topology: NATSTopology{Partitions: 4, Shards: 16}, JSDomain: "hub", TopologyWait: time.Second, + }) + require.ErrorIs(t, err, ErrTopology) + var te *TopologyError + require.ErrorAs(t, err, &te) + assert.Contains(t, err.Error(), "consumer WH_INGEST_3/wh-ingest-15: permissions") + assert.Contains(t, err.Error(), "wavehouse mq permissions --shards 16 --js-domain hub") + assert.Equal(t, 4*8, countRequired(te.Findings)) +} + +// A consumer request the server refuses after boot marks the topology +// faulty at once, before the next re-check names it. +func TestExternalNATS_RefusedConsumerRequestIsATopologyFault(t *testing.T) { + t.Parallel() + f := shippedFixture(t) + e := f.broker(t, nil) + require.True(t, e.topologyOK.Load()) + require.NoError(t, e.nc.Publish("$JS.API.CONSUMER.MSG.NEXT.WH_INGEST_0.wh-ingest-9", nil)) + require.Eventually(t, func() bool { return !e.topologyOK.Load() }, 5*time.Second, 10*time.Millisecond) + + // A refused request to another API is not a topology fault. + g := shippedFixture(t) + e2 := g.broker(t, nil) + require.NoError(t, e2.nc.Publish("$JS.API.STREAM.DELETE.WH_DLQ", nil)) + require.NoError(t, e2.nc.Flush()) + require.Eventually(t, func() bool { return e2.perms.deniedSince("$JS.API.STREAM.DELETE.WH_DLQ", 0) }, 5*time.Second, 10*time.Millisecond) + assert.True(t, e2.topologyOK.Load()) +} + +// The shipped permissions let the wavehouse user ack only what its shard +// durables deliver: an ack for a consumer of its own on the history (which +// it may create and pull) is refused, and the row stays pending. +func TestNATSPermissions_RefuseAcksOutsideTheShards(t *testing.T) { + t.Parallel() + f := shippedFixture(t) + e := f.broker(t, nil) + require.NoError(t, e.Publish(t.Context(), Topic{Tenant: "acme", Table: "t"}, []byte("row"))) + + js := f.connect(t, "wavehouse", nats.ErrorHandler(func(*nats.Conn, *nats.Subscription, error) {})) + c, err := js.CreateConsumer(t.Context(), "WH_HISTORY", jetstream.ConsumerConfig{AckPolicy: jetstream.AckExplicitPolicy, InactiveThreshold: time.Minute}) + require.NoError(t, err) + msg, err := c.Next(jetstream.FetchMaxWait(5 * time.Second)) + require.NoError(t, err) + require.True(t, strings.HasPrefix(msg.Reply(), "$JS.ACK."), msg.Reply()) + require.Error(t, call(t.Context(), msg.DoubleAck), "an ack outside the shard durables is refused") + info, err := f.admin.Consumer(t.Context(), "WH_HISTORY", c.CachedInfo().Name) + require.NoError(t, err) + assert.Equal(t, 1, info.CachedInfo().NumAckPending) +} + +// Rows are acked under either ack subject layout the server may use: the +// default (v1) and the js_ack_fc_v2 feature flag's, which puts the domain and +// the account ahead of the stream and consumer. +func TestExternalNATS_AcksUnderBothAckSubjectLayouts(t *testing.T) { + t.Parallel() + for _, v2 := range []bool{false, true} { + t.Run(fmt.Sprintf("v2=%t", v2), func(t *testing.T) { + t.Parallel() + f := newNATSFixtureFrom(t, natstest.ShippedValues(), func(o *natsserver.Options) { + o.FeatureFlags = map[string]bool{"js_ack_fc_v2": v2} + }) + f.apply(t, shippedTopology(t)) + drainsWithAcks(t, f, f.broker(t, nil)) + + // The layout the server used, by token count. + require.NoError(t, f.broker(t, nil).Publish(t.Context(), Topic{Tenant: "acme", Table: "t"}, []byte("row"))) + c, err := f.admin.CreateConsumer(t.Context(), "WH_HISTORY", jetstream.ConsumerConfig{AckPolicy: jetstream.AckExplicitPolicy}) + require.NoError(t, err) + msg, err := c.Next(jetstream.FetchMaxWait(5 * time.Second)) + require.NoError(t, err) + tokens := map[bool]int{false: 9, true: 11}[v2] + assert.Len(t, strings.Split(msg.Reply(), "."), tokens, msg.Reply()) + }) + } +} + +// drainsWithAcks publishes a row, consumes and acks it through e, and +// requires the work-queue partition to have dropped it: the ack was taken. +func drainsWithAcks(t *testing.T, f *natsFixture, e *ExternalNATS) { + t.Helper() + topic := Topic{Tenant: "acme", Table: "t"} + require.NoError(t, e.Publish(t.Context(), topic, []byte("row"))) + cons, err := e.CreateConsumer(t.Context(), ConsumerConfig{Durable: workerDurable}) + require.NoError(t, err) + got := make(chan string, 4) + stop, _, err := cons.Consume(func(m *Message) { + assert.NoError(t, m.DoubleAck(m.Ctx)) + got <- string(m.Data) + }, 4) + require.NoError(t, err) + t.Cleanup(stop) + require.Equal(t, "row", receive(t, got)) + partition := shippedPartition(mustPartition(topic)) + require.Eventually(t, func() bool { return f.streamMsgs(t, partition) == 0 }, 5*time.Second, 10*time.Millisecond) +} + +// Under mq.nats.js_domain, WaveHouse connected to a server in the domain +// boots, consumes, acks and holds a lease with `wavehouse mq permissions +// --js-domain`: that server maps $JS..API.… to $JS.API.… before it +// checks permissions. +func TestNewNATS_JSDomainOnAServerInTheDomain(t *testing.T) { + t.Parallel() + f := newNATSFixtureFrom(t, valuesWithPermissions(t, 8, "hub"), func(o *natsserver.Options) { o.JetStreamDomain = "hub" }) + f.apply(t, shippedTopology(t)) + e := f.broker(t, func(c *NATSConfig) { c.JSDomain = "hub" }) + drainsWithAcks(t, f, e) + holdsALease(t, e) +} + +// Through a leafnode outside the domain, the permissions are checked on the +// subjects as sent, $JS..API.…, which a set without --js-domain does +// not allow. +func TestNewNATS_JSDomainThroughALeafnode(t *testing.T) { + t.Parallel() + hub := domainHub(t) + l := domainLeaf(t, hub, "hub") + e := l.broker(t, func(c *NATSConfig) { c.JSDomain = "hub" }) + drainsWithAcks(t, l, e) + holdsALease(t, e) + + plain := domainLeaf(t, hub, "") + _, err := NewNATS(t.Context(), NATSConfig{ + URLs: []string{plain.server.ClientURL()}, User: "wavehouse", PasswordFile: writeSecret(t, fixturePassword("wavehouse")), + Topology: NATSTopology{Partitions: 4, Shards: 8}, JSDomain: "hub", TopologyWait: time.Second, + }) + require.ErrorContains(t, err, "the server refused publishes to $JS.hub.API.", "a set without the domain's subjects lets nothing through the leafnode") +} + +// A halted shard keeps its pin with raw pulls sent under the domain's API +// prefix. The generated permissions must allow them in both forms: a server +// in the domain checks the plain one, a leafnode the one as sent. A refused +// renewal would let the pin lapse after its 10s TTL and be recorded as a +// refusal. +func TestExternalNATS_PinRenewalUnderAJSDomain(t *testing.T) { + t.Parallel() + renews := func(t *testing.T, f *natsFixture) { + t.Helper() + e := f.broker(t, func(c *NATSConfig) { c.JSDomain = "hub" }) + topic := Topic{Tenant: "acme", Table: "events"} + p, s := natsRoute(topic, 4, 8) + c, stop, got := unitConsumer(t, e, shippedPartition(p)+"/"+natsShardDurable("wh-ingest", s), nil) + require.NoError(t, e.Publish(t.Context(), topic, []byte("1"))) + require.Equal(t, "1", receive(t, got)) + pin := unitPin(t, f, topic) + require.NotEmpty(t, pin) + c.(Halter).Halt() + time.Sleep(15 * time.Second) // past the pinned TTL: only the renewals keep it + assert.Equal(t, pin, unitPin(t, f, topic), "the renewals kept the pin") + assert.Empty(t, e.perms.list(), "the server refused no request") + stop() + } + t.Run("server in the domain", func(t *testing.T) { + t.Parallel() + f := newNATSFixtureFrom(t, valuesWithPermissions(t, 8, "hub"), func(o *natsserver.Options) { o.JetStreamDomain = "hub" }) + f.apply(t, shippedTopology(t)) + renews(t, f) + }) + t.Run("leafnode outside it", func(t *testing.T) { + t.Parallel() + renews(t, domainLeaf(t, domainHub(t), "hub")) + }) +} + +// domainHub is a server in JetStream domain hub holding the shipped +// topology, with a leafnode port. +func domainHub(t *testing.T) *natsFixture { + t.Helper() + hub := newNATSFixtureFrom(t, natstest.ShippedValues(), func(o *natsserver.Options) { + o.JetStreamDomain = "hub" + o.LeafNode.Host, o.LeafNode.Port = "127.0.0.1", -1 + }) + hub.apply(t, shippedTopology(t)) + return hub +} + +// domainLeaf is a leafnode of hub without JetStream, whose wavehouse user has +// the permissions generated for domain. +func domainLeaf(t *testing.T, hub *natsFixture, domain string) *natsFixture { + t.Helper() + varz, err := hub.server.Varz(nil) + require.NoError(t, err) + remote, err := url.Parse(fmt.Sprintf("nats-leaf://%s:%s@127.0.0.1:%d", natstest.OperatorUser, fixturePassword(natstest.OperatorUser), varz.LeafNode.Port)) + require.NoError(t, err) + l := newNATSFixtureFrom(t, valuesWithPermissions(t, 8, domain), func(o *natsserver.Options) { + o.JetStream = false + o.LeafNode.Remotes = []*natsserver.RemoteLeafOpts{{URLs: []*url.URL{remote}, LocalAccount: "WAVEHOUSE"}} + }) + require.Eventually(t, func() bool { return l.server.NumLeafNodes() == 1 }, 10*time.Second, 10*time.Millisecond) + l.admin = hub.admin // the topology lives on the hub + return l +} + +// holdsALease acquires a lease in the shipped lease bucket through e, and +// a second holder then reads it as held: a KV write and a direct get. +func holdsALease(t *testing.T, e *ExternalNATS) { + t.Helper() + coordinator := func(holder string) coord.Coordinator { + c, err := e.Leases(t.Context(), natstest.CoordBucket, holder) + require.NoError(t, err) + t.Cleanup(func() { _ = c.Close(context.Background()) }) + return c + } + _, err := coordinator("holder-a").TryAcquire(t.Context(), "probe") + require.NoError(t, err) + _, err = coordinator("holder-b").TryAcquire(t.Context(), "probe") + require.ErrorIs(t, err, coord.ErrHeld) +} + +// valuesWithPermissions writes the shipped Helm values with the wavehouse +// user's permissions generated for shards and domain. +func valuesWithPermissions(t *testing.T, shards int, domain string) string { + t.Helper() + raw, err := os.ReadFile(natstest.ShippedValues()) + require.NoError(t, err) + var values map[string]any + require.NoError(t, yaml.Unmarshal(raw, &values)) + p := natsPermissions(NATSTopology{Shards: shards}, domain) + accounts := values["config"].(map[string]any)["merge"].(map[string]any)["accounts"].(map[string]any) + for _, acc := range accounts { + for _, u := range acc.(map[string]any)["users"].([]any) { + if user := u.(map[string]any); user["user"] == natstest.WaveHouseUser { + user["permissions"] = map[string]any{ + "publish": map[string][]string{"allow": p.PublishAllow, "deny": p.PublishDeny}, + "subscribe": map[string][]string{"allow": p.SubscribeAllow}, + } + } + } + } + out, err := yaml.Marshal(values) + require.NoError(t, err) + path := filepath.Join(t.TempDir(), "values.yaml") + require.NoError(t, os.WriteFile(path, out, 0o600)) + return path +} diff --git a/internal/mq/external_test.go b/internal/mq/external_test.go new file mode 100644 index 000000000..a4e8f799d --- /dev/null +++ b/internal/mq/external_test.go @@ -0,0 +1,1279 @@ +//go:build integration + +package mq + +import ( + "bytes" + "context" + "errors" + "log/slog" + "net" + "os" + "path/filepath" + "slices" + "strconv" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/Wave-RF/WaveHouse/internal/tenant" + "github.com/Wave-RF/WaveHouse/internal/testutil/logtest" + natsserver "github.com/nats-io/nats-server/v2/server" + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel" + sdkmetric "go.opentelemetry.io/otel/sdk/metric" + "go.opentelemetry.io/otel/sdk/metric/metricdata" +) + +// shippedFixture is a fixture server with the shipped topology applied. +func shippedFixture(t *testing.T) *natsFixture { + t.Helper() + f := newNATSFixture(t) + f.apply(t, shippedTopology(t)) + return f +} + +// restart stops the server and starts it again on the same port and store. +func (f *natsFixture) restart(t *testing.T) { + t.Helper() + if f.server.Running() { + f.stop() + } + s, err := natsserver.NewServer(f.opts.Clone()) + require.NoError(t, err) + s.Start() + require.True(t, s.ReadyForConnections(10*time.Second), "nats server not ready after restart") + t.Cleanup(s.Shutdown) + f.server = s +} + +// stop shuts the server down, keeping its port for restart. +func (f *natsFixture) stop() { + f.opts = f.opts.Clone() + f.opts.Port = f.server.Addr().(*net.TCPAddr).Port + f.server.Shutdown() + f.server.WaitForShutdown() +} + +// writeSecret writes a secret file, as a mounted Kubernetes Secret would be. +func writeSecret(t *testing.T, secret string) string { + t.Helper() + path := filepath.Join(t.TempDir(), "secret") + require.NoError(t, os.WriteFile(path, []byte(secret+"\n"), 0o600)) + return path +} + +// The broker rides out a server restart: a publish while the server is down +// is ErrUnavailable, not a 500's plain error, and the next one is stored; +// consumption resumes on its own, and failed stays quiet. +func TestExternalNATS_Reconnect(t *testing.T) { + t.Parallel() + f := shippedFixture(t) + e := f.broker(t, func(c *NATSConfig) { c.Topology.PublishTimeout = 300 * time.Millisecond }) + topic := Topic{Tenant: "acme", Table: "t"} + + cons, err := e.CreateConsumer(t.Context(), ConsumerConfig{Durable: workerDurable}) + require.NoError(t, err) + got := make(chan string, 16) + stop, failed, err := cons.Consume(func(m *Message) { + assert.NoError(t, m.DoubleAck(m.Ctx)) + got <- string(m.Data) + }, 16) + require.NoError(t, err) + t.Cleanup(stop) + + require.NoError(t, e.Publish(t.Context(), topic, []byte("before"))) + require.Equal(t, "before", receive(t, got)) + + f.stop() + require.Eventually(t, func() bool { return !e.connected.Load() }, 5*time.Second, 10*time.Millisecond) + err = e.Publish(t.Context(), topic, []byte("down")) + require.ErrorIs(t, err, ErrUnavailable) + assert.NotErrorIs(t, err, ErrQueueFull) + + f.restart(t) + require.Eventually(t, func() bool { return e.Publish(t.Context(), topic, []byte("after")) == nil }, + 15*time.Second, 100*time.Millisecond, "publishing never resumed") + for data := receive(t, got); data != "after"; data = receive(t, got) { + // A publish that timed out before the restart may have been stored. + assert.Equal(t, "down", data) + } + select { + case err := <-failed: + t.Fatalf("a reconnect reported failed: %v", err) + case <-time.After(300 * time.Millisecond): + } +} + +func receive(t *testing.T, got <-chan string) string { + t.Helper() + select { + case s := <-got: + return s + case <-time.After(15 * time.Second): + t.Fatal("nothing delivered") + return "" + } +} + +// flakyPublish loses the answers to the first publishes it passes on, as a +// dropped connection or a leader change could, and records each attempt's +// Nats-Msg-Id. +type flakyPublish struct { + jetstream.JetStream + lose atomic.Int32 + ids []string +} + +func (j *flakyPublish) PublishMsg(ctx context.Context, m *nats.Msg, opts ...jetstream.PublishOpt) (*jetstream.PubAck, error) { + ack, err := j.JetStream.PublishMsg(ctx, m, opts...) + j.ids = append(j.ids, m.Header.Get(jetstream.MsgIDHeader)) + if err == nil && j.lose.Add(-1) >= 0 { + return nil, context.DeadlineExceeded + } + return ack, err +} + +// A publish whose answer is lost is sent again with the same Nats-Msg-Id, +// so the partition stores it once; a fresh publish gets a fresh id. +func TestExternalNATS_PublishRetryStoresOnce(t *testing.T) { + t.Parallel() + f := shippedFixture(t) + e := f.broker(t, nil) + flaky := &flakyPublish{JetStream: e.js} + flaky.lose.Store(publishRetries) + e.js = flaky + topic := Topic{Tenant: "acme", Table: "t"} + stream := shippedPartition(mustPartition(topic)) + + require.NoError(t, e.Publish(t.Context(), topic, []byte("x"))) + require.Len(t, flaky.ids, publishRetries+1) + for _, id := range flaky.ids { + assert.Equal(t, flaky.ids[0], id) + } + assert.Equal(t, uint64(1), f.streamMsgs(t, stream)) + + require.NoError(t, e.Publish(t.Context(), topic, []byte("y"))) + assert.NotEqual(t, flaky.ids[0], flaky.ids[len(flaky.ids)-1]) + assert.Equal(t, uint64(2), f.streamMsgs(t, stream)) + + // Lost every time: the publish gives up as unavailable. + flaky.lose.Store(publishRetries + 1) + require.ErrorIs(t, e.Publish(t.Context(), topic, []byte("z")), ErrUnavailable) +} + +// A topic at the partition's max_msgs_per_subject is refused as full, like a +// partition at max_bytes, and only that topic. +func TestExternalNATS_TopicAtItsCapIsFull(t *testing.T) { + t.Parallel() + f := newNATSFixture(t) + tp := shippedTopology(t) + for i := range 4 { + tp.stream(t, shippedPartition(i)).MaxMsgsPerSubject = 2 + } + f.apply(t, tp) + e := f.broker(t, nil) + topic := Topic{Tenant: "acme", Table: "t"} + for range 2 { + require.NoError(t, e.Publish(t.Context(), topic, []byte("x"))) + } + require.ErrorIs(t, e.Publish(t.Context(), topic, []byte("x")), ErrQueueFull) + require.NoError(t, e.Publish(t.Context(), Topic{Tenant: "acme", Table: "other"}, []byte("x"))) +} + +// Under nats, WaveHouse does not require sync_always: against a server with +// it off, a publish to a one-replica partition is acked and stored, and boot +// reports the replica count as recommended only. TestShippedValues_SetNoSync +// covers the shipped values. +func TestExternalNATS_PublishesWithoutSyncAlways(t *testing.T) { + t.Parallel() + f := shippedFixture(t) + require.False(t, f.server.JetStreamConfig().SyncAlways) + + e := f.broker(t, nil) + topic := Topic{Tenant: "acme", Table: "t"} + stream := shippedPartition(mustPartition(topic)) + s, err := f.admin.Stream(t.Context(), stream) + require.NoError(t, err) + require.Equal(t, 1, s.CachedInfo().Config.Replicas) + + require.NoError(t, e.Publish(t.Context(), topic, []byte("x"))) + assert.Equal(t, uint64(1), f.streamMsgs(t, stream)) + + findings, err := verifyNATSTopology(t.Context(), e.js, e.topo, e.perms) + require.NoError(t, err) + for _, got := range findings { + assert.Equal(t, FindingRecommended, got.Severity, "unexpected finding %v", got) + } + assert.True(t, slices.ContainsFunc(findings, func(got Finding) bool { + return got.Object == "stream "+stream && got.Field == "num_replicas" + }), "findings: %v", findings) +} + +// A partition stream the operator deleted is ErrUnavailable, and the +// topology gauge drops at once; the broker creates nothing. +func TestExternalNATS_MissingPartitionIsUnavailable(t *testing.T) { + t.Parallel() + f := shippedFixture(t) + e := f.broker(t, func(c *NATSConfig) { c.Topology.PublishTimeout = 300 * time.Millisecond }) + topic := Topic{Tenant: "acme", Table: "t"} + stream := shippedPartition(mustPartition(topic)) + require.True(t, e.topologyOK.Load()) + + require.NoError(t, f.admin.DeleteStream(t.Context(), stream)) + err := e.Publish(t.Context(), topic, []byte("x")) + require.ErrorIs(t, err, ErrUnavailable) + assert.ErrorContains(t, err, stream) + assert.False(t, e.topologyOK.Load()) + _, err = f.admin.Stream(t.Context(), stream) + require.ErrorIs(t, err, jetstream.ErrStreamNotFound, "the broker must not recreate the stream") +} + +// Boot refuses a topology the operator never made, listing what is missing. +func TestNewNATS_RefusesAMissingTopology(t *testing.T) { + t.Parallel() + f := newNATSFixture(t) + tp := shippedTopology(t) + tp.drop("WH_DLQ") + f.apply(t, tp) + _, err := NewNATS(t.Context(), NATSConfig{ + URLs: []string{f.server.ClientURL()}, User: "wavehouse", PasswordFile: writeSecret(t, fixturePassword("wavehouse")), + Topology: NATSTopology{Partitions: 4, Shards: 8}, TopologyWait: 500 * time.Millisecond, + }) + require.ErrorIs(t, err, ErrTopology) + var te *TopologyError + require.ErrorAs(t, err, &te) + assert.Contains(t, err.Error(), "no stream holds wh.dlq.x") +} + +// Boot that never reaches a server is ErrUnavailable, once the wait is out. +func TestNewNATS_Unreachable(t *testing.T) { + t.Parallel() + f := newNATSFixture(t) + url := f.server.ClientURL() + f.server.Shutdown() + start := time.Now() + _, err := NewNATS(t.Context(), NATSConfig{URLs: []string{url}, TopologyWait: 500 * time.Millisecond}) + require.ErrorIs(t, err, ErrUnavailable) + assert.Less(t, time.Since(start), 10*time.Second) +} + +// Conflicting auth or half a TLS key pair is refused before dialing. +func TestNewNATS_RefusesConflictingOptions(t *testing.T) { + t.Parallel() + for name, cfg := range map[string]NATSConfig{ + "no urls": {}, + "two auths": {URLs: []string{"nats://127.0.0.1:1"}, User: "u", CredsFile: "x.creds"}, + "cert, no key": {URLs: []string{"nats://127.0.0.1:1"}, TLS: NATSTLS{CertFile: "c.pem"}}, + "bad prefix": {URLs: []string{"nats://127.0.0.1:1"}, Topology: NATSTopology{Prefix: "a.b"}}, + "password file": {URLs: []string{"nats://127.0.0.1:1"}, User: "u", PasswordFile: "/nonexistent"}, + } { + _, err := NewNATS(t.Context(), cfg) + assert.Error(t, err, name) + } +} + +// The re-check reports a topology the operator broke after boot, and its +// repair. The history poll reports how far the history trails the partitions: +// a row stored while the history refused it never reaches it, which shows +// as a gap until the next row republished closes it. +func TestExternalNATS_Recheck(t *testing.T) { + t.Parallel() + f := shippedFixture(t) + e := f.broker(t, func(c *NATSConfig) { + c.recheckEvery = 50 * time.Millisecond + c.historyEvery = 50 * time.Millisecond + }) + require.True(t, e.topologyOK.Load()) + + s, err := f.admin.Stream(t.Context(), "WH_DLQ") + require.NoError(t, err) + dlq := s.CachedInfo().Config + require.NoError(t, f.admin.DeleteStream(t.Context(), "WH_DLQ")) + require.Eventually(t, func() bool { return !e.topologyOK.Load() }, 5*time.Second, 10*time.Millisecond) + _, err = f.admin.CreateStream(t.Context(), dlq) + require.NoError(t, err) + require.Eventually(t, func() bool { return e.topologyOK.Load() }, 5*time.Second, 10*time.Millisecond) + + topic := Topic{Tenant: "acme", Table: "events"} + require.NoError(t, e.Publish(t.Context(), topic, []byte("1"))) + require.Eventually(t, func() bool { return e.historyBehind.Load() == 0 }, 5*time.Second, 10*time.Millisecond, "healthy") + + // A full history with discard new refuses the next row's copy. + h, err := f.admin.Stream(t.Context(), "WH_HISTORY") + require.NoError(t, err) + history := h.CachedInfo().Config + full := history + full.MaxMsgs, full.Discard = int64(h.CachedInfo().State.Msgs), jetstream.DiscardNew + _, err = f.admin.UpdateStream(t.Context(), full) + require.NoError(t, err) + time.Sleep(50 * time.Millisecond) // a gap the gauge can see + published := time.Now() + require.NoError(t, e.Publish(t.Context(), topic, []byte("2")), "ingest does not depend on the history") + require.Eventually(t, func() bool { return e.historyBehind.Load() > 0 }, 5*time.Second, 10*time.Millisecond, + "a row the history refused shows as a gap") + assert.Less(t, time.Duration(e.historyBehind.Load()), time.Since(published)+time.Second, "the gap is the missed row's, bounded") + + _, err = f.admin.UpdateStream(t.Context(), history) + require.NoError(t, err) + require.NoError(t, e.Publish(t.Context(), topic, []byte("3"))) + require.Eventually(t, func() bool { return e.historyBehind.Load() == 0 }, 5*time.Second, 10*time.Millisecond, + "the next row republished closes it") +} + +// A publish the ack says another stream stored — the operator replaced a +// partition's stream under the same subjects — is a topology fault. +func TestExternalNATS_PublishStoredElsewhere(t *testing.T) { + t.Parallel() + f := shippedFixture(t) + e := f.broker(t, nil) + topic := Topic{Tenant: "acme", Table: "events"} + stream := shippedPartition(mustPartition(topic)) + s, err := f.admin.Stream(t.Context(), stream) + require.NoError(t, err) + rogue := s.CachedInfo().Config + rogue.Name = "ROGUE" + require.NoError(t, f.admin.DeleteStream(t.Context(), stream)) + _, err = f.admin.CreateStream(t.Context(), rogue) + require.NoError(t, err) + + err = e.Publish(t.Context(), topic, []byte("x")) + require.ErrorIs(t, err, ErrUnavailable) + assert.ErrorContains(t, err, "stored by stream ROGUE") + assert.False(t, e.topologyOK.Load()) +} + +// The gauges report the connection, the topology and the history's gap. +func TestExternalNATS_Gauges(t *testing.T) { //nolint:paralleltest // sets the global meter provider + reader := sdkmetric.NewManualReader() + prev := otel.GetMeterProvider() + otel.SetMeterProvider(sdkmetric.NewMeterProvider(sdkmetric.WithReader(reader))) + t.Cleanup(func() { otel.SetMeterProvider(prev) }) + + f := shippedFixture(t) + f.broker(t, nil) + var rm metricdata.ResourceMetrics + require.NoError(t, reader.Collect(t.Context(), &rm)) + got := map[string][]float64{} + for _, sm := range rm.ScopeMetrics { + for _, m := range sm.Metrics { + switch g := m.Data.(type) { + case metricdata.Gauge[int64]: + for _, dp := range g.DataPoints { + got[m.Name] = append(got[m.Name], float64(dp.Value)) + } + case metricdata.Gauge[float64]: + for _, dp := range g.DataPoints { + got[m.Name] = append(got[m.Name], dp.Value) + } + } + } + } + assert.Equal(t, []float64{1}, got["wavehouse_mq_connected"]) + assert.Equal(t, []float64{1}, got["wavehouse_mq_topology_ok"]) + assert.Equal(t, []float64{0}, got["wavehouse_mq_history_behind_seconds"]) +} + +// PurgeAcked removes nothing. CheckReplayWindows warns for a tenant whose +// gap window the history cannot hold, once per window: the same windows warn +// nobody again, a longer one warns anew, and a window equal to max_age fits. +func TestExternalNATS_ReplayWindowsAgainstTheHistory(t *testing.T) { //nolint:paralleltest // captures the default logger + logs := logtest.Capture(t, slog.LevelWarn) + f := shippedFixture(t) + e := f.broker(t, nil) + maxAge := time.Duration(e.historyMaxAge.Load()) + require.Positive(t, maxAge, "the shipped history has a max_age") + + purged, err := e.PurgeAcked(t.Context(), workerDurable, map[tenant.ID]time.Time{"acme": time.Now().Add(-2 * maxAge)}) + require.NoError(t, err) + assert.False(t, purged) + _, err = e.PurgeAcked(t.Context(), "someone-else", nil) + require.ErrorIs(t, err, ErrConsumerNotFound) + + warned := func() int { return strings.Count(logs.String(), "keeps less than this tenant's gap window") } + windows := map[tenant.ID]time.Duration{"acme": 2 * maxAge, "globex": time.Minute, "initech": maxAge} + e.CheckReplayWindows(windows) + assert.Equal(t, 1, warned(), "acme only: %s", logs.String()) + assert.Contains(t, logs.String(), `"tenant":"acme"`) + e.CheckReplayWindows(windows) + assert.Equal(t, 1, warned(), "the same window warns once") + windows["acme"] = 3 * maxAge + e.CheckReplayWindows(windows) + assert.Equal(t, 2, warned(), "a longer window warns again") + + // The operator shortens the history: the poll checks the last windows again. + h, err := f.admin.Stream(t.Context(), e.topo.HistoryStream) + require.NoError(t, err) + shorter := h.CachedInfo().Config + shorter.MaxAge = maxAge / 2 + _, err = f.admin.UpdateStream(t.Context(), shorter) + require.NoError(t, err) + require.NoError(t, e.readHistory(t.Context())) + assert.Equal(t, 3, warned(), "initech's window no longer fits: %s", logs.String()) +} + +// CreateConsumer finds the operator's durable and holds it to the worker's +// ask; it never creates one. +func TestExternalNATS_CreateConsumerChecksTheDurable(t *testing.T) { + t.Parallel() + f := shippedFixture(t) + e := f.broker(t, nil) + + _, err := e.CreateConsumer(t.Context(), ConsumerConfig{Durable: workerDurable, AckWait: time.Hour}) + require.ErrorContains(t, err, "ack_wait") + require.ErrorIs(t, err, ErrConsumerMismatch) + _, err = e.CreateConsumer(t.Context(), ConsumerConfig{Durable: "someone-else"}) + require.ErrorIs(t, err, ErrConsumerNotFound) + _, err = e.CreateConsumer(t.Context(), ConsumerConfig{Durable: DefaultNATSIngestConsumer, AckWait: time.Minute}) + require.NoError(t, err) + + require.NoError(t, f.admin.DeleteConsumer(t.Context(), shippedPartition(0), "wh-ingest-3")) + _, err = e.CreateConsumer(t.Context(), ConsumerConfig{Durable: workerDurable}) + require.ErrorIs(t, err, ErrConsumerNotFound) + _, err = f.admin.Consumer(t.Context(), shippedPartition(0), "wh-ingest-3") + require.ErrorIs(t, err, jetstream.ErrConsumerNotFound, "the broker must not recreate the durable") + + require.NoError(t, f.admin.DeleteStream(t.Context(), shippedPartition(1))) + _, err = e.CreateConsumer(t.Context(), ConsumerConfig{Durable: workerDurable, Units: []string{shippedPartition(1) + "/wh-ingest-0"}}) + require.ErrorIs(t, err, ErrConsumerNotFound, "a durable whose stream is gone is gone") +} + +// The shipped permissions refuse the wavehouse user everything that would +// change the operator's topology. +func TestNATSPermissions_RefuseTopologyChanges(t *testing.T) { + t.Parallel() + f := shippedFixture(t) + js := f.connect(t, "wavehouse", nats.ErrorHandler(func(*nats.Conn, *nats.Subscription, error) {})) + ctx := t.Context() + partition := shippedPartition(0) + denied := func(what string, err error) { + t.Helper() + require.Error(t, err, what) + assert.False(t, errors.Is(err, context.Canceled), what) + } + + s, err := f.admin.Stream(ctx, partition) + require.NoError(t, err) + cfg := s.CachedInfo().Config + + denied("create a stream", call(ctx, func(ctx context.Context) error { + _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "ROGUE", Subjects: []string{"rogue.>"}}) + return err + })) + cfg.MaxBytes++ + denied("update a partition", call(ctx, func(ctx context.Context) error { + _, err := js.UpdateStream(ctx, cfg) + return err + })) + denied("purge a partition", call(ctx, func(ctx context.Context) error { + ws, err := js.Stream(ctx, partition) + if err != nil { + return err + } + return ws.Purge(ctx) + })) + denied("delete a partition", call(ctx, func(ctx context.Context) error { return js.DeleteStream(ctx, partition) })) + denied("create a durable on a partition", call(ctx, func(ctx context.Context) error { + _, err := js.CreateConsumer(ctx, partition, jetstream.ConsumerConfig{Durable: "rogue", AckPolicy: jetstream.AckExplicitPolicy}) + return err + })) + denied("delete a shard durable", call(ctx, func(ctx context.Context) error { + return js.DeleteConsumer(ctx, partition, "wh-ingest-0") + })) + denied("forge a history row", call(ctx, func(ctx context.Context) error { + _, err := js.Publish(ctx, "wh.hist.acme.events", []byte("forged")) + return err + })) + + _, err = f.admin.Stream(ctx, "ROGUE") + require.ErrorIs(t, err, jetstream.ErrStreamNotFound) + h, err := f.admin.Stream(ctx, "WH_HISTORY") + require.NoError(t, err) + assert.Zero(t, h.CachedInfo().State.Msgs, "only the partitions' republish writes the history") + _, err = f.admin.Consumer(ctx, partition, "wh-ingest-0") + require.NoError(t, err) +} + +// Each partition republishes a row to the history under its topic, the +// partition token dropped, headers and body intact: what the hub and a +// replay read, whatever the partition count. +func TestExternalNATS_HistoryDropsThePartition(t *testing.T) { + t.Parallel() + f := shippedFixture(t) + e := f.broker(t, nil) + topics := []Topic{{Tenant: "acme", Table: "events"}, {Tenant: "acme", Table: "events", Scope: "eu"}, {Tenant: "globex", Table: "a.b"}} + for _, topic := range topics { + require.NoError(t, e.Publish(t.Context(), topic, []byte(topic.Table), WithHeader("X-Test", "kept"))) + } + h, err := f.admin.Stream(t.Context(), "WH_HISTORY") + require.NoError(t, err) + require.Eventually(t, func() bool { + info, err := h.Info(t.Context(), jetstream.WithSubjectFilter(">")) + return err == nil && info.State.Msgs == uint64(len(topics)) + }, 5*time.Second, 10*time.Millisecond) + info, err := h.Info(t.Context(), jetstream.WithSubjectFilter(">")) + require.NoError(t, err) + var subjects []string + for subj := range info.State.Subjects { + subjects = append(subjects, subj) + } + assert.ElementsMatch(t, []string{"wh.hist.acme.events", "wh.hist.acme.events.eu", "wh.hist.globex.a%2Eb"}, subjects) + m, err := h.GetLastMsgForSubject(t.Context(), "wh.hist.acme.events") + require.NoError(t, err) + assert.Equal(t, "events", string(m.Data)) + assert.Equal(t, "kept", m.Header.Get("X-Test")) +} + +// call runs a request that a permission violation answers by never +// answering, bounded. +func call(ctx context.Context, fn func(context.Context) error) error { + ctx, cancel := context.WithTimeout(ctx, 500*time.Millisecond) + defer cancel() + return fn(ctx) +} + +// measure skips a measurement unless WH_MQ_MEASURE is set: it reports numbers +// for the PR record, and asserts nothing a loaded CI machine could fail. +func measure(t *testing.T) { + t.Helper() + if os.Getenv("WH_MQ_MEASURE") == "" { + t.Skip("set WH_MQ_MEASURE=1 to measure") + } +} + +// A publish's latency until the hub's Subscribe sees it, through the +// partition and its republish to the history. Design risk 3 moves the hub off the +// history if p99 passes 50ms. +func TestExternalNATS_MeasurePublishToHub(t *testing.T) { + measure(t) + e := shippedFixture(t).broker(t, nil) + seen := make(chan time.Time, 1) + require.NoError(t, e.Subscribe(t.Context(), "hub-bridge", func(*Message) error { + seen <- time.Now() + return nil + })) + topic := Topic{Tenant: "acme", Table: "t"} + const n = 2000 + lat := make([]time.Duration, 0, n) + for range n { + start := time.Now() + require.NoError(t, e.Publish(t.Context(), topic, []byte(`{"a":1}`))) + lat = append(lat, (<-seen).Sub(start)) + } + slices.Sort(lat) + t.Logf("publish to hub over %d events: p50 %s, p99 %s, max %s", n, lat[n/2], lat[n*99/100], lat[n-1]) +} + +// One tenant's publish throughput into its partition, which bounds a hot +// tenant (design risk 4). +func TestExternalNATS_MeasurePublishThroughput(t *testing.T) { + measure(t) + e := shippedFixture(t).broker(t, nil) + topic := Topic{Tenant: "acme", Table: "t"} + payload := make([]byte, 256) + const workers, each = 32, 500 + start := time.Now() + var wg sync.WaitGroup + for range workers { + wg.Go(func() { + for range each { + assert.NoError(t, e.Publish(t.Context(), topic, payload)) + } + }) + } + wg.Wait() + elapsed := time.Since(start) + t.Logf("%d publishes of %d bytes by %d callers in %s: %.0f/s", workers*each, len(payload), workers, elapsed, float64(workers*each)/elapsed.Seconds()) +} + +// The duplicate window must cover every attempt of a retried publish, not +// just two publish timeouts: a window of exactly 2 × PublishTimeout is too +// short for the last retry. +func TestExternalNATS_DuplicateWindowCoversEveryRetry(t *testing.T) { + t.Parallel() + f := newNATSFixture(t) + tp := shippedTopology(t) + topo := NATSTopology{Partitions: 4, Shards: 8, PublishTimeout: 30 * time.Second} + for i := range 4 { + tp.stream(t, shippedPartition(i)).Duplicates = 2 * topo.PublishTimeout + } + f.apply(t, tp) + findings, err := verifyNATSTopology(t.Context(), f.connect(t, "wavehouse"), topo, nil) + require.NoError(t, err) + n := 0 + for _, fd := range findings { + if fd.Field == "duplicate_window" { + n++ + assert.Equal(t, FindingRequired, fd.Severity) + } + } + assert.Equal(t, 4, n) +} + +// A replay sends the events counted when it began and stops: events that +// arrive during it reach an SSE client through its live subscription. +func TestExternalNATS_ReplayDoesNotChaseTheTail(t *testing.T) { + t.Parallel() + e := shippedFixture(t).broker(t, nil) + topic := Topic{Tenant: "acme", Table: "r"} + for _, d := range []string{"a", "b", "c"} { + require.NoError(t, e.Publish(t.Context(), topic, []byte(d))) + } + require.Eventually(t, func() bool { + n := 0 + require.NoError(t, e.ReplaySince(t.Context(), topic, time.Time{}, func([]byte) bool { n++; return true })) + return n == 3 + }, 5*time.Second, 20*time.Millisecond) + + var got []string + require.NoError(t, e.ReplaySince(t.Context(), topic, time.Time{}, func(data []byte) bool { + if len(got) == 0 { + for range 5 { + assert.NoError(t, e.Publish(t.Context(), topic, []byte("late"))) + } + time.Sleep(200 * time.Millisecond) // let the history copy them + } + got = append(got, string(data)) + return true + })) + assert.Equal(t, []string{"a", "b", "c"}, got) +} + +// A replay longer than one fetched batch, sent to a slow client, arrives +// whole: its consumer outlives the time a batch takes to send. +func TestExternalNATS_SlowReplayArrivesWhole(t *testing.T) { + t.Parallel() + e := shippedFixture(t).broker(t, nil) + topic := Topic{Tenant: "acme", Table: "slow"} + const n = replayBatch + 44 + for i := range n { + require.NoError(t, e.Publish(t.Context(), topic, []byte(strconv.Itoa(i)))) + } + require.Eventually(t, func() bool { + got := 0 + require.NoError(t, e.ReplaySince(t.Context(), topic, time.Time{}, func([]byte) bool { got++; return got < n })) + return got == n + }, 5*time.Second, 20*time.Millisecond) + + var got []string + require.NoError(t, e.ReplaySince(t.Context(), topic, time.Time{}, func(data []byte) bool { + time.Sleep(25 * time.Millisecond) // 256 of these outlast the old 5s threshold + got = append(got, string(data)) + return true + })) + require.Len(t, got, n) + for i, d := range got { + require.Equal(t, strconv.Itoa(i), d) + } +} + +// Closing the broker is not a disconnect worth a warning; losing the server +// still is. +func TestExternalNATS_CloseLogsNoWarning(t *testing.T) { //nolint:paralleltest // captures the default logger + f := shippedFixture(t) + e := f.broker(t, nil) + other := f.broker(t, nil) + cons, err := e.CreateConsumer(t.Context(), ConsumerConfig{Durable: workerDurable}) + require.NoError(t, err) + _, _, err = cons.Consume(func(*Message) {}, 16) + require.NoError(t, err) + require.NoError(t, e.Subscribe(t.Context(), "hub", func(*Message) error { return nil })) + + logs := logtest.Capture(t, slog.LevelWarn) + require.NoError(t, e.Close()) + assert.Empty(t, logs.String(), "a deliberate close logged at WARN or above") + + f.stop() + require.Eventually(t, func() bool { return !other.connected.Load() }, 5*time.Second, 10*time.Millisecond) + require.Eventually(t, func() bool { return strings.Contains(logs.String(), "mq: disconnected from nats; reconnecting") }, + 5*time.Second, 10*time.Millisecond, "a lost server must still warn") +} + +// generatedTopology is `wavehouse mq manifests --partitions n --shards v` +// at one replica. +func generatedTopology(t *testing.T, n, v int) *fixtureTopology { + t.Helper() + var buf bytes.Buffer + require.NoError(t, WriteNATSManifests(&buf, NATSManifestOptions{Topology: NATSTopology{Partitions: n, Shards: v}, Replicas: 1})) + path := filepath.Join(t.TempDir(), "manifests.yaml") + require.NoError(t, os.WriteFile(path, buf.Bytes(), 0o600)) + return loadNATSManifests(t, path) +} + +// Lowering N from 2 to 1 the way deployment.md says — apply the regenerated +// manifests, roll out the smaller N while an old-N process keeps publishing — +// loses none of partition 1's rows: the new worker drains them through its +// shard durables, which are extra units now, and the operator deleting the +// emptied stream ends only those units' delivery. +func TestExternalNATS_LoweringNDrainsTheRemovedPartition(t *testing.T) { //nolint:paralleltest // captures the default logger + f := newNATSFixture(t) + f.apply(t, generatedTopology(t, 2, 2)) + const removed = "WH_INGEST_1" + topic := topicIn(t, "acme", 1, 2, 2) + _, shard := natsRoute(topic, 2, 2) + unit := removed + "/" + natsShardDurable(DefaultNATSIngestConsumer, shard) + old := f.broker(t, func(c *NATSConfig) { c.Topology.Partitions, c.Topology.Shards = 2, 2 }) + before := []string{"a", "b", "c", "d", "e"} + for _, row := range before { + require.NoError(t, old.Publish(t.Context(), topic, []byte(row))) + } + require.Equal(t, uint64(len(before)), f.streamMsgs(t, removed)) + + require.NoError(t, generatedTopology(t, 1, 2).Apply(t.Context(), f.admin)) + e := f.broker(t, func(c *NATSConfig) { c.Topology.Partitions, c.Topology.Shards = 1, 2 }) + _, extra := e.IngestUnits() + assert.Equal(t, []string{removed + "/wh-ingest-0", removed + "/wh-ingest-1"}, extra) + findings, err := verifyNATSTopology(t.Context(), e.js, e.topo, e.perms) + require.NoError(t, err) + assert.True(t, slices.ContainsFunc(findings, func(got Finding) bool { + return got.Severity == FindingRecommended && got.Object == "consumer "+unit && strings.Contains(got.Problem, "drain its 5 rows") + }), "no finding names the removed partition's rows among %v", findings) + + logs := logtest.Capture(t, slog.LevelInfo) + cons, err := e.CreateConsumer(t.Context(), ConsumerConfig{Durable: workerDurable}) + require.NoError(t, err) + got := make(chan string, 16) + stop, failed, err := cons.Consume(func(m *Message) { + assert.NoError(t, m.DoubleAck(m.Ctx)) + got <- string(m.Data) + }, 16) + require.NoError(t, err) + t.Cleanup(stop) + drained := make([]string, 0, len(before)) + for range before { + drained = append(drained, receive(t, got)) + } + assert.Equal(t, before, drained) + + // The old-N process is still up during the rollout, publishing to the + // removed partition. + require.NoError(t, old.Publish(t.Context(), topic, []byte("during"))) + require.Equal(t, "during", receive(t, got)) + require.NoError(t, old.Close()) + require.Eventually(t, func() bool { return f.streamMsgs(t, removed) == 0 }, 5*time.Second, 10*time.Millisecond) + require.NoError(t, e.Publish(t.Context(), topic, []byte("moved"))) + require.Equal(t, "moved", receive(t, got)) + + require.NoError(t, f.admin.DeleteStream(t.Context(), removed)) + require.Eventually(t, func() bool { + return strings.Contains(logs.String(), "mq: stopped draining a shard durable outside the configured ones") + }, 15*time.Second, 50*time.Millisecond, "the removed partition's delivery never ended") + select { + case err := <-failed: + t.Fatalf("deleting a drained removed partition reported failed: %v", err) + default: + } + require.NoError(t, e.Publish(t.Context(), topic, []byte("still"))) + require.Equal(t, "still", receive(t, got)) +} + +// The queue's units are every partition's shard durables, in partition then +// shard order, and a publish lands on its table's (partition, shard). +func TestExternalNATS_ShardUnits(t *testing.T) { + t.Parallel() + f := shippedFixture(t) + e := f.broker(t, nil) + configured, extra := e.IngestUnits() + require.Len(t, configured, 32) + assert.Equal(t, "WH_INGEST_0/wh-ingest-0", configured[0]) + assert.Equal(t, "WH_INGEST_3/wh-ingest-7", configured[31]) + assert.Empty(t, extra) + + topic := Topic{Tenant: "acme", Table: "events"} + require.NoError(t, e.Publish(t.Context(), topic, []byte("x"))) + p, s := natsRoute(topic, 4, 8) + c, err := f.admin.Consumer(t.Context(), shippedPartition(p), natsShardDurable("wh-ingest", s)) + require.NoError(t, err) + assert.Equal(t, uint64(1), c.CachedInfo().NumPending, "the row waits on its shard's durable") + + _, err = e.CreateConsumer(t.Context(), ConsumerConfig{Durable: workerDurable, Units: []string{"WH_INGEST_9/wh-ingest-0"}}) + require.ErrorIs(t, err, ErrUnitsUnsupported) +} + +// unitConsumer consumes one unit of e, sending each row's body to the +// returned channel and acking it unless hold is set. +func unitConsumer(t *testing.T, e *ExternalNATS, unit string, hold *atomic.Bool) (Consumer, func(), <-chan string) { + t.Helper() + c, err := e.CreateConsumer(t.Context(), ConsumerConfig{Durable: workerDurable, Units: []string{unit}}) + require.NoError(t, err) + got := make(chan string, 64) + stop, _, err := c.Consume(func(m *Message) { + if hold == nil || !hold.Load() { + assert.NoError(t, m.Ack()) + } + got <- string(m.Data) + }, 16) + require.NoError(t, err) + t.Cleanup(stop) + return c, stop, got +} + +// Two processes pull one unit and only the pinned one receives. Release by +// the one that never received leaves the owner's pin alone; the owner's own +// release, after it stopped, hands the unit over at once. +func TestExternalNATS_ReleaseUnpinsOnlyItsOwnPin(t *testing.T) { + t.Parallel() + f := shippedFixture(t) + a, b := f.broker(t, nil), f.broker(t, nil) + topic := Topic{Tenant: "acme", Table: "events"} + p, s := natsRoute(topic, 4, 8) + unit := shippedPartition(p) + "/" + natsShardDurable("wh-ingest", s) + + ca, stopA, gotA := unitConsumer(t, a, unit, nil) + require.NoError(t, a.Publish(t.Context(), topic, []byte("1"))) + require.Equal(t, "1", receive(t, gotA)) + cb, _, gotB := unitConsumer(t, b, unit, nil) + require.NoError(t, a.Publish(t.Context(), topic, []byte("2"))) + require.Equal(t, "2", receive(t, gotA), "A holds the pin") + + pinnedTo := func() string { + c, err := f.admin.Consumer(t.Context(), shippedPartition(p), natsShardDurable("wh-ingest", s)) + require.NoError(t, err) + return pinnedClient(c.CachedInfo()) + } + pin := pinnedTo() + require.NotEmpty(t, pin) + require.NoError(t, cb.(Releaser).Release(t.Context())) + assert.Equal(t, pin, pinnedTo(), "B never held the unit, so its release leaves A's pin") + + stopA() + released := time.Now() + require.NoError(t, ca.(Releaser).Release(t.Context())) + assert.Empty(t, pinnedTo()) + require.NoError(t, a.Publish(t.Context(), topic, []byte("3"))) + require.Equal(t, "3", receive(t, gotB)) + assert.Less(t, time.Since(released), 5*time.Second, "at once, not after the pinned ttl") +} + +// quietAfter shortens how recently a unit must have been active to count +// as someone's, the durable's pinned_ttl otherwise. +func quietAfter(d time.Duration) func(*NATSConfig) { + return func(cfg *NATSConfig) { cfg.activeWithin = d } +} + +// A unit whose holder is gone with rows unacked: ResetOrphaned leaves it +// alone while a client still holds the pin, or was active on it within the +// window, and once neither, redelivers the held rows to the next owner at +// once, not after ack_wait. +func TestExternalNATS_ResetOrphaned(t *testing.T) { + t.Parallel() + f := shippedFixture(t) + const quiet = time.Second + a, b := f.broker(t, nil), f.broker(t, quietAfter(quiet)) + topic := Topic{Tenant: "acme", Table: "events"} + p, s := natsRoute(topic, 4, 8) + unit := shippedPartition(p) + "/" + natsShardDurable("wh-ingest", s) + + var hold atomic.Bool + hold.Store(true) + unitConsumer(t, a, unit, &hold) + for _, row := range []string{"x", "y"} { + require.NoError(t, a.Publish(t.Context(), topic, []byte(row))) + } + reset, err := b.ResetOrphaned(t.Context(), unit) + require.ErrorIs(t, err, ErrUnitHeld, "a live holder's rows are its own") + assert.False(t, reset) + + a.nc.Close() // A dies holding x and y + st, err := f.admin.Stream(t.Context(), shippedPartition(p)) + require.NoError(t, err) + require.NoError(t, st.UnpinConsumer(t.Context(), natsShardDurable("wh-ingest", s), natsPriorityGroup), "stands in for the pin lapsing") + reset, err = b.ResetOrphaned(t.Context(), unit) + require.ErrorIs(t, err, ErrUnitHeld, "no pin alone is no proof: it delivered just now") + assert.False(t, reset) + time.Sleep(quiet) + reset, err = b.ResetOrphaned(t.Context(), unit) + require.NoError(t, err) + require.True(t, reset) + _, _, gotB := unitConsumer(t, b, unit, nil) + assert.ElementsMatch(t, []string{"x", "y"}, []string{receive(t, gotB), receive(t, gotB)}) + + reset, err = b.ResetOrphaned(t.Context(), unit) + require.ErrorIs(t, err, ErrUnitHeld, "b holds it now") + assert.False(t, reset) +} + +// unitPin is the pin the server holds on topic's unit in the shipped topology. +func unitPin(t *testing.T, f *natsFixture, topic Topic) string { + t.Helper() + p, s := natsRoute(topic, 4, 8) + c, err := f.admin.Consumer(t.Context(), shippedPartition(p), natsShardDurable("wh-ingest", s)) + require.NoError(t, err) + return pinnedClient(c.CachedInfo()) +} + +// A consumer of every unit releases each unit it held once its stop has +// drained, so the next process receives at once. +func TestExternalNATS_WholeConsumerReleasesOnStop(t *testing.T) { + t.Parallel() + f := shippedFixture(t) + e := f.broker(t, nil) + topic := Topic{Tenant: "acme", Table: "events"} + c, err := e.CreateConsumer(t.Context(), ConsumerConfig{Durable: workerDurable}) + require.NoError(t, err) + got := make(chan string, 4) + stop, _, err := c.Consume(func(m *Message) { assert.NoError(t, m.Ack()); got <- string(m.Data) }, 16) + require.NoError(t, err) + require.NoError(t, e.Publish(t.Context(), topic, []byte("x"))) + require.Equal(t, "x", receive(t, got)) + require.NotEmpty(t, unitPin(t, f, topic)) + stop() + require.Eventually(t, func() bool { return unitPin(t, f, topic) == "" }, 3*time.Second, 20*time.Millisecond, + "released after the drain, well before the 10s pinned ttl") +} + +// A consumer of every unit takes over a unit whose holder is gone with rows +// unacked: it receives them at once, not after ack_wait. +func TestExternalNATS_WholeConsumerTakesOrphansOver(t *testing.T) { + t.Parallel() + f := shippedFixture(t) + a := f.broker(t, nil) + topic := Topic{Tenant: "acme", Table: "events"} + p, s := natsRoute(topic, 4, 8) + var hold atomic.Bool + hold.Store(true) + _, _, gotA := unitConsumer(t, a, shippedPartition(p)+"/"+natsShardDurable("wh-ingest", s), &hold) + require.NoError(t, a.Publish(t.Context(), topic, []byte("held"))) + require.Equal(t, "held", receive(t, gotA)) + a.nc.Close() + st, err := f.admin.Stream(t.Context(), shippedPartition(p)) + require.NoError(t, err) + require.NoError(t, st.UnpinConsumer(t.Context(), natsShardDurable("wh-ingest", s), natsPriorityGroup), "stands in for the pin lapsing") + const quiet = time.Second + time.Sleep(quiet) // and for its owner's silence since + + b := f.broker(t, quietAfter(quiet)) + c, err := b.CreateConsumer(t.Context(), ConsumerConfig{Durable: workerDurable}) + require.NoError(t, err) + got := make(chan string, 4) + began := time.Now() + stop, _, err := c.Consume(func(m *Message) { assert.NoError(t, m.Ack()); got <- string(m.Data) }, 16) + require.NoError(t, err) + t.Cleanup(stop) + require.Equal(t, "held", receive(t, got)) + assert.Less(t, time.Since(began), 5*time.Second, "at once, not after the minute's ack_wait") +} + +// A consumer whose pin was taken over since it last received leaves the new +// holder's pin alone when it releases. +func TestExternalNATS_ReleaseChecksThePinIsStillItsOwn(t *testing.T) { + t.Parallel() + f := shippedFixture(t) + a, b := f.broker(t, nil), f.broker(t, nil) + topic := Topic{Tenant: "acme", Table: "events"} + p, s := natsRoute(topic, 4, 8) + unit := shippedPartition(p) + "/" + natsShardDurable("wh-ingest", s) + ca, stopA, gotA := unitConsumer(t, a, unit, nil) + require.NoError(t, a.Publish(t.Context(), topic, []byte("1"))) + require.Equal(t, "1", receive(t, gotA)) + stopA() + + st, err := f.admin.Stream(t.Context(), shippedPartition(p)) + require.NoError(t, err) + require.NoError(t, st.UnpinConsumer(t.Context(), natsShardDurable("wh-ingest", s), natsPriorityGroup)) + _, _, gotB := unitConsumer(t, b, unit, nil) + require.NoError(t, b.Publish(t.Context(), topic, []byte("2"))) + require.Equal(t, "2", receive(t, gotB)) + pin := unitPin(t, f, topic) + require.NotEmpty(t, pin) + + require.NoError(t, ca.(Releaser).Release(t.Context())) + assert.Equal(t, pin, unitPin(t, f, topic), "A's release leaves B's pin") +} + +// Unowned counts the units with rows waiting that no consumer holds. +func TestExternalNATS_Unowned(t *testing.T) { + t.Parallel() + f := shippedFixture(t) + e := f.broker(t, nil) + n, err := e.Unowned(t.Context()) + require.NoError(t, err) + assert.Zero(t, n, "no rows, nothing to own") + + topic := Topic{Tenant: "acme", Table: "events"} + require.NoError(t, e.Publish(t.Context(), topic, []byte("x"))) + n, err = e.Unowned(t.Context()) + require.NoError(t, err) + assert.Equal(t, 1, n, "a row nobody pulls") + + var hold atomic.Bool + hold.Store(true) + p, s := natsRoute(topic, 4, 8) + _, _, got := unitConsumer(t, e, shippedPartition(p)+"/"+natsShardDurable("wh-ingest", s), &hold) + require.Equal(t, "x", receive(t, got)) + n, err = e.Unowned(t.Context()) + require.NoError(t, err) + assert.Zero(t, n, "held by the consumer that received it") +} + +// One unit's consume throughput: a backlog of rows on one shard, drained by +// one consumer that acks each row as it arrives. +func TestExternalNATS_MeasureUnitConsumeThroughput(t *testing.T) { + measure(t) + f := shippedFixture(t) + e := f.broker(t, nil) + topic := Topic{Tenant: "acme", Table: "t"} + payload := make([]byte, 256) + const n = 20_000 + var wg sync.WaitGroup + for w := range 32 { + wg.Go(func() { + for i := w; i < n; i += 32 { + assert.NoError(t, e.Publish(t.Context(), topic, payload)) + } + }) + } + wg.Wait() + p, s := natsRoute(topic, 4, 8) + c, err := e.CreateConsumer(t.Context(), ConsumerConfig{Durable: workerDurable, Units: []string{shippedPartition(p) + "/" + natsShardDurable("wh-ingest", s)}}) + require.NoError(t, err) + var got atomic.Int64 + done := make(chan struct{}) + start := time.Now() + stop, _, err := c.Consume(func(m *Message) { + _ = m.Ack() + if got.Add(1) == n { + close(done) + } + }, 500) + require.NoError(t, err) + t.Cleanup(stop) + select { + case <-done: + case <-time.After(2 * time.Minute): + t.Fatalf("received %d of %d", got.Load(), n) + } + elapsed := time.Since(start) + t.Logf("one unit drained %d rows of %d bytes in %s: %.0f rows/s", n, len(payload), elapsed.Round(time.Millisecond), float64(n)/elapsed.Seconds()) +} + +// A shard durable deleted while its consumer's handler is busy and no pull +// of it is waiting ends delivery with failed, not a silent stall: the next +// pin-keeping pull gets no responders, and the durable is looked up. +func TestExternalNATS_DurableDeletedWhileBusyEndsDelivery(t *testing.T) { + t.Parallel() + f := shippedFixture(t) + e := f.broker(t, nil) + topic := Topic{Tenant: "acme", Table: "events"} + p, s := natsRoute(topic, 4, 8) + stream, durable := shippedPartition(p), natsShardDurable("wh-ingest", s) + c, err := e.CreateConsumer(t.Context(), ConsumerConfig{ + Durable: workerDurable, Units: []string{stream + "/" + durable}, + MaxHeld: func() int { return 2 }, + }) + require.NoError(t, err) + busy := make(chan struct{}) + t.Cleanup(func() { close(busy) }) + entered := make(chan struct{}, 8) + stop, failed, err := c.Consume(func(*Message) { entered <- struct{}{}; <-busy }, 16) + require.NoError(t, err) + t.Cleanup(stop) + for i := range 5 { + require.NoError(t, e.Publish(t.Context(), topic, []byte(strconv.Itoa(i)))) + } + <-entered // the handler holds the first row; the unit is at its cap + + require.Eventually(t, func() bool { + cons, err := f.admin.Consumer(t.Context(), stream, durable) + return err == nil && cons.CachedInfo().NumAckPending == 2 && cons.CachedInfo().NumWaiting == 0 + }, 10*time.Second, 5*time.Millisecond, "a moment with no pull waiting") + require.NoError(t, f.admin.DeleteConsumer(t.Context(), stream, durable)) + deleted := time.Now() + select { + case err := <-failed: + assert.ErrorIs(t, err, ErrDeliveryEnded) + assert.ErrorIs(t, err, ErrConsumerNotFound) + t.Logf("failed %s after the delete: %v", time.Since(deleted).Round(10*time.Millisecond), err) + case <-time.After(20 * time.Second): + t.Fatal("a durable deleted under a busy handler stalled silently") + } +} + +// Rows NAKed with a delay that come due while their unit is at its cap come +// back in order. A pin renewal that held a due row back would have the server +// requeue it behind the others (measured with max_bytes-1 renewals: 2 3 0 1), +// so at the cap the renewal takes the row instead. +func TestExternalNATS_RedeliveryOrderKeptAtTheCap(t *testing.T) { + t.Parallel() + f := shippedFixture(t) + e := f.broker(t, nil) + topic := Topic{Tenant: "acme", Table: "events"} + p, s := natsRoute(topic, 4, 8) + c, err := e.CreateConsumer(t.Context(), ConsumerConfig{ + Durable: workerDurable, + Units: []string{shippedPartition(p) + "/" + natsShardDurable("wh-ingest", s)}, MaxHeld: func() int { return 4 }, + }) + require.NoError(t, err) + var mu sync.Mutex + var got []string + var held []*Message + stop, _, err := c.Consume(func(m *Message) { + mu.Lock() + got = append(got, string(m.Data)) + held = append(held, m) + mu.Unlock() + }, 16) + require.NoError(t, err) + t.Cleanup(stop) + for i := range 8 { + require.NoError(t, e.Publish(t.Context(), topic, []byte(strconv.Itoa(i)))) + } + take := func(n int) []*Message { + require.Eventually(t, func() bool { mu.Lock(); defer mu.Unlock(); return len(held) >= n }, 20*time.Second, 10*time.Millisecond) + mu.Lock() + defer mu.Unlock() + out := held[:n] + held = held[n:] + return out + } + for _, m := range take(4) { // 0..3 come due in 3s, while 4..7 fill the cap + require.NoError(t, m.NakWithDelay(3*time.Second)) + } + later := take(4) + time.Sleep(14 * time.Second) // two renewals at least + for _, m := range later { + require.NoError(t, m.Ack()) + } + redelivered := take(4) + var order []string + for _, m := range redelivered { + order = append(order, string(m.Data)) + _ = m.Ack() + } + mu.Lock() + first := slices.Clone(got[:8]) + mu.Unlock() + assert.Equal(t, []string{"0", "1", "2", "3", "4", "5", "6", "7"}, first) + assert.Equal(t, []string{"0", "1", "2", "3"}, order, "redelivered in order across the renewals") +} + +// A unit whose handler is blocked, at its cap, keeps its pin, and takes no +// more rows to renew it than one ack_wait's worth of renewals (shortened to +// 10s, so 2 here); past that it renews without taking any. +func TestExternalNATS_BlockedHandlerTakesABoundedOvershoot(t *testing.T) { + t.Parallel() + f := shippedFixture(t) + e := f.broker(t, nil) + topic := Topic{Tenant: "acme", Table: "events"} + p, s := natsRoute(topic, 4, 8) + stream, durable := shippedPartition(p), natsShardDurable("wh-ingest", s) + c, err := e.CreateConsumer(t.Context(), ConsumerConfig{ + Durable: workerDurable, Units: []string{stream + "/" + durable}, + MaxHeld: func() int { return 2 }, + }) + require.NoError(t, err) + c.(*externalConsumer).parts[0].ackWait = 2 * renewEvery + busy := make(chan struct{}) + entered := make(chan struct{}, 16) + stop, _, err := c.Consume(func(*Message) { entered <- struct{}{}; <-busy }, 16) + require.NoError(t, err) + t.Cleanup(stop) + t.Cleanup(func() { close(busy) }) // first, so stop's cleanup is not left waiting + for i := range 10 { + require.NoError(t, e.Publish(t.Context(), topic, []byte(strconv.Itoa(i)))) + } + <-entered + state := func() (string, uint64) { + cons, err := f.admin.Consumer(t.Context(), stream, durable) + require.NoError(t, err) + return pinnedClient(cons.CachedInfo()), cons.CachedInfo().Delivered.Consumer + } + require.Eventually(t, func() bool { _, d := state(); return d >= 2 }, 5*time.Second, 10*time.Millisecond) + pin, _ := state() + require.NotEmpty(t, pin) + // The row in the handler stops counting at its ack_wait (10s here), as + // the server would redeliver it then, so one more is taken; after that + // nothing, however many renewals follow. + time.Sleep(18 * time.Second) // the last row-taking renewal is at about 15s + _, settled := state() + time.Sleep(11 * time.Second) // two more renewals + now, delivered := state() + assert.Equal(t, pin, now, "the pin was kept") + assert.Equal(t, settled, delivered, "renewals past the bound take no rows") + assert.Equal(t, uint64(5), delivered, "the cap of 2, one ack_wait (10s) of renewals at 5s, and the expired row's slot") +} + +// A halted unit keeps its pin for its ack_wait counted from when what it +// fetched has reached the handler, not from the halt: a slow drain does not +// eat the time a handover then waits for those rows to settle. +func TestExternalNATS_HaltKeepsThePinPastASlowDrain(t *testing.T) { + t.Parallel() + f := shippedFixture(t) + e := f.broker(t, nil) + topic := Topic{Tenant: "acme", Table: "events"} + p, s := natsRoute(topic, 4, 8) + c, err := e.CreateConsumer(t.Context(), ConsumerConfig{ + Durable: workerDurable, + Units: []string{shippedPartition(p) + "/" + natsShardDurable("wh-ingest", s)}, + }) + require.NoError(t, err) + c.(*externalConsumer).parts[0].ackWait = 2 * renewEvery // 10s + release := make(chan struct{}) + var once sync.Once + stop, _, err := c.Consume(func(*Message) { once.Do(func() { <-release }) }, 16) + require.NoError(t, err) + t.Cleanup(stop) + t.Cleanup(func() { closeOnceMQ(release) }) + for i := range 3 { + require.NoError(t, e.Publish(t.Context(), topic, []byte(strconv.Itoa(i)))) + } + require.Eventually(t, func() bool { return unitPin(t, f, topic) != "" }, 5*time.Second, 10*time.Millisecond) + pin := unitPin(t, f, topic) + halted := make(chan struct{}) + go func() { c.(Halter).Halt(); close(halted) }() + time.Sleep(15 * time.Second) // the drain outlasts the ack_wait + closeOnceMQ(release) + <-halted + time.Sleep(10 * time.Second) // within ack_wait of the drain, past it from the halt + assert.Equal(t, pin, unitPin(t, f, topic), "the pin is kept for ack_wait after the drain") +} + +func closeOnceMQ(ch chan struct{}) { + select { + case <-ch: + default: + close(ch) + } +} + +// The same bound holds where the prefetch share, not MaxHeld, is the cap: a +// blocked unit's renewal rows stop one ack_wait of renewals past its share. +func TestExternalNATS_BlockedHandlerOvershootBoundedByTheShare(t *testing.T) { + t.Parallel() + f := shippedFixture(t) + e := f.broker(t, nil) + topic := Topic{Tenant: "acme", Table: "events"} + p, s := natsRoute(topic, 4, 8) + stream, durable := shippedPartition(p), natsShardDurable("wh-ingest", s) + c, err := e.CreateConsumer(t.Context(), ConsumerConfig{ + Durable: workerDurable, Units: []string{stream + "/" + durable}, + MaxHeld: func() int { return 1000 }, + }) + require.NoError(t, err) + c.(*externalConsumer).parts[0].ackWait = 2 * renewEvery + busy := make(chan struct{}) + stop, _, err := c.Consume(func(*Message) { <-busy }, 2) + require.NoError(t, err) + t.Cleanup(stop) + t.Cleanup(func() { close(busy) }) + for i := range 12 { + require.NoError(t, e.Publish(t.Context(), topic, []byte(strconv.Itoa(i)))) + } + delivered := func() uint64 { + cons, err := f.admin.Consumer(t.Context(), stream, durable) + require.NoError(t, err) + return cons.CachedInfo().Delivered.Consumer + } + time.Sleep(16 * time.Second) + settled := delivered() + time.Sleep(11 * time.Second) // two more renewals + assert.Equal(t, settled, delivered(), "renewals past the bound take no rows") + // One in the handler, a share of 2 queued, and 2 renewals' worth past it; + // the handled row's expiry frees no room here, the queue being the cap. + assert.Equal(t, uint64(5), settled) +} diff --git a/internal/mq/lease.go b/internal/mq/lease.go new file mode 100644 index 000000000..9c91276b5 --- /dev/null +++ b/internal/mq/lease.go @@ -0,0 +1,421 @@ +package mq + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "sync" + "time" + + "github.com/nats-io/nats.go/jetstream" + "github.com/nats-io/nuid" + + "github.com/Wave-RF/WaveHouse/internal/coord" +) + +// Lease timings, client-go's leader-election defaults. A holder steps down +// when it has not renewed within the renew deadline, before any candidate can +// take the lease over (the lease duration). +const ( + defaultLeaseDuration = 15 * time.Second + defaultRenewDeadline = 10 * time.Second +) + +// leaseKeyPrefix leads every lease's key in the bucket. +const leaseKeyPrefix = "lease." + +// LeaseOption adjusts a lease coordinator's timings (tests shorten them). +type LeaseOption func(*leaseTimings) + +type leaseTimings struct { + duration, renewDeadline, renewEvery time.Duration +} + +// WithLeaseTimings sets how long a candidate must see a lease unchanged +// before taking it over, how long a holder keeps a lease it cannot renew, and +// how often it renews. Each must be shorter than the one before. +func WithLeaseTimings(duration, renewDeadline, renewEvery time.Duration) LeaseOption { + return func(t *leaseTimings) { *t = leaseTimings{duration, renewDeadline, renewEvery} } +} + +// leaseValue is what a lease's key holds: who holds it, for how long a +// candidate must see it unchanged, the holding coordinator's session, so a +// coordinator recognizes its own writes and no one else's (two processes +// misconfigured with one instance_id still contend), and the term's own id, +// so a term recognizes its own writes and not a later term's. +type leaseValue struct { + Holder string `json:"holder"` + DurationMS int64 `json:"duration_ms"` + Session string `json:"session"` + Term string `json:"term"` +} + +// Leases returns a coord.Coordinator over the operator's KV bucket on this +// broker's connection, holding leases as holder (the process's instance_id). +// A bucket that does not exist is ErrTopology: WaveHouse never creates it. +// +// The KV revision a term was acquired at is its fencing token. Expiry is +// judged on the candidate's own clock, never by comparing clocks: a lease is +// taken over only once a candidate has seen the same revision unchanged for +// the lease duration. The bucket keeps no per-key TTL, since a renewal cannot +// extend one. +func (e *ExternalNATS) Leases(ctx context.Context, bucket, holder string, opts ...LeaseOption) (coord.Coordinator, error) { + kv, err := e.js.KeyValue(ctx, bucket) + if errors.Is(err, jetstream.ErrBucketNotFound) { + return nil, fmt.Errorf("%w: kv bucket %s does not exist; the operator creates it (wavehouse mq manifests)", ErrTopology, bucket) + } + if err != nil { + return nil, fmt.Errorf("kv bucket %s: %w", bucket, err) + } + return newNATSLeases(kv, holder, opts...), nil +} + +func newNATSLeases(kv jetstream.KeyValue, holder string, opts ...LeaseOption) *natsLeases { + t := leaseTimings{defaultLeaseDuration, defaultRenewDeadline, coord.RetryPeriod} + for _, opt := range opts { + opt(&t) + } + return &natsLeases{ + kv: kv, timings: t, + value: leaseValue{Holder: holder, DurationMS: t.duration.Milliseconds(), Session: nuid.Next()}, + held: map[string]*natsTerm{}, + acquiring: map[string]struct{}{}, + seen: map[string]leaseSighting{}, + } +} + +// natsLeases is the coord.Coordinator over a KV bucket. +type natsLeases struct { + kv jetstream.KeyValue + timings leaseTimings + value leaseValue + + // mu guards the fields below and is never held across a request, so a + // slow campaign cannot hold up another term ending. + mu sync.Mutex + closed bool + held map[string]*natsTerm + // acquiring holds the names a TryAcquire is campaigning for. + acquiring map[string]struct{} + // seen is, per lease another holder has, the revision last seen and when + // it was first seen on this process's monotonic clock. + seen map[string]leaseSighting +} + +type leaseSighting struct { + revision uint64 + since time.Time +} + +var ( + _ coord.Coordinator = (*natsLeases)(nil) + _ coord.Observer = (*natsLeases)(nil) +) + +// TryAcquire implements coord.Coordinator. +func (l *natsLeases) TryAcquire(ctx context.Context, name string) (coord.Term, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + l.mu.Lock() + if l.closed { + l.mu.Unlock() + return nil, coord.ErrClosed + } + _, held := l.held[name] + _, busy := l.acquiring[name] + if held || busy { + l.mu.Unlock() + return nil, coord.ErrHeld + } + l.acquiring[name] = struct{}{} + l.mu.Unlock() + defer func() { + l.mu.Lock() + delete(l.acquiring, name) + l.mu.Unlock() + }() + + key := leaseKeyPrefix + name + v := l.value + v.Term = nuid.Next() + val, err := json.Marshal(v) + if err != nil { + return nil, err + } + // The renew deadline runs from before the write, never from its answer: + // a candidate's clock can start as soon as the write is stored. + sent := time.Now() + rev, err := l.campaign(ctx, name, key, val) + if err != nil { + return nil, err + } + + l.mu.Lock() + if l.closed { + // Close ran while the write was in flight; hand the lease straight back. + l.mu.Unlock() + _ = l.kv.Delete(ctx, key, jetstream.LastRevision(rev)) + return nil, coord.ErrClosed + } + delete(l.seen, name) + tctx, cancel := context.WithCancel(context.Background()) + t := &natsTerm{ + owner: l, name: name, key: key, val: val, token: rev, rev: rev, + done: make(chan struct{}), stopped: make(chan struct{}), ctx: tctx, cancel: cancel, + } + l.held[name] = t + l.mu.Unlock() + go t.renew(sent) + return t, nil +} + +// campaign writes this coordinator's value to key if the lease is free, +// quiet past its duration, or already this coordinator's own write, and +// returns the revision written; coord.ErrHeld otherwise. +func (l *natsLeases) campaign(ctx context.Context, name, key string, val []byte) (uint64, error) { + entry, err := l.kv.Get(ctx, key) + var rev uint64 + switch { + case errors.Is(err, jetstream.ErrKeyNotFound): + // Never held, or resigned: a delete marker is as good as absent. + rev, err = l.kv.Create(ctx, key, val) + case err != nil: + default: + var cur leaseValue + mine := json.Unmarshal(entry.Value(), &cur) == nil && cur.Session == l.value.Session + if !mine && !l.expired(name, entry.Revision(), cur) { + return 0, coord.ErrHeld + } + // This coordinator's own write outlived a term it gave up (a renewal + // past its deadline), or the holder went quiet: take it at the + // revision seen, so a renewal in between wins instead. + rev, err = l.kv.Update(ctx, key, val, entry.Revision()) + } + if casConflict(err) { + return 0, coord.ErrHeld + } + if err != nil { + return 0, fmt.Errorf("coord lease %s: %w", name, err) + } + return rev, nil +} + +// Held implements coord.Observer: a lease this coordinator holds is held; a +// key that is absent, resigned, or this coordinator's own leftover write is +// not; another holder's is held until this coordinator has seen its revision +// unchanged for the lease duration, the clock TryAcquire keeps. +func (l *natsLeases) Held(ctx context.Context, name string) (bool, error) { + l.mu.Lock() + _, mine := l.held[name] + l.mu.Unlock() + if mine { + return true, nil + } + entry, err := l.kv.Get(ctx, leaseKeyPrefix+name) + if errors.Is(err, jetstream.ErrKeyNotFound) { + return false, nil + } + if err != nil { + return false, fmt.Errorf("coord lease %s: %w", name, err) + } + var cur leaseValue + if json.Unmarshal(entry.Value(), &cur) == nil && cur.Session == l.value.Session { + return false, nil + } + return !l.expired(name, entry.Revision(), cur), nil +} + +// expired reports whether another holder's lease at revision has been seen +// unchanged for its duration (the holder's own, or this coordinator's when +// the value does not say), starting the clock on a revision not seen before. +func (l *natsLeases) expired(name string, revision uint64, cur leaseValue) bool { + now := time.Now() + l.mu.Lock() + defer l.mu.Unlock() + s, ok := l.seen[name] + if !ok || s.revision != revision { + l.seen[name] = leaseSighting{revision: revision, since: now} + return false + } + d := l.timings.duration + if cur.DurationMS > 0 { + d = time.Duration(cur.DurationMS) * time.Millisecond + } + return now.Sub(s.since) >= d +} + +// Close implements coord.Coordinator: every held term is resigned, deleting +// its key, so a candidate need not wait out the lease duration. +func (l *natsLeases) Close(ctx context.Context) error { + l.mu.Lock() + l.closed = true + terms := make([]*natsTerm, 0, len(l.held)) + for _, t := range l.held { + terms = append(terms, t) + } + l.mu.Unlock() + var errs []error + for _, t := range terms { + errs = append(errs, t.Resign(ctx)) + } + return errors.Join(errs...) +} + +// casConflict reports a write refused because the key is not at the +// revision it expected: someone else wrote it first. Create over a delete +// marker returns the server's error unmapped, and a replicated bucket +// reports the conflict under a code of its own. +func casConflict(err error) bool { + if errors.Is(err, jetstream.ErrKeyExists) || errors.Is(err, jetstream.ErrKeyRevisionMismatch) { + return true + } + var apiErr *jetstream.APIError + return errors.As(err, &apiErr) && (apiErr.ErrorCode == jetstream.JSErrCodeStreamWrongLastSequence || + apiErr.ErrorCode == jetstream.JSErrCodeStreamWrongLastSequenceConstant) +} + +// natsTerm is one holding of a lease, renewed until it ends. +type natsTerm struct { + owner *natsLeases + name string + key string + val []byte + token uint64 + + // rev is the revision last written, owned by the renew loop while it + // runs and read by Resign after it has stopped. + rev uint64 + + done chan struct{} + stopped chan struct{} + // ctx bounds the renew loop's requests; Resign cancels it, so an + // in-flight renewal never holds a resign up. + ctx context.Context + cancel context.CancelFunc + err error // guarded by owner.mu, set before done closes +} + +var _ coord.Term = (*natsTerm)(nil) + +func (t *natsTerm) Name() string { return t.name } +func (t *natsTerm) Token() uint64 { return t.token } +func (t *natsTerm) Done() <-chan struct{} { return t.done } + +func (t *natsTerm) Err() error { + t.owner.mu.Lock() + defer t.owner.mu.Unlock() + return t.err +} + +// renew rewrites the key at the revision last written, every renewEvery. A +// write at the wrong revision means another holder took the lease. The term +// also ends the moment the renew deadline passes without a stored renewal, +// timed from when that renewal was sent (last): a candidate may start its +// lease-duration clock as soon as the write is stored, so the holder steps +// down no later than renewDeadline after it, before any takeover. +func (t *natsTerm) renew(last time.Time) { + defer close(t.stopped) + tm := t.owner.timings + deadline := time.NewTimer(time.Until(last.Add(tm.renewDeadline))) + defer deadline.Stop() + tick := time.NewTicker(tm.renewEvery) + defer tick.Stop() + var lastErr error + for { + select { + case <-t.ctx.Done(): + return + case <-deadline.C: + err := fmt.Errorf("%w: %s not renewed within %s", coord.ErrLost, t.name, tm.renewDeadline) + if lastErr != nil { + err = fmt.Errorf("%w: %w", err, lastErr) + } + t.end(err) + return + case <-tick.C: + } + sent := time.Now() + ctx, cancel := context.WithDeadline(t.ctx, last.Add(tm.renewDeadline)) + rev, err := t.owner.kv.Update(ctx, t.key, t.val, t.rev) + if casConflict(err) { + rev, err = t.adoptLostReply(ctx) + } + cancel() + switch { + case err == nil: + t.rev, last = rev, sent + deadline.Reset(time.Until(last.Add(tm.renewDeadline))) + case errors.Is(err, coord.ErrLost): + t.end(err) + return + default: + // Retried next tick; the deadline timer ends the term on time. + lastErr = err + } + } +} + +// adoptLostReply handles a renewal refused for its revision: when the key +// holds this term's own value at a later revision, an earlier renewal was +// stored and only its answer lost, so the term carries on from there. +// Anything else means another holder has the lease. +func (t *natsTerm) adoptLostReply(ctx context.Context) (uint64, error) { + entry, err := t.owner.kv.Get(ctx, t.key) + if err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) { + return 0, err + } + if err == nil && entry.Revision() > t.rev && string(entry.Value()) == string(t.val) { + return t.owner.kv.Update(ctx, t.key, t.val, entry.Revision()) + } + return 0, fmt.Errorf("%w: %s taken by another holder", coord.ErrLost, t.name) +} + +// end records why the term ended and closes Done, once; it frees the name +// for this coordinator to campaign again. +func (t *natsTerm) end(err error) bool { + l := t.owner + l.mu.Lock() + defer l.mu.Unlock() + if l.held[t.name] != t { + return false + } + delete(l.held, t.name) + t.err = err + close(t.done) + return true +} + +// Resign implements coord.Term: it stops renewing, aborting a renewal in +// flight, and deletes the key at the revision last written, so a lease +// someone else has taken is left alone. If ctx ends first the term still +// ends here, and the lease runs out on its own. +func (t *natsTerm) Resign(ctx context.Context) error { + t.cancel() + select { + case <-t.stopped: + case <-ctx.Done(): + t.end(nil) + return fmt.Errorf("resign coord lease %s: %w", t.name, ctx.Err()) + } + if !t.end(nil) { + return nil // already ended: lost, or resigned before + } + err := t.owner.kv.Delete(ctx, t.key, jetstream.LastRevision(t.rev)) + if casConflict(err) { + // A renewal the cancel cut short may still have been stored: delete + // this term's own value at its later revision, and nothing else. + var entry jetstream.KeyValueEntry + if entry, err = t.owner.kv.Get(ctx, t.key); err == nil && entry.Revision() > t.rev && string(entry.Value()) == string(t.val) { + err = t.owner.kv.Delete(ctx, t.key, jetstream.LastRevision(entry.Revision())) + } else if err == nil || errors.Is(err, jetstream.ErrKeyNotFound) { + err = nil + } + } + if err != nil && !casConflict(err) { + // The term has ended here; the lease runs out on its own. + return fmt.Errorf("resign coord lease %s: %w", t.name, err) + } + return nil +} diff --git a/internal/mq/lease_test.go b/internal/mq/lease_test.go new file mode 100644 index 000000000..eb8ff477f --- /dev/null +++ b/internal/mq/lease_test.go @@ -0,0 +1,513 @@ +//go:build integration + +package mq + +import ( + "context" + "encoding/json" + "errors" + "sync" + "testing" + "time" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/Wave-RF/WaveHouse/internal/coord" + "github.com/Wave-RF/WaveHouse/internal/coord/coordtest" + "github.com/Wave-RF/WaveHouse/internal/mq/natstest" +) + +// Shortened lease timings, in the production ratios' order: a holder steps +// down (renew deadline) before a candidate may take over (duration). +const ( + testLeaseDuration = 400 * time.Millisecond + testRenewDeadline = 300 * time.Millisecond + testRenewEvery = 50 * time.Millisecond +) + +func testTimings() LeaseOption { + return WithLeaseTimings(testLeaseDuration, testRenewDeadline, testRenewEvery) +} + +// leaseFixture is a server with only the shipped lease bucket on it. +func leaseFixture(t *testing.T) *natsFixture { + t.Helper() + f := newNATSFixture(t) + tp := shippedTopology(t) + tp.Streams, tp.Consumers = nil, nil + require.NoError(t, f.create(t.Context(), tp)) + return f +} + +// bucketAs opens the lease bucket as the restricted wavehouse user, on a +// connection of its own: one process's view. +func (f *natsFixture) bucketAs(t *testing.T) jetstream.KeyValue { + t.Helper() + kv, err := f.connect(t, "wavehouse").KeyValue(t.Context(), natstest.CoordBucket) + require.NoError(t, err) + return kv +} + +// The shared suite, each pair two processes' coordinators connected as the +// restricted wavehouse user. Loss is the operator overwriting the key, as a +// takeover would. +func TestLeases_Conformance(t *testing.T) { + t.Parallel() + var cur *natsFixture + coordtest.Conformance(t, func(t *testing.T) (a, b coord.Coordinator) { + cur = leaseFixture(t) + return newNATSLeases(cur.bucketAs(t), "a", testTimings()), newNATSLeases(cur.bucketAs(t), "b", testTimings()) + }, coordtest.WithLoss(func(t *testing.T, name string) { + kv, err := cur.admin.KeyValue(t.Context(), natstest.CoordBucket) + require.NoError(t, err) + _, err = kv.Put(t.Context(), leaseKeyPrefix+name, []byte(`{"holder":"operator"}`)) + require.NoError(t, err) + }), coordtest.WithWait(2*time.Second)) +} + +// The token is the KV revision the term was taken at, and grows across every +// kind of handover: a resign, a takeover of a quiet lease, and a resume. +func TestLeases_TokensAreMonotonicRevisions(t *testing.T) { + t.Parallel() + f := leaseFixture(t) + a := newNATSLeases(f.bucketAs(t), "a", testTimings()) + b := newNATSLeases(f.bucketAs(t), "b", testTimings()) + t.Cleanup(func() { _ = a.Close(context.Background()); _ = b.Close(context.Background()) }) + admin, err := f.admin.KeyValue(t.Context(), natstest.CoordBucket) + require.NoError(t, err) + + var last uint64 + check := func(term coord.Term) { + t.Helper() + assert.Greater(t, term.Token(), last) + last = term.Token() + // Renewals move the revision on; the token stays the acquisition's. + hist, err := admin.History(t.Context(), leaseKeyPrefix+"sweeper") + require.NoError(t, err) + require.Contains(t, revisions(hist), term.Token()) + } + for i := range 3 { + c := []*natsLeases{a, b}[i%2] + term, err := c.TryAcquire(t.Context(), "sweeper") + require.NoError(t, err) + check(term) + time.Sleep(3 * testRenewEvery) + require.NoError(t, term.Resign(t.Context())) + } + + // A holder gone quiet: b takes over once the revision has sat still. + _, err = admin.Put(t.Context(), leaseKeyPrefix+"sweeper", []byte(`{"holder":"gone","duration_ms":400}`)) + require.NoError(t, err) + term := acquireEventually(t, b, "sweeper", 5*time.Second) + check(term) +} + +func revisions(entries []jetstream.KeyValueEntry) []uint64 { + out := make([]uint64, len(entries)) + for i, e := range entries { + out[i] = e.Revision() + } + return out +} + +func acquireEventually(t *testing.T, c coord.Coordinator, name string, within time.Duration) coord.Term { + t.Helper() + deadline := time.Now().Add(within) + for { + term, err := c.TryAcquire(t.Context(), name) + if err == nil { + return term + } + require.ErrorIs(t, err, coord.ErrHeld) + require.True(t, time.Now().Before(deadline), "%s never acquired", name) + time.Sleep(testRenewEvery / 2) + } +} + +// stallableKV is a bucket whose writes can be held up, as a holder in a long +// GC pause or behind a partition would be. +type stallableKV struct { + jetstream.KeyValue + mu sync.Mutex + stalled bool + // loseReplies stores each write, then withholds its answer. + loseReplies bool +} + +func (s *stallableKV) stall() { + s.mu.Lock() + defer s.mu.Unlock() + s.stalled = true +} + +func (s *stallableKV) Update(ctx context.Context, key string, value []byte, revision uint64) (uint64, error) { + s.mu.Lock() + stalled, lose := s.stalled, s.loseReplies + s.mu.Unlock() + if lose { + if _, err := s.KeyValue.Update(ctx, key, value, revision); err != nil { + return 0, err + } + } + if stalled || lose { + <-ctx.Done() + return 0, ctx.Err() + } + return s.KeyValue.Update(ctx, key, value, revision) +} + +// A live holder is never replaced; a stalled one steps down at its renew +// deadline, and a candidate takes over only once it has seen the revision +// unchanged for the lease duration on its own clock. +func TestLeases_StalledHolderIsReplacedAfterTheWindow(t *testing.T) { + t.Parallel() + f := leaseFixture(t) + akv := &stallableKV{KeyValue: f.bucketAs(t)} + a := newNATSLeases(akv, "a", testTimings()) + b := newNATSLeases(f.bucketAs(t), "b", testTimings()) + t.Cleanup(func() { _ = a.Close(context.Background()); _ = b.Close(context.Background()) }) + + held, err := a.TryAcquire(t.Context(), "sweeper") + require.NoError(t, err) + + // Renewing, a holder outlasts many lease durations of campaigning. + until := time.Now().Add(3 * testLeaseDuration) + for time.Now().Before(until) { + _, err := b.TryAcquire(t.Context(), "sweeper") + require.ErrorIs(t, err, coord.ErrHeld) + time.Sleep(testRenewEvery / 2) + } + require.NoError(t, held.Err()) + + akv.stall() + stalledAt := time.Now() + var term coord.Term + for term == nil { + got, err := b.TryAcquire(t.Context(), "sweeper") + if err == nil { + term = got + break + } + require.ErrorIs(t, err, coord.ErrHeld) + require.Less(t, time.Since(stalledAt), 5*time.Second, "never taken over") + time.Sleep(testRenewEvery / 2) + } + tookOver := time.Now() + + select { + case <-held.Done(): + default: + t.Fatal("the stalled holder had not stepped down when the lease was taken over") + } + require.ErrorIs(t, held.Err(), coord.ErrLost) + assert.Greater(t, term.Token(), held.Token()) + // The holder's last write landed at most one renewal before the stall, + // and the window runs from b's first sighting of it. + assert.GreaterOrEqual(t, tookOver.Sub(stalledAt), testLeaseDuration-testRenewEvery) +} + +// A renewal stuck on an unanswering server never holds a resign up: Resign +// aborts it, however long the renew deadline. +func TestLeases_ResignAbortsAStuckRenewal(t *testing.T) { + t.Parallel() + f := leaseFixture(t) + akv := &stallableKV{KeyValue: f.bucketAs(t)} + a := newNATSLeases(akv, "a", WithLeaseTimings(2*time.Hour, time.Hour, testRenewEvery)) + t.Cleanup(func() { _ = a.Close(context.Background()) }) + term, err := a.TryAcquire(t.Context(), "sweeper") + require.NoError(t, err) + akv.stall() + time.Sleep(3 * testRenewEvery) // a renewal is now waiting on the stall + start := time.Now() + require.NoError(t, term.Resign(t.Context())) + assert.Less(t, time.Since(start), time.Second) + assertTermEnded(t, term) + require.NoError(t, term.Err()) +} + +// A renewal that was stored but whose answer Resign cut off still leaves +// the key this term's own; Resign deletes it, so the lease is free at once. +func TestLeases_ResignClearsARenewalItCutShort(t *testing.T) { + t.Parallel() + f := leaseFixture(t) + akv := &stallableKV{KeyValue: f.bucketAs(t)} + a := newNATSLeases(akv, "a", WithLeaseTimings(2*time.Hour, time.Hour, testRenewEvery)) + b := newNATSLeases(f.bucketAs(t), "b", WithLeaseTimings(2*time.Hour, time.Hour, testRenewEvery)) + t.Cleanup(func() { _ = a.Close(context.Background()); _ = b.Close(context.Background()) }) + term, err := a.TryAcquire(t.Context(), "sweeper") + require.NoError(t, err) + admin, err := f.admin.KeyValue(t.Context(), natstest.CoordBucket) + require.NoError(t, err) + akv.mu.Lock() + akv.loseReplies = true + akv.mu.Unlock() + require.Eventually(t, func() bool { + e, err := admin.Get(t.Context(), leaseKeyPrefix+"sweeper") + return err == nil && e.Revision() > term.Token() + }, 2*time.Second, testRenewEvery/2, "a renewal is stored with its answer withheld") + require.NoError(t, term.Resign(t.Context())) + _, err = admin.Get(t.Context(), leaseKeyPrefix+"sweeper") + require.ErrorIs(t, err, jetstream.ErrKeyNotFound, "the resign deleted the renewal it cut short") + _, err = b.TryAcquire(t.Context(), "sweeper") + require.NoError(t, err, "free at once, with no lease duration to wait out") +} + +// Each term writes its own id, so a term never mistakes its coordinator's +// later term for itself. +func TestLeases_TermsWriteTheirOwnIDs(t *testing.T) { + t.Parallel() + f := leaseFixture(t) + a := newNATSLeases(f.bucketAs(t), "a", testTimings()) + t.Cleanup(func() { _ = a.Close(context.Background()) }) + admin, err := f.admin.KeyValue(t.Context(), natstest.CoordBucket) + require.NoError(t, err) + var ids []string + for range 2 { + term, err := a.TryAcquire(t.Context(), "sweeper") + require.NoError(t, err) + e, err := admin.Get(t.Context(), leaseKeyPrefix+"sweeper") + require.NoError(t, err) + var v leaseValue + require.NoError(t, json.Unmarshal(e.Value(), &v)) + assert.Equal(t, "a", v.Holder) + assert.NotEmpty(t, v.Term) + ids = append(ids, v.Term) + require.NoError(t, term.Resign(t.Context())) + } + assert.NotEqual(t, ids[0], ids[1]) +} + +// gatedCreateKV holds each Create, once stored, until released. +type gatedCreateKV struct { + jetstream.KeyValue + stored, release chan struct{} +} + +func (g *gatedCreateKV) Create(ctx context.Context, key string, value []byte, opts ...jetstream.KVCreateOpt) (uint64, error) { + rev, err := g.KeyValue.Create(ctx, key, value, opts...) + close(g.stored) + <-g.release + return rev, err +} + +// A Close that lands while a TryAcquire's write is in flight wins: the +// campaign hands the lease straight back and reports ErrClosed. +func TestLeases_CloseDuringACampaign(t *testing.T) { + t.Parallel() + f := leaseFixture(t) + g := &gatedCreateKV{KeyValue: f.bucketAs(t), stored: make(chan struct{}), release: make(chan struct{})} + a := newNATSLeases(g, "a", testTimings()) + got := make(chan error, 1) + go func() { + _, err := a.TryAcquire(context.Background(), "sweeper") + got <- err + }() + <-g.stored + require.NoError(t, a.Close(t.Context())) + close(g.release) + require.ErrorIs(t, <-got, coord.ErrClosed) + admin, err := f.admin.KeyValue(t.Context(), natstest.CoordBucket) + require.NoError(t, err) + _, err = admin.Get(t.Context(), leaseKeyPrefix+"sweeper") + require.ErrorIs(t, err, jetstream.ErrKeyNotFound, "the lease was handed back") +} + +func assertTermEnded(t *testing.T, term coord.Term) { + t.Helper() + select { + case <-term.Done(): + default: + t.Fatal("the term is still live") + } +} + +// Close resigns by deleting the key, so a candidate takes the lease at once +// rather than after the lease duration. +func TestLeases_CloseHandsOverAtOnce(t *testing.T) { + t.Parallel() + f := leaseFixture(t) + a := newNATSLeases(f.bucketAs(t), "a", WithLeaseTimings(time.Hour, 50*time.Minute, testRenewEvery)) + b := newNATSLeases(f.bucketAs(t), "b", testTimings()) + t.Cleanup(func() { _ = b.Close(context.Background()) }) + _, err := a.TryAcquire(t.Context(), "sweeper") + require.NoError(t, err) + _, err = b.TryAcquire(t.Context(), "sweeper") + require.ErrorIs(t, err, coord.ErrHeld) + require.NoError(t, a.Close(t.Context())) + _, err = b.TryAcquire(t.Context(), "sweeper") + require.NoError(t, err) +} + +// A coordinator whose term ended without the key moving on (its renewals +// failed past the deadline) resumes its own write without the wait. +func TestLeases_ResumesItsOwnWrite(t *testing.T) { + t.Parallel() + f := leaseFixture(t) + akv := &stallableKV{KeyValue: f.bucketAs(t)} + a := newNATSLeases(akv, "a", WithLeaseTimings(time.Hour, testRenewDeadline, testRenewEvery)) + t.Cleanup(func() { _ = a.Close(context.Background()) }) + first, err := a.TryAcquire(t.Context(), "sweeper") + require.NoError(t, err) + akv.stall() + select { + case <-first.Done(): + case <-time.After(5 * time.Second): + t.Fatal("the term outlived its renew deadline") + } + require.ErrorIs(t, first.Err(), coord.ErrLost) + akv.mu.Lock() + akv.stalled = false + akv.mu.Unlock() + second, err := a.TryAcquire(t.Context(), "sweeper") + require.NoError(t, err, "its own write, not a stranger's: no wait") + assert.Greater(t, second.Token(), first.Token()) +} + +// Leases refuses a bucket the operator has not created, and boot with the +// bucket in the topology waits for it and then refuses naming it. +func TestNewNATS_RefusesAMissingLeaseBucket(t *testing.T) { + t.Parallel() + f := newNATSFixture(t) + tp := shippedTopology(t) + tp.KeyValues = nil + f.apply(t, tp) + + e := f.broker(t, nil) + _, err := e.Leases(t.Context(), natstest.CoordBucket, "a") + require.ErrorIs(t, err, ErrTopology) + assert.ErrorContains(t, err, "kv bucket wh_coord does not exist") + + _, err = NewNATS(t.Context(), NATSConfig{ + URLs: []string{f.server.ClientURL()}, User: "wavehouse", PasswordFile: writeSecret(t, fixturePassword("wavehouse")), + Topology: NATSTopology{Partitions: 4, Shards: 8, CoordBucket: natstest.CoordBucket}, + TopologyWait: 300 * time.Millisecond, + }) + var terr *TopologyError + require.ErrorAs(t, err, &terr) + assert.ErrorContains(t, err, "kv bucket wh_coord: bucket: does not exist") +} + +// Through the broker, as the restricted user, a lease works end to end. +func TestExternalNATS_Leases(t *testing.T) { + t.Parallel() + f := shippedFixture(t) + e := f.broker(t, func(c *NATSConfig) { c.Topology.CoordBucket = natstest.CoordBucket }) + c, err := e.Leases(t.Context(), natstest.CoordBucket, "a", testTimings()) + require.NoError(t, err) + term, err := c.TryAcquire(t.Context(), "sweeper") + require.NoError(t, err) + time.Sleep(3 * testRenewEvery) + require.NoError(t, term.Err(), "renewals as the wavehouse user") + require.NoError(t, c.Close(t.Context())) + require.NoError(t, term.Err()) +} + +// The shipped permissions let the wavehouse user do exactly what a lease +// needs: read and write lease keys. Not the bucket itself, nor other keys. +func TestNATSPermissions_RefuseBucketChanges(t *testing.T) { + t.Parallel() + f := leaseFixture(t) + js := f.connect(t, "wavehouse", nats.ErrorHandler(func(*nats.Conn, *nats.Subscription, error) {})) + ctx := t.Context() + kv, err := js.KeyValue(ctx, natstest.CoordBucket) + require.NoError(t, err) + denied := func(what string, err error) { + t.Helper() + require.Error(t, err, what) + assert.False(t, errors.Is(err, context.Canceled), what) + } + + rev, err := kv.Create(ctx, leaseKeyPrefix+"x", []byte("v")) + require.NoError(t, err, "create a lease key") + rev, err = kv.Update(ctx, leaseKeyPrefix+"x", []byte("v"), rev) + require.NoError(t, err, "renew a lease key") + _, err = kv.Get(ctx, leaseKeyPrefix+"x") + require.NoError(t, err, "read a lease key") + require.NoError(t, kv.Delete(ctx, leaseKeyPrefix+"x", jetstream.LastRevision(rev)), "resign a lease key") + + denied("write a key outside lease.", call(ctx, func(ctx context.Context) error { + _, err := kv.Put(ctx, "other", []byte("v")) + return err + })) + denied("read a key outside lease.", call(ctx, func(ctx context.Context) error { + _, err := kv.Get(ctx, "other") + return err + })) + denied("create a bucket", call(ctx, func(ctx context.Context) error { + _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: "rogue"}) + return err + })) + denied("delete the bucket", call(ctx, func(ctx context.Context) error { + return js.DeleteKeyValue(ctx, natstest.CoordBucket) + })) + denied("purge the bucket", call(ctx, func(ctx context.Context) error { + s, err := js.Stream(ctx, "KV_"+natstest.CoordBucket) + if err != nil { + return err + } + return s.Purge(ctx) + })) + _, err = f.admin.KeyValue(ctx, natstest.CoordBucket) + require.NoError(t, err, "the bucket is still there") +} + +// Every rule the verifier holds the lease bucket to, one mutation each. It is +// checked only when the process holds leases there (CoordBucket set). +func TestLeases_VerifierChecksTheBucket(t *testing.T) { + t.Parallel() + const obj = "kv bucket wh_coord" + coordSpec := NATSTopology{Partitions: 4, Shards: 8, CoordBucket: natstest.CoordBucket} + bucket := func(mut func(*jetstream.KeyValueConfig)) func(*fixtureTopology) { + return func(tp *fixtureTopology) { mut(&tp.KeyValues[0]) } + } + // raw stands the bucket's stream up by hand, for what CreateKeyValue + // would not create. + raw := func(mut func(*jetstream.StreamConfig)) func(*fixtureTopology) { + return func(tp *fixtureTopology) { + tp.KeyValues = nil + cfg := jetstream.StreamConfig{ + Name: "KV_wh_coord", Subjects: []string{"$KV.wh_coord.>"}, MaxMsgsPerSubject: 1, + AllowDirect: true, Storage: jetstream.FileStorage, Discard: jetstream.DiscardNew, + } + mut(&cfg) + tp.Streams = append(tp.Streams, cfg) + } + } + cases := []struct { + name string + mutate func(*fixtureTopology) + spec NATSTopology + sev FindingSeverity + object string + field string + }{ + {"missing", func(tp *fixtureTopology) { tp.KeyValues = nil }, coordSpec, FindingRequired, obj, "bucket"}, + {"named elsewhere", nil, NATSTopology{Partitions: 4, Shards: 8, CoordBucket: "other"}, FindingRequired, "kv bucket other", "bucket"}, + {"ttl", bucket(func(kv *jetstream.KeyValueConfig) { kv.TTL = time.Hour }), coordSpec, FindingRequired, obj, "ttl"}, + {"no direct get", raw(func(s *jetstream.StreamConfig) { s.AllowDirect = false }), coordSpec, FindingRequired, obj, "allow_direct"}, + {"keeps no value", raw(func(s *jetstream.StreamConfig) { s.MaxMsgsPerSubject = 0 }), coordSpec, FindingRequired, obj, "history"}, + {"memory storage", bucket(func(kv *jetstream.KeyValueConfig) { kv.Storage = jetstream.MemoryStorage }), coordSpec, FindingRecommended, obj, "storage"}, + } + f := newNATSFixture(t) + js := f.connect(t, "wavehouse") + for _, tc := range cases { + f.reset(t) + tp := shippedTopology(t) + if tc.mutate != nil { + tc.mutate(tp) + } + require.NoError(t, f.create(t.Context(), tp), tc.name) + findings, err := verifyNATSTopology(t.Context(), js, tc.spec, nil) + require.NoError(t, err, tc.name) + found := false + for _, got := range findings { + if got.Severity == tc.sev && got.Object == tc.object && got.Field == tc.field { + found = true + } + } + assert.True(t, found, "%s: want %s %s/%s among %v", tc.name, tc.sev, tc.object, tc.field, findings) + } +} diff --git a/internal/mq/mq.go b/internal/mq/mq.go index c1123b674..cff6d042f 100644 --- a/internal/mq/mq.go +++ b/internal/mq/mq.go @@ -5,7 +5,8 @@ // ingest queue, park a message on the dead-letter queue, replay since a time, // drop what is both written and expired — in the types below. How that maps to // subjects, streams, sequences, and consumers is the implementation's -// (EmbeddedNATS), so a broker change lands here once. The behavior below is +// (EmbeddedNATS, or ExternalNATS over an operator-owned cluster), so a broker +// change lands here once. The behavior below is // what mqtest checks: every implementation passes its suite. package mq @@ -239,9 +240,14 @@ type Subscriber interface { } // ConsumerConfig describes a durable, explicit-ack consumer of every event on -// the ingest queue. Zero values take the broker's defaults. +// the ingest queue, or of some of its units (see Sharded). Zero values take +// the broker's defaults. type ConsumerConfig struct { Durable string + // Units limits the consumer to these units of a Sharded broker's queue, + // by the names IngestUnits gives. Nil is every unit. A broker that does + // not implement Sharded refuses a non-nil Units. + Units []string // AckWait is the redelivery timeout: a message not acked within it is // delivered again. AckWait time.Duration @@ -250,8 +256,75 @@ type ConsumerConfig struct { // pauses when its unacked messages hit it (backpressure), and no other // unit's does. MaxAckPending int + // MaxHeld, when set, caps the messages each unit of a Sharded broker's + // queue has delivered to this process and not yet had settled: at the + // cap the consumer fetches no more from that unit than it takes to keep + // its hold on the unit, until one settles. It is read before every fetch, so + // the cap may change while consuming. Nil is no cap but the broker's own. + // A broker that does not implement Sharded ignores it. + MaxHeld func() int } +// Sharded is implemented by a broker whose ingest queue is split into units +// that one consumer at a time owns: every event of one tenant table is in one +// unit, and the broker lets exactly one puller receive from a unit at a time. +// A broker that does not implement it is one queue every consumer shares. +type Sharded interface { + // IngestUnits names the units: the configured ones, in a fixed order the + // same in every process, then any extra ones a lower unit count left + // holding rows, as last read, which are drained like the others. + IngestUnits() (configured, extra []string) + // ResetOrphaned redelivers at once what a unit's previous owner received + // and never settled, when no consumer holds the unit, none has received + // from it or settled a message of it for a while, and rows are awaiting + // a settlement; it reports whether it did, and ErrUnitHeld while a + // consumer holds the unit or was recently active on it. A missing hold + // alone is no proof the owner is gone. Call it only for a unit whose + // owner the caller has other evidence is dead (the membership view), and + // before consuming it, never while consuming it: a reset also redelivers + // what the caller itself holds. + ResetOrphaned(ctx context.Context, unit string) (bool, error) + // Unowned counts the units with rows waiting that no consumer holds and + // none has received from or settled a message of for a while: rows + // nobody is writing. + Unowned(ctx context.Context) (int, error) +} + +// Releaser is implemented by a consumer of a Sharded broker's units. Release +// gives the units up so that their next owner is served at once, rather than +// once the broker judges this consumer gone: call it after Consume's stop, +// once every message delivered has been settled. A unit this consumer never +// held is left alone. +type Releaser interface { + Release(ctx context.Context) error +} + +// Halter is implemented by a Consumer that can stop fetching ahead of its +// stop, so that its caller can write what it holds before the consumer lets +// its units go. Halt returns once no handler invocation is running and none +// will: everything already fetched has reached the handler. The consumer +// keeps its hold on its units until stop, but not indefinitely: a Sharded +// broker's for at least its ack wait after the halt has handed on what it +// fetched. +type Halter interface { + Halt() +} + +// ErrUnitHeld is ResetOrphaned's answer while a consumer still holds the +// unit, or has received from it or settled a message of it too recently to +// be judged gone. +var ErrUnitHeld = errors.New("unit held by a consumer") + +// ErrConsumerMismatch is CreateConsumer's answer when a consumer the broker's +// operator owns no longer fits the ConsumerConfig: its ack_wait is shorter, +// or its max_ack_pending unset. +var ErrConsumerMismatch = errors.New("consumer does not match its config") + +// ErrUnitsUnsupported is CreateConsumer's answer to a ConsumerConfig naming +// units on a broker that does not implement Sharded, or naming one it does +// not have. +var ErrUnitsUnsupported = errors.New("consumer units not supported") + // Consumer is a live durable consumer created by ConsumerManager. type Consumer interface { // Consume delivers each message to handler on the delivery goroutine of @@ -259,7 +332,8 @@ type Consumer interface { // it: one per unit, so a tenant's messages arrive in order, one at a // time, while different units' arrive concurrently — handler must be // safe for that. A handler that blocks holds back its unit's delivery — - // that is the backpressure the ingest worker relies on. About prefetch + // that is the backpressure the ingest worker relies on; a Sharded + // broker's consumer keeps its hold on the unit meanwhile. About prefetch // messages are fetched ahead across the units together: the units when // delivery starts split it, and a queue joined later fetches ahead its // share of it at that point, at least one message each (0 = the client @@ -365,8 +439,8 @@ type Replayer interface { } // Broker is everything the process wiring needs from the MQ: every interface -// above plus the lifecycle and the byte budgets. EmbeddedNATS is the one -// implementation; internal/app depends on this, not on it. +// above plus the lifecycle and the byte budgets. EmbeddedNATS and +// ExternalNATS implement it; internal/app depends on this, not on either. type Broker interface { Publisher Subscriber diff --git a/internal/mq/mq_test.go b/internal/mq/mq_test.go index c8d8d7d6c..b319e2018 100644 --- a/internal/mq/mq_test.go +++ b/internal/mq/mq_test.go @@ -2,10 +2,12 @@ package mq import ( "context" + "errors" "testing" "time" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestMessage_AckNak(t *testing.T) { @@ -72,3 +74,39 @@ func TestWithHeader(t *testing.T) { WithHeader("X-B", "b")(h) assert.Equal(t, Headers{"X-A": {"1", "2"}, "X-B": {"b"}}, h) } + +// OnSettled runs once, on the first settlement attempted, whichever of the +// four it is and whether or not the broker confirms it: the consumer has let +// go of the message either way. NakWithDelay without a delayed form falls +// back through the wrapped Nak. +func TestMessage_OnSettled(t *testing.T) { + t.Parallel() + fail := errors.New("no answer") + for name, settle := range map[string]func(*Message) error{ + "double ack": func(m *Message) error { return m.DoubleAck(context.Background()) }, + "ack": (*Message).Ack, + "nak": (*Message).Nak, + "nak with delay": func(m *Message) error { return m.NakWithDelay(time.Second) }, + } { + t.Run(name, func(t *testing.T) { + t.Parallel() + var calls, fired int + err := fail + cb := func() error { calls++; return err } + m := NewMessage(context.Background(), Topic{Tenant: "acme", Table: "t"}, nil, time.Now(), + func(context.Context) error { return cb() }, cb, cb) + m.OnSettled(func() { fired++ }) + require.ErrorIs(t, settle(m), fail, "the broker's answer passes through") + assert.Equal(t, 1, fired, "a failed settlement counts: the message is let go") + err = nil + require.NoError(t, settle(m)) + assert.Equal(t, 1, fired, "once") + assert.Equal(t, 2, calls, "every call reaches the broker") + }) + } + m := NewMessage(context.Background(), Topic{Tenant: "acme", Table: "t"}, nil, time.Now(), nil, nil, nil) + fired := 0 + m.OnSettled(func() { fired++ }) + require.NoError(t, m.Ack()) + assert.Equal(t, 1, fired, "a message without callbacks settles at once") +} diff --git a/internal/mq/nats_fixture_test.go b/internal/mq/nats_fixture_test.go new file mode 100644 index 000000000..37e312788 --- /dev/null +++ b/internal/mq/nats_fixture_test.go @@ -0,0 +1,166 @@ +package mq + +import ( + "context" + "os" + "path/filepath" + "slices" + "testing" + "time" + + natsserver "github.com/nats-io/nats-server/v2/server" + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + "github.com/stretchr/testify/require" + + "github.com/Wave-RF/WaveHouse/internal/mq/natstest" +) + +// The external-NATS fixture: an in-process server listening on TCP whose +// accounts, users and permissions are the shipped Helm values' config.merge +// block, verbatim, and whose streams and consumers are the shipped nack +// manifests. So the tests exercise what an operator deploys, not a copy of it. + +// fixturePassword is the password every fixture user gets in place of the +// Helm values' secret reference. +func fixturePassword(user string) string { return natstest.Password(user) } + +type natsFixture struct { + server *natsserver.Server + // opts is what server was started with, for a restart on the same port + // and store. + opts *natsserver.Options + // admin is nack's stand-in: the operator's user, with full access. + admin jetstream.JetStream +} + +// newNATSFixture starts a server configured from the shipped Helm values, +// shut down by the test framework. +func newNATSFixture(t *testing.T) *natsFixture { + t.Helper() + return newNATSFixtureFrom(t, natstest.ShippedValues(), nil) +} + +// newNATSFixtureFrom starts a server configured from the Helm values at +// valuesPath, with edit's changes to its options, shut down by the test +// framework. +func newNATSFixtureFrom(t *testing.T, valuesPath string, edit func(*natsserver.Options)) *natsFixture { + t.Helper() + dir := t.TempDir() + conf, err := natstest.ServerConfig(valuesPath, dir) + require.NoError(t, err) + confPath := filepath.Join(dir, "nats.conf") + require.NoError(t, os.WriteFile(confPath, conf, 0o600)) + opts, err := natsserver.ProcessConfigFile(confPath) + require.NoError(t, err) + opts.Host, opts.Port, opts.NoSigs, opts.NoLog = "127.0.0.1", -1, true, true + if edit != nil { + edit(opts) + } + + s, err := natsserver.NewServer(opts) + require.NoError(t, err) + s.Start() + require.True(t, s.ReadyForConnections(10*time.Second), "nats server not ready") + t.Cleanup(s.Shutdown) + f := &natsFixture{server: s, opts: opts} + f.admin = f.connect(t, natstest.OperatorUser) + return f +} + +// connect opens a JetStream context as user, closed by the test framework. +// The wavehouse user gets the inbox prefix its permissions allow. +func (f *natsFixture) connect(t *testing.T, user string, opts ...nats.Option) jetstream.JetStream { + t.Helper() + opts = append([]nats.Option{nats.UserInfo(user, fixturePassword(user))}, opts...) + if user == "wavehouse" { + opts = append(opts, nats.CustomInboxPrefix(natsInboxPrefix(DefaultNATSSubjectPrefix))) + } + nc, err := nats.Connect(f.server.ClientURL(), opts...) + require.NoError(t, err) + t.Cleanup(nc.Close) + js, err := jetstream.New(nc) + require.NoError(t, err) + return js +} + +// fixtureTopology is a set of stream and consumer configs to create, in +// order: each stream, then its consumers. +type fixtureTopology struct{ *natstest.Manifests } + +// shippedFileStore is the max_file_store the NATS chart sets from the +// shipped values, in bytes. +func shippedFileStore(t *testing.T) int64 { + t.Helper() + size, err := natstest.FileStore(natstest.ShippedValues()) + require.NoError(t, err) + n, err := ParseStoreSize(size) + require.NoError(t, err) + return n +} + +// shippedTopology is the shipped manifests (N=4) at one replica, which is +// all a single server can hold. +func shippedTopology(t *testing.T) *fixtureTopology { + t.Helper() + tp := loadNATSManifests(t, natstest.ShippedManifests()) + tp.SingleReplica() + return tp +} + +// loadNATSManifests parses nack Stream and Consumer resources into the +// JetStream configs nack would create from them. +func loadNATSManifests(t *testing.T, path string) *fixtureTopology { + t.Helper() + m, err := natstest.LoadManifests(path) + require.NoError(t, err) + return &fixtureTopology{m} +} + +// stream is the named stream's config, to mutate before apply. +func (tp *fixtureTopology) stream(t *testing.T, name string) *jetstream.StreamConfig { + t.Helper() + i := slices.IndexFunc(tp.Streams, func(s jetstream.StreamConfig) bool { return s.Name == name }) + require.GreaterOrEqual(t, i, 0, "no stream %s in the fixture", name) + return &tp.Streams[i] +} + +// consumer is shard 0's durable on the named stream, to mutate before apply. +func (tp *fixtureTopology) consumer(t *testing.T, stream string) *jetstream.ConsumerConfig { + t.Helper() + i := slices.IndexFunc(tp.Consumers[stream], func(c jetstream.ConsumerConfig) bool { return c.Durable == "wh-ingest-0" }) + require.GreaterOrEqual(t, i, 0, "no wh-ingest-0 on %s", stream) + return &tp.Consumers[stream][i] +} + +// drop removes the named stream and its consumers. +func (tp *fixtureTopology) drop(name string) { + tp.Streams = slices.DeleteFunc(tp.Streams, func(s jetstream.StreamConfig) bool { return s.Name == name }) + delete(tp.Consumers, name) +} + +// apply creates tp as the operator would. +func (f *natsFixture) apply(t *testing.T, tp *fixtureTopology) { + t.Helper() + require.NoError(t, f.create(t.Context(), tp)) +} + +// reset deletes every stream, and with them their consumers. +func (f *natsFixture) reset(t *testing.T) { + t.Helper() + names := f.admin.StreamNames(t.Context()) + var all []string + for name := range names.Name() { + all = append(all, name) + } + require.NoError(t, names.Err()) + for _, name := range all { + require.NoError(t, f.admin.DeleteStream(t.Context(), name)) + } +} + +// create creates tp's streams and consumers without waiting for anything, so +// a goroutine can call it. +func (f *natsFixture) create(ctx context.Context, tp *fixtureTopology) error { + return tp.Create(ctx, f.admin) +} diff --git a/internal/mq/nats_manifests.go b/internal/mq/nats_manifests.go new file mode 100644 index 000000000..8d1b0600c --- /dev/null +++ b/internal/mq/nats_manifests.go @@ -0,0 +1,503 @@ +package mq + +import ( + "errors" + "fmt" + "io" + "regexp" + "strconv" + "strings" + "time" + + "gopkg.in/yaml.v3" +) + +// NATSManifestOptions sizes the nack CRs WriteNATSManifests renders. Zero +// fields take the defaults below, which are starting points to tune, not +// recommendations for any particular load. +type NATSManifestOptions struct { + Topology NATSTopology + // Replicas is every stream's replica count (default 3). + Replicas int + // FileStore is each server's JetStream file store, max_file_store + // (default 100 GiB, the shipped Helm values' PVC; the NATS chart sets + // max_file_store to fileStore.maxSize, else the PVC's size). A server + // reserves every stream replica's max_bytes it holds against it and + // refuses a stream that does not fit (JetStream error 10047), so the + // streams' max_bytes together must fit in it. + FileStore int64 + // PartitionMaxBytes caps each ingest partition: the backlog of unwritten + // rows its tenants share (default 15% of FileStore). It does not follow + // the partition count, so lowering N (a larger partition while the + // removed ones drain) never needs more store than before; at the defaults + // four partitions, the history and the DLQ reserve three quarters of + // FileStore, the quarter left being headroom for the Raft logs, the lease + // bucket and the store's own overhead. + PartitionMaxBytes int64 + // MaxMsgsPerSubject caps one topic's backlog in a partition (default + // 1,000,000). + MaxMsgsPerSubject int64 + // HistoryMaxAge is how long SSE can replay (default 15m, the seed's + // stream.gap_window_minutes); at least the longest tenant gap window. + HistoryMaxAge time.Duration + // HistoryMaxBytes caps the history (default a tenth of FileStore). + HistoryMaxBytes int64 + // DLQMaxBytes caps the dead-letter stream (default a twentieth of + // FileStore). + DLQMaxBytes int64 + // DLQMaxMsgsPerSubject caps one topic's parked rows (default 100,000). + DLQMaxMsgsPerSubject int64 + // PinnedTTL is how long a shard's pin outlives its holder's last pull + // (default 10s): at least minPinnedTTL, and under the membership lease. + PinnedTTL time.Duration +} + +// DefaultNATSFileStore is the file store the generated manifests are sized +// for by default: the shipped Helm values' JetStream PVC. +const DefaultNATSFileStore int64 = 100 << 30 + +// reserved is what the streams reserve on each server holding a replica of +// every one: their max_bytes together. +func (o NATSManifestOptions) reserved() int64 { + return int64(o.Topology.Partitions)*o.PartitionMaxBytes + o.HistoryMaxBytes + o.DLQMaxBytes +} + +// fits reports streams that together reserve more than the file store, which +// the server would refuse one of. +func (o NATSManifestOptions) fits() error { + if o.FileStore < 0 || o.PartitionMaxBytes <= 0 || o.HistoryMaxBytes <= 0 || o.DLQMaxBytes <= 0 { + return errors.New("the file store and every stream's max bytes must be positive") + } + if r := o.reserved(); r > o.FileStore { + return fmt.Errorf("the streams reserve %s (%d partitions of %s, history %s, dlq %s), more than the %s file store, and JetStream refuses a stream that does not fit: raise --file-store with the servers' volumes, or lower --partition-max-bytes", + FormatStoreSize(r), o.Topology.Partitions, FormatStoreSize(o.PartitionMaxBytes), FormatStoreSize(o.HistoryMaxBytes), FormatStoreSize(o.DLQMaxBytes), FormatStoreSize(o.FileStore)) + } + return nil +} + +// storeUnits are the Kubernetes quantity suffixes ParseStoreSize takes, as +// the NATS config parser reads them too. +var storeUnits = []struct { + suffix string + n int64 +}{ + {"Ti", 1 << 40}, + {"Gi", 1 << 30}, + {"Mi", 1 << 20}, + {"Ki", 1 << 10}, + {"T", 1e12}, + {"G", 1e9}, + {"M", 1e6}, + {"k", 1e3}, +} + +// ParseStoreSize reads a byte count written as a Kubernetes quantity: a +// whole number with an optional k, M, G, T or Ki, Mi, Gi, Ti suffix (100Gi). +func ParseStoreSize(s string) (int64, error) { + num, unit := s, int64(1) + for _, u := range storeUnits { + if strings.HasSuffix(s, u.suffix) { + num, unit = strings.TrimSuffix(s, u.suffix), u.n + break + } + } + n, err := strconv.ParseInt(num, 10, 64) + if err != nil || n < 0 || n > (1<<62)/unit { + return 0, fmt.Errorf("size %q must be a whole number of bytes, optionally with a k, M, G, T, Ki, Mi, Gi or Ti suffix", s) + } + return n * unit, nil +} + +// FormatStoreSize writes n in the largest binary unit that divides it. +func FormatStoreSize(n int64) string { + for _, u := range storeUnits[:4] { + if n != 0 && n%u.n == 0 { + return strconv.FormatInt(n/u.n, 10) + u.suffix + } + } + return strconv.FormatInt(n, 10) +} + +func (o NATSManifestOptions) withDefaults() NATSManifestOptions { + o.Topology = o.Topology.withDefaults() + if o.Replicas == 0 { + o.Replicas = 3 + } + if o.MaxMsgsPerSubject == 0 { + o.MaxMsgsPerSubject = 1_000_000 + } + if o.HistoryMaxAge == 0 { + o.HistoryMaxAge = 15 * time.Minute + } + if o.FileStore == 0 { + o.FileStore = DefaultNATSFileStore + } + if o.HistoryMaxBytes == 0 { + o.HistoryMaxBytes = o.FileStore / 10 + } + if o.DLQMaxBytes == 0 { + o.DLQMaxBytes = o.FileStore / 20 + } + if o.PartitionMaxBytes == 0 { + o.PartitionMaxBytes = o.FileStore / 100 * 15 + } + if o.DLQMaxMsgsPerSubject == 0 { + o.DLQMaxMsgsPerSubject = 100_000 + } + if o.PinnedTTL == 0 { + o.PinnedTTL = 10 * time.Second + } + return o +} + +// The nack (jetstream.nats.io/v1beta2) custom resources, only the fields the +// topology sets. nack's own defaults are not JetStream's (storage memory, +// ackWait 1ns), so every field the verifier checks is written out. +type nackObject struct { + APIVersion string `yaml:"apiVersion"` + Kind string `yaml:"kind"` + Metadata nackMetadata `yaml:"metadata"` + Spec any `yaml:"spec"` +} + +type nackMetadata struct { + Name string `yaml:"name"` +} + +type nackStream struct { + Name string `yaml:"name"` + Subjects []string `yaml:"subjects,omitempty"` + Retention string `yaml:"retention"` + Discard string `yaml:"discard"` + DiscardPerSubject bool `yaml:"discardPerSubject,omitempty"` + MaxBytes int64 `yaml:"maxBytes"` + MaxAge string `yaml:"maxAge,omitempty"` + MaxMsgsPerSubject int64 `yaml:"maxMsgsPerSubject,omitempty"` + Storage string `yaml:"storage"` + Replicas int `yaml:"replicas"` + DuplicateWindow string `yaml:"duplicateWindow,omitempty"` + DenyPurge bool `yaml:"denyPurge,omitempty"` + DenyDelete bool `yaml:"denyDelete,omitempty"` + Metadata map[string]string `yaml:"metadata,omitempty"` + PreventDelete bool `yaml:"preventDelete,omitempty"` + Republish *nackRepublish `yaml:"republish,omitempty"` +} + +// nackRepublish is nack's stream republish: every row the stream stores is +// also published, by the server, to the destination. +type nackRepublish struct { + Source string `yaml:"source"` + Destination string `yaml:"destination"` +} + +// nackKeyValue is nack's KeyValue spec. nack creates the bucket the way +// `nats kv add` does, which sets allow_direct. +type nackKeyValue struct { + Bucket string `yaml:"bucket"` + History int `yaml:"history"` + Storage string `yaml:"storage"` + Replicas int `yaml:"replicas"` +} + +type nackConsumer struct { + StreamName string `yaml:"streamName"` + DurableName string `yaml:"durableName"` + DeliverPolicy string `yaml:"deliverPolicy"` + AckPolicy string `yaml:"ackPolicy"` + AckWait string `yaml:"ackWait"` + MaxDeliver int `yaml:"maxDeliver"` + MaxAckPending int `yaml:"maxAckPending"` + FilterSubject string `yaml:"filterSubject,omitempty"` + PriorityPolicy string `yaml:"priorityPolicy,omitempty"` + PriorityGroups []string `yaml:"priorityGroups,omitempty"` + PinnedTTL string `yaml:"pinnedTtl,omitempty"` + PreventDelete bool `yaml:"preventDelete,omitempty"` +} + +// nackDuration renders d in the largest whole unit, as an operator would +// write it ("2h", not "2h0m0s"); nack parses it with time.ParseDuration. +func nackDuration(d time.Duration) string { + switch { + case d%time.Hour == 0: + return strconv.FormatInt(int64(d/time.Hour), 10) + "h" + case d%time.Minute == 0: + return strconv.FormatInt(int64(d/time.Minute), 10) + "m" + default: + return d.String() + } +} + +// natsManifestObjects is the topology as nack CRs: each partition and its +// shard durables, then the history, the dead-letter stream and the lease +// bucket. +func natsManifestObjects(o NATSManifestOptions) []nackObject { + o = o.withDefaults() + t := o.Topology + lower := func(kind string) string { return t.Prefix + "-" + kind } + var objs []nackObject + for p := range t.Partitions { + stream := t.streamName("INGEST_" + strconv.Itoa(p)) + name := lower("ingest-" + strconv.Itoa(p)) + rp := natsRepublish(t.Prefix, p) + objs = append(objs, nackObject{ + APIVersion: "jetstream.nats.io/v1beta2", Kind: "Stream", Metadata: nackMetadata{Name: name}, + Spec: nackStream{ + Name: stream, + Subjects: []string{natsIngestPartition(t.Prefix, p)}, + Retention: "workqueue", + Discard: "new", + DiscardPerSubject: true, + MaxBytes: o.PartitionMaxBytes, + MaxMsgsPerSubject: o.MaxMsgsPerSubject, + Storage: "file", + Replicas: o.Replicas, + DuplicateWindow: nackDuration(max(2*time.Minute, t.minDuplicateWindow(), t.dedupeDuplicateWindow())), + DenyPurge: true, + DenyDelete: true, + Metadata: map[string]string{ + "wavehouse.dev/partition": strconv.Itoa(p), + "wavehouse.dev/partitions": strconv.Itoa(t.Partitions), + "wavehouse.dev/shards": strconv.Itoa(t.Shards), + }, + PreventDelete: true, + Republish: &nackRepublish{Source: rp.Source, Destination: rp.Destination}, + }, + }) + for sh := range t.Shards { + objs = append(objs, nackObject{ + APIVersion: "jetstream.nats.io/v1beta2", Kind: "Consumer", + Metadata: nackMetadata{Name: name + "-" + strconv.Itoa(sh)}, + Spec: nackConsumer{ + StreamName: stream, + DurableName: natsShardDurable(t.IngestConsumer, sh), + DeliverPolicy: "all", + AckPolicy: "explicit", + AckWait: nackDuration(t.AckWait), + MaxDeliver: -1, + MaxAckPending: t.MaxAckPending, + FilterSubject: natsShardFilter(t.Prefix, p, sh), + PriorityPolicy: "pinned_client", + PriorityGroups: []string{natsPriorityGroup}, + PinnedTTL: nackDuration(o.PinnedTTL), + PreventDelete: true, + }, + }) + } + } + objs = append(objs, nackObject{ + APIVersion: "jetstream.nats.io/v1beta2", Kind: "Stream", Metadata: nackMetadata{Name: lower("history")}, + Spec: nackStream{ + Name: t.HistoryStream, + Subjects: []string{natsHistorySubjects(t.Prefix)}, + Retention: "limits", + Discard: "old", + MaxBytes: o.HistoryMaxBytes, + MaxAge: nackDuration(o.HistoryMaxAge), + Storage: "file", + Replicas: o.Replicas, + }, + }, nackObject{ + APIVersion: "jetstream.nats.io/v1beta2", Kind: "Stream", Metadata: nackMetadata{Name: lower("dlq")}, + Spec: nackStream{ + Name: t.streamName("DLQ"), + Subjects: []string{natsDLQSubjects(t.Prefix)}, + Retention: "limits", + Discard: "old", + MaxBytes: o.DLQMaxBytes, + MaxMsgsPerSubject: o.DLQMaxMsgsPerSubject, + Storage: "file", + Replicas: o.Replicas, + }, + }, nackObject{ + APIVersion: "jetstream.nats.io/v1beta2", Kind: "KeyValue", Metadata: nackMetadata{Name: lower("coord")}, + // No ttl: a lease expires on its candidates' clocks, not the server's. + Spec: nackKeyValue{Bucket: t.coordBucket(), History: 1, Storage: "file", Replicas: o.Replicas}, + }) + return objs +} + +// manifestFlags is the `wavehouse mq manifests` command line that renders o: +// the flags that always shape it, then any other set to other than its +// default. +func manifestFlags(o NATSManifestOptions) string { + t := o.Topology + flags := fmt.Sprintf("--partitions %d --shards %d --prefix %s --replicas %d --file-store %s", t.Partitions, t.Shards, t.Prefix, o.Replicas, FormatStoreSize(o.FileStore)) + if o.PartitionMaxBytes != o.FileStore/100*15 { + flags += " --partition-max-bytes " + FormatStoreSize(o.PartitionMaxBytes) + } + if t.IngestConsumer != DefaultNATSIngestConsumer { + flags += " --ingest-consumer " + t.IngestConsumer + } + if t.HistoryStream != t.streamName("HISTORY") { + flags += " --history-stream " + t.HistoryStream + } + if t.PublishTimeout != DefaultNATSPublishTimeout { + flags += " --publish-timeout " + t.PublishTimeout.String() + } + if t.CoordBucket != "" && t.CoordBucket != DefaultNATSCoordBucket(t.Prefix) { + flags += " --coord-bucket " + t.CoordBucket + } + return flags +} + +// WriteNATSManifests writes the nack CRs for the topology WaveHouse checks at +// boot, as one multi-document YAML stream. +func WriteNATSManifests(w io.Writer, o NATSManifestOptions) error { + o = o.withDefaults() + t := o.Topology + if err := t.validate(); err != nil { + return err + } + if o.PinnedTTL < minPinnedTTL { + return fmt.Errorf("pinned ttl %s must be at least %s, twice the time between pulls", o.PinnedTTL, minPinnedTTL) + } + if err := o.fits(); err != nil { + return err + } + if _, err := fmt.Fprintf(w, `# WaveHouse's JetStream topology as nack (jetstream.nats.io/v1beta2) resources: +# %d ingest partition(s) with work-queue retention, each split into %d shards +# with a pinned-client durable per shard (%s-), and republishing every +# row to the %s history stream; the dead-letter stream; and the %s KV +# bucket that coord.backend=nats holds its leases in. +# Generated by: wavehouse mq manifests %s +# Sized for a JetStream file store (max_file_store) of %s on every server: +# the streams' maxBytes reserve %s of it, and JetStream refuses a stream that +# does not fit. The NATS Helm chart sets max_file_store to +# config.jetstream.fileStore.maxSize, else to the JetStream PVC's size. +# Sizes (maxBytes, maxAge, maxMsgsPerSubject) are starting points to tune. +# Set the history's maxAge (%s) to at least the longest +# stream.gap_window_minutes among the tenants served. +# WaveHouse publishes nothing until all of it exists, so apply order is free: +# a partition keeps every row until the ingest worker acks it, even while its +# durable is missing, and the history is best effort (a row republished while +# it is unavailable reaches ClickHouse but not SSE replay). +`, t.Partitions, t.Shards, t.IngestConsumer, t.HistoryStream, t.coordBucket(), manifestFlags(o), FormatStoreSize(o.FileStore), FormatStoreSize(o.reserved()), nackDuration(o.HistoryMaxAge)); err != nil { + return err + } + enc := yaml.NewEncoder(w) + enc.SetIndent(2) + for _, obj := range natsManifestObjects(o) { + if err := enc.Encode(obj); err != nil { + return err + } + } + return enc.Close() +} + +// natsPermissionSet is a NATS user's permission set, as the server config +// writes it. +type natsPermissionSet struct { + PublishAllow []string + PublishDeny []string + SubscribeAllow []string +} + +// natsInboxPrefix is the reply-subject prefix WaveHouse's connection uses, so +// its subscribe permission can be narrowed to its own replies. +func natsInboxPrefix(prefix string) string { return "_INBOX_" + prefix } + +// natsAPIPrefix is the JetStream API prefix a client of domain uses: +// $JS.API., or $JS..API. under a JetStream domain. +func natsAPIPrefix(domain string) string { + if domain == "" { + return "$JS.API." + } + return "$JS." + domain + ".API." +} + +// natsPermissions is exactly what WaveHouse's NATS user needs under t: +// publish to its ingest and dead-letter subjects (never the history's), read +// stream and consumer state and list consumer names, pull from, unpin and +// reset each shard durable and ack what it delivers, create, pull from and +// delete the auto-expiring consumers it reads the history through, and read +// and write the lease keys in the coord bucket (a KV write is a publish to +// the key's subject; a read, a direct get). It cannot create, change, purge +// or delete a stream, nor create a durable on a partition. +// +// Under a JetStream domain the client sends every API request, and each KV +// write, under $JS..API. A server in the domain maps that to the +// plain subject before it checks permissions, and a server outside it (a +// leafnode) checks it as sent, so each is allowed and denied in both forms. +func natsPermissions(t NATSTopology, domain string) natsPermissionSet { + t = t.withDefaults() + h, kv := t.HistoryStream, t.coordBucket() + api := func(subjects ...string) []string { + var out []string + for _, s := range subjects { + out = append(out, "$JS.API."+s) + if domain != "" { + out = append(out, natsAPIPrefix(domain)+s) + } + } + return out + } + allow := []string{t.Prefix + ".ingest.>", t.Prefix + ".dlq.>"} + allow = append(allow, api("INFO", "STREAM.NAMES", "STREAM.INFO.*", "CONSUMER.INFO.*.*", "CONSUMER.NAMES.*")...) + // One entry per shard durable: a NATS wildcard is a whole token, so + // wh-ingest-* would name no durable at all. The stream is a wildcard, so + // a partition a lower N left behind is covered too. + for _, verb := range []string{"MSG.NEXT", "UNPIN", "RESET"} { + for s := range t.Shards { + allow = append(allow, api("CONSUMER."+verb+".*."+natsShardDurable(t.IngestConsumer, s))...) + } + } + // An ack goes to the reply subject the server delivered the row with: + // $JS.ACK...… (v1, the server's default), or + // $JS.ACK.....… (v2, the + // js_ack_fc_v2 feature flag). The history consumers ack nothing. + for s := range t.Shards { + allow = append(allow, "$JS.ACK.*."+natsShardDurable(t.IngestConsumer, s)+".>") + } + for s := range t.Shards { + allow = append(allow, "$JS.ACK.*.*.*."+natsShardDurable(t.IngestConsumer, s)+".>") + } + allow = append(allow, api("CONSUMER.CREATE."+h+".>", "CONSUMER.MSG.NEXT."+h+".>", "CONSUMER.DELETE."+h+".>")...) + lease := "$KV." + kv + "." + leaseKeyPrefix + ">" + allow = append(allow, lease) + if domain != "" { + allow = append(allow, natsAPIPrefix(domain)+lease) + } + allow = append(allow, api("DIRECT.GET.KV_"+kv+"."+lease)...) + return natsPermissionSet{ + PublishAllow: allow, + // Only the partitions' republish writes the history. + PublishDeny: append([]string{natsHistorySubjects(t.Prefix)}, + api("STREAM.CREATE.>", "STREAM.UPDATE.>", "STREAM.DELETE.>", "STREAM.PURGE.>", "CONSUMER.DURABLE.CREATE.>")...), + SubscribeAllow: []string{natsInboxPrefix(t.Prefix) + ".>"}, + } +} + +// WriteNATSPermissions writes the wavehouse user's permissions under t, for +// a connection to JetStream domain jsDomain ("" for none, mq.nats.js_domain), +// as the permissions block of a NATS Helm values file's user entry. +func WriteNATSPermissions(w io.Writer, t NATSTopology, jsDomain string) error { + t = t.withDefaults() + if err := t.validate(); err != nil { + return err + } + if jsDomain != "" && !natsDomainName.MatchString(jsDomain) { + return fmt.Errorf("js domain %q must be one token of [a-zA-Z0-9_-]", jsDomain) + } + p := natsPermissions(t, jsDomain) + doc := map[string]any{"permissions": map[string]any{ + "publish": map[string][]string{"allow": p.PublishAllow, "deny": p.PublishDeny}, + "subscribe": map[string][]string{"allow": p.SubscribeAllow}, + }} + domain := "" + if jsDomain != "" { + domain = ", JetStream domain " + jsDomain + } + if _, err := fmt.Fprintf(w, "# The wavehouse user's permissions for prefix %s, %d shard(s), history stream %s\n# and lease bucket %s%s. Generated by: wavehouse mq permissions\n", t.Prefix, t.Shards, t.HistoryStream, t.coordBucket(), domain); err != nil { + return err + } + enc := yaml.NewEncoder(w) + enc.SetIndent(2) + if err := enc.Encode(doc); err != nil { + return err + } + return enc.Close() +} + +// natsDomainName is the grammar WriteNATSPermissions takes for a JetStream +// domain: one subject token. +var natsDomainName = regexp.MustCompile(`^[a-zA-Z0-9_-]+$`) diff --git a/internal/mq/nats_manifests_test.go b/internal/mq/nats_manifests_test.go new file mode 100644 index 000000000..689a8064a --- /dev/null +++ b/internal/mq/nats_manifests_test.go @@ -0,0 +1,206 @@ +package mq + +import ( + "bytes" + "fmt" + "os" + "path/filepath" + "strconv" + "strings" + "testing" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/Wave-RF/WaveHouse/internal/mq/natstest" +) + +// The shipped manifests reserve at most three quarters of the file store the +// NATS chart gives each server from the shipped values, the rest being +// headroom for what else the store holds. +func TestShippedManifests_FitTheShippedFileStore(t *testing.T) { + t.Parallel() + m, err := natstest.LoadManifests(natstest.ShippedManifests()) + require.NoError(t, err) + var reserved int64 + for _, s := range m.Streams { + require.Positive(t, s.MaxBytes, "stream %s reserves nothing", s.Name) + reserved += s.MaxBytes + } + store := shippedFileStore(t) + assert.LessOrEqual(t, reserved, store/4*3, "the shipped streams reserve %s of a %s file store", FormatStoreSize(reserved), FormatStoreSize(store)) +} + +// Streams that together reserve more than the file store are refused, as +// JetStream would refuse one of them; partitions are sized independently of +// their count, so lowering N never needs more store. +func TestWriteNATSManifests_RefusesStreamsOverTheFileStore(t *testing.T) { + t.Parallel() + var b bytes.Buffer + require.NoError(t, WriteNATSManifests(&b, NATSManifestOptions{Topology: NATSTopology{Partitions: 5}})) + err := WriteNATSManifests(&b, NATSManifestOptions{Topology: NATSTopology{Partitions: 6}}) + require.ErrorContains(t, err, "reserve 105Gi (6 partitions of 15Gi, history 10Gi, dlq 5Gi), more than the 100Gi file store") + require.NoError(t, WriteNATSManifests(&b, NATSManifestOptions{Topology: NATSTopology{Partitions: 6}, PartitionMaxBytes: 10 << 30})) + err = WriteNATSManifests(&b, NATSManifestOptions{Topology: NATSTopology{Partitions: 1}, PartitionMaxBytes: 86 << 30}) + require.ErrorContains(t, err, "more than the 100Gi file store") + + one, two := NATSManifestOptions{Topology: NATSTopology{Partitions: 1}}.withDefaults(), NATSManifestOptions{Topology: NATSTopology{Partitions: 2}}.withDefaults() + assert.Equal(t, one.PartitionMaxBytes, two.PartitionMaxBytes) +} + +// The fixture's server reserves streams against the file store the chart +// would give it, so a topology that does not fit the shipped volume does not +// fit the tests either. +func TestNATSFixture_ReservesAgainstTheShippedFileStore(t *testing.T) { + t.Parallel() + f := newNATSFixture(t) + _, err := f.admin.CreateStream(t.Context(), jetstream.StreamConfig{ + Name: "TOO_BIG", Subjects: []string{"too.big.>"}, Storage: jetstream.FileStorage, MaxBytes: shippedFileStore(t) + 1, + }) + var apiErr *jetstream.APIError + require.ErrorAs(t, err, &apiErr) + assert.Equal(t, jetstream.ErrorCode(10047), apiErr.ErrorCode, "insufficient storage resources: %v", err) +} + +func TestStoreSize(t *testing.T) { + t.Parallel() + for in, want := range map[string]int64{"100Gi": 100 << 30, "15Gi": 15 << 30, "1024": 1024, "2Ti": 2 << 40, "5G": 5e9, "64Mi": 64 << 20, "3k": 3000} { + got, err := ParseStoreSize(in) + require.NoError(t, err, in) + assert.Equal(t, want, got, in) + } + for _, in := range []string{"", "Gi", "1.5Gi", "-1", "10GB", "10 Gi", "9999999999Ti"} { + _, err := ParseStoreSize(in) + assert.Error(t, err, in) + } + for in, want := range map[int64]string{100 << 30: "100Gi", 75 << 30: "75Gi", 1536 << 20: "1536Mi", 1000: "1000", 2 << 40: "2Ti"} { + assert.Equal(t, want, FormatStoreSize(in)) + } +} + +// Under a JetStream domain every API request and KV write is allowed and +// denied both as sent ($JS..API.…, what a leafnode checks) and as a +// server in the domain maps it ($JS.API.…). +func TestNATSPermissions_JSDomain(t *testing.T) { + t.Parallel() + plain := natsPermissions(NATSTopology{Shards: 2}, "") + hub := natsPermissions(NATSTopology{Shards: 2}, "hub") + for _, s := range plain.PublishAllow { + assert.Contains(t, hub.PublishAllow, s) + if rest, ok := strings.CutPrefix(s, "$JS.API."); ok { + assert.Contains(t, hub.PublishAllow, "$JS.hub.API."+rest) + } + } + for _, s := range plain.PublishDeny { + assert.Contains(t, hub.PublishDeny, s) + if rest, ok := strings.CutPrefix(s, "$JS.API."); ok { + assert.Contains(t, hub.PublishDeny, "$JS.hub.API."+rest) + } + } + assert.Contains(t, hub.PublishAllow, "$JS.hub.API.$KV.wh_coord.lease.>") + assert.NotContains(t, plain.PublishAllow, "$JS.ACK.>") + var b bytes.Buffer + require.Error(t, WriteNATSPermissions(&b, NATSTopology{}, "a.b")) +} + +// The validator's case: the shipped permissions, which name 8 shards' +// durables, against a topology of 16. The verifier probes each durable as +// the connecting user and names every shard the user may not pull. +func TestVerifyNATSTopology_ProbesTheShardPermissions(t *testing.T) { + t.Parallel() + f := newNATSFixture(t) + var buf bytes.Buffer + require.NoError(t, WriteNATSManifests(&buf, NATSManifestOptions{Topology: NATSTopology{Partitions: 4, Shards: 16}, Replicas: 1})) + path := filepath.Join(t.TempDir(), "manifests.yaml") + require.NoError(t, os.WriteFile(path, buf.Bytes(), 0o600)) + f.apply(t, loadNATSManifests(t, path)) + + perms := newNATSPermissionWatch() + js := f.connect(t, natstest.WaveHouseUser, nats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, err error) { perms.record(err) })) + findings, err := verifyNATSTopology(t.Context(), js, NATSTopology{Partitions: 4, Shards: 16}, perms) + require.NoError(t, err) + var denied []string + for _, got := range findings { + if got.Field == "permissions" { + require.Equal(t, FindingRequired, got.Severity) + denied = append(denied, strings.TrimPrefix(got.Object, "consumer ")) + assert.Contains(t, got.Problem, "wavehouse mq permissions --shards 16") + } + } + var want []string + for p := range 4 { + for s := 8; s < 16; s++ { + want = append(want, "WH_INGEST_"+strconv.Itoa(p)+"/"+natsShardDurable(DefaultNATSIngestConsumer, s)) + } + } + assert.ElementsMatch(t, want, denied) +} + +// The probe changes nothing it touches: a durable the user may pull keeps +// its pending rows and delivers them to the next pull. +func TestVerifyNATSTopology_ProbeLeavesTheDurablesAlone(t *testing.T) { + t.Parallel() + f := newNATSFixture(t) + f.apply(t, shippedTopology(t)) + _, err := f.admin.Publish(t.Context(), "wh.ingest.0.0.acme.t", []byte("row")) + require.NoError(t, err) + perms := newNATSPermissionWatch() + js := f.connect(t, natstest.WaveHouseUser, nats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, err error) { perms.record(err) })) + findings, err := verifyNATSTopology(t.Context(), js, shippedSpec, perms) + require.NoError(t, err) + assert.True(t, replicaWarnings(findings), "findings: %v", findings) + + c, err := f.admin.Consumer(t.Context(), "WH_INGEST_0", "wh-ingest-0") + require.NoError(t, err) + info := c.CachedInfo() + assert.Equal(t, uint64(1), info.NumPending) + assert.Zero(t, info.NumAckPending) + assert.Zero(t, info.Delivered.Stream) + assert.Empty(t, info.PriorityGroups[0].PinnedClientID, "a probe pinned no one") +} + +// A refusal seen before a probe says nothing about it: permissions change, +// and a durable the user may now pull is not reported for an old refusal. +func TestVerifyNATSTopology_ProbeIgnoresAnEarlierRefusal(t *testing.T) { + t.Parallel() + f := newNATSFixture(t) + f.apply(t, shippedTopology(t)) + perms := newNATSPermissionWatch() + _, _, ok := perms.record(fmt.Errorf(`%w: Permissions Violation for Publish to "$JS.API.CONSUMER.MSG.NEXT.WH_INGEST_0.wh-ingest-0"`, nats.ErrPermissionViolation)) + require.True(t, ok) + js := f.connect(t, natstest.WaveHouseUser, nats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, err error) { perms.record(err) })) + findings, err := verifyNATSTopology(t.Context(), js, shippedSpec, perms) + require.NoError(t, err) + assert.True(t, replicaWarnings(findings), "findings: %v", findings) +} + +// The watch keeps only JetStream API subjects, in either domain form: an +// ack's subject is new with every row, so keeping those would grow without +// bound. +func TestNATSPermissionWatch_KeepsOnlyAPISubjects(t *testing.T) { + t.Parallel() + w := newNATSPermissionWatch() + refuse := func(subject string) bool { + _, _, ok := w.record(fmt.Errorf(`%w: Permissions Violation for Publish to "%s"`, nats.ErrPermissionViolation, subject)) + return ok + } + assert.False(t, refuse("$JS.ACK.WH_INGEST_0.wh-ingest-0.1.2.3.4.5")) + assert.False(t, refuse("wh.hist.acme.t")) + assert.True(t, refuse("$JS.API.CONSUMER.MSG.NEXT.WH_INGEST_0.wh-ingest-9")) + assert.Len(t, w.list(), 1) + assert.True(t, w.deniedSince("$JS.hub.API.CONSUMER.MSG.NEXT.WH_INGEST_0.wh-ingest-9", 0), "a server in the domain reports the plain form") + assert.True(t, isConsumerAPI("$JS.hub.API.CONSUMER.UNPIN.S.d")) + assert.False(t, isConsumerAPI("$JS.hub.ACK.S.d")) +} + +// The remedy a permission finding names regenerates the permissions for the +// settings in force, not the defaults. +func TestPermissionFlags(t *testing.T) { + t.Parallel() + assert.Equal(t, "--shards 16", permissionFlags(NATSTopology{Shards: 16}.withDefaults(), "")) + assert.Equal(t, "--shards 4 --prefix acme --ingest-consumer acme-in --history-stream H --coord-bucket leases --js-domain hub", + permissionFlags(NATSTopology{Shards: 4, Prefix: "acme", IngestConsumer: "acme-in", HistoryStream: "H", CoordBucket: "leases"}.withDefaults(), "hub")) + assert.Equal(t, "--shards 4 --prefix acme", permissionFlags(NATSTopology{Shards: 4, Prefix: "acme", CoordBucket: "acme_coord"}.withDefaults(), "")) +} diff --git a/internal/mq/nats_permprobe.go b/internal/mq/nats_permprobe.go new file mode 100644 index 000000000..bdfe0ee53 --- /dev/null +++ b/internal/mq/nats_permprobe.go @@ -0,0 +1,249 @@ +package mq + +import ( + "context" + "errors" + "fmt" + "maps" + "regexp" + "slices" + "strconv" + "strings" + "sync" + "time" + + "github.com/nats-io/nats.go" +) + +// natsPermissionWatch holds the subjects the server refused a connection's +// publish to. The server reports a refused publish only as an asynchronous +// -ERR naming the subject, which nats.go hands to the connection's error +// handler and to no request: a refused request just goes unanswered. +type natsPermissionWatch struct { + mu sync.Mutex + // denied is each refused subject's latest refusal, numbered by seq. + denied map[string]uint64 + seq uint64 + // changed is closed, and replaced, on every refusal. + changed chan struct{} +} + +func newNATSPermissionWatch() *natsPermissionWatch { + return &natsPermissionWatch{denied: map[string]uint64{}, changed: make(chan struct{})} +} + +// mark is the latest refusal's number, for deniedSince. +func (w *natsPermissionWatch) mark() uint64 { + w.mu.Lock() + defer w.mu.Unlock() + return w.seq +} + +// changes closes at the next refusal. +func (w *natsPermissionWatch) changes() <-chan struct{} { + w.mu.Lock() + defer w.mu.Unlock() + return w.changed +} + +// publishViolation matches the server's report of a refused publish. +var publishViolation = regexp.MustCompile(`(?i)permissions violation for publish to "([^"]+)"`) + +// record notes the subject of err when it reports a refused publish to a +// JetStream API subject, and returns it, and whether it is the first refusal +// of that subject. Only API subjects are kept: they are bounded by the +// topology, where an ack's subject is new with every row. +func (w *natsPermissionWatch) record(err error) (subject string, first, ok bool) { + if err == nil || !errors.Is(err, nats.ErrPermissionViolation) { + return "", false, false + } + m := publishViolation.FindStringSubmatch(err.Error()) + if m == nil || plainAPISubject(m[1]) == "" { + return "", false, false + } + w.mu.Lock() + defer w.mu.Unlock() + _, seen := w.denied[m[1]] + w.seq++ + w.denied[m[1]] = w.seq + close(w.changed) + w.changed = make(chan struct{}) + return m[1], !seen, true +} + +// deniedSince reports whether the server refused a publish to subject after +// the refusal numbered since: permissions change, so an older refusal says +// nothing about a new request. A server in the subject's JetStream domain +// maps it to the plain $JS.API form before it checks, and reports that one. +func (w *natsPermissionWatch) deniedSince(subject string, since uint64) bool { + w.mu.Lock() + defer w.mu.Unlock() + return w.denied[subject] > since || w.denied[plainAPISubject(subject)] > since +} + +// list is every subject the server refused, sorted. +func (w *natsPermissionWatch) list() []string { + w.mu.Lock() + defer w.mu.Unlock() + return slices.Sorted(maps.Keys(w.denied)) +} + +// plainAPISubject is a JetStream API subject in its plain form, +// $JS.API.…, whether it came as that or as $JS..API.…; "" for any +// other subject. +func plainAPISubject(subject string) string { + rest, ok := strings.CutPrefix(subject, "$JS.") + if !ok { + return "" + } + if !strings.HasPrefix(rest, "API.") { + _, rest, _ = strings.Cut(rest, ".") + if !strings.HasPrefix(rest, "API.") { + return "" + } + } + return "$JS." + rest +} + +// isConsumerAPI reports whether subject is a JetStream consumer API request +// under any domain's prefix: $JS.API.CONSUMER.… or $JS..API.CONSUMER.…. +func isConsumerAPI(subject string) bool { + return strings.HasPrefix(plainAPISubject(subject), "$JS.API.CONSUMER.") +} + +// natsProbeWait bounds one permission probe. The server answers an allowed +// probe at once and reports a refused one at once, so it only has to cover a +// slow round trip. +const natsProbeWait = 2 * time.Second + +// natsProbeParallel bounds the probes in flight. +const natsProbeParallel = 64 + +// natsPermissionProbes are the shard durable requests a probe makes, each one +// the server refuses on its merits and acts on in no other way: a pull whose +// heartbeat is more than half its expiry, and an unpin naming no priority +// group. RESET is not probed: an empty or valid one resets the durable. +var natsPermissionProbes = []struct { + verb string + payload []byte +}{ + {"MSG.NEXT", []byte(`{"expires":1000000000,"heartbeat":1000000000}`)}, + {"UNPIN", []byte(`{}`)}, +} + +// probePermissions adds a required finding for each shard durable the +// connecting user may not pull from or unpin. A user's permissions name every +// shard's durable, so a set generated for fewer shards than the topology has +// leaves the others' rows unpulled while their publishes still succeed. It +// needs the connection's refused publishes in v.perms; a probe that got +// neither an answer nor a refusal is an error, a check that could not run. +func (v *topologyVerifier) probePermissions(ctx context.Context) error { + if v.perms == nil || len(v.probe) == 0 { + return nil + } + opts := v.js.Options() + api := natsAPIPrefix(opts.Domain) + if opts.Domain == "" && opts.APIPrefix != "" { + api = strings.TrimSuffix(opts.APIPrefix, ".") + "." + } + nc := v.js.Conn() + type result struct { + unit natsUnit + subject string + denied bool + err error + } + var ( + wg sync.WaitGroup + results = make(chan result, len(v.probe)*len(natsPermissionProbes)) + slots = make(chan struct{}, natsProbeParallel) + ) + for _, u := range v.probe { + for _, p := range natsPermissionProbes { + subject := api + "CONSUMER." + p.verb + "." + u.stream + "." + u.durable + wg.Go(func() { + slots <- struct{}{} + defer func() { <-slots }() + denied, err := v.probeOne(ctx, nc, subject, p.payload) + if denied || err != nil { + results <- result{unit: u, subject: subject, denied: denied, err: err} + } + }) + } + } + wg.Wait() + close(results) + sorted := slices.SortedFunc(func(yield func(result) bool) { + for r := range results { + if !yield(r) { + return + } + } + }, func(a, b result) int { return strings.Compare(a.subject, b.subject) }) + hint := "wavehouse mq permissions " + permissionFlags(v.t, opts.Domain) + var errs []error + denied := map[string]bool{} + for _, r := range sorted { + switch { + case r.denied && !denied[r.unit.id()]: + denied[r.unit.id()] = true + v.add(FindingRequired, "consumer "+r.unit.id(), "permissions", + "the connecting user may not publish to %s, so this shard's rows are never pulled; regenerate its permissions with `%s`", r.subject, hint) + case r.err != nil: + errs = append(errs, fmt.Errorf("probe %s: %w", r.subject, r.err)) + } + } + return errors.Join(errs...) +} + +// probeOne sends one probe request, returning once it is answered (allowed) +// or the server refuses it (denied), whichever comes first. No answer and no +// refusal within natsProbeWait is an error. +func (v *topologyVerifier) probeOne(ctx context.Context, nc *nats.Conn, subject string, payload []byte) (denied bool, err error) { + since := v.perms.mark() + rctx, cancel := context.WithTimeout(ctx, natsProbeWait) + defer cancel() + answered := make(chan error, 1) + go func() { + _, err := nc.RequestWithContext(rctx, subject, payload) + answered <- err + }() + for { + changed := v.perms.changes() + if v.perms.deniedSince(subject, since) { + cancel() + <-answered + return true, nil + } + select { + case err := <-answered: + if err != nil && v.perms.deniedSince(subject, since) { + return true, nil + } + return false, err + case <-changed: + } + } +} + +// permissionFlags is the `wavehouse mq permissions` command line for t under +// domain: the shard count, then each other setting that is not its default. +func permissionFlags(t NATSTopology, domain string) string { + flags := "--shards " + strconv.Itoa(t.Shards) + if t.Prefix != DefaultNATSSubjectPrefix { + flags += " --prefix " + t.Prefix + } + if t.IngestConsumer != DefaultNATSIngestConsumer { + flags += " --ingest-consumer " + t.IngestConsumer + } + if t.HistoryStream != t.streamName("HISTORY") { + flags += " --history-stream " + t.HistoryStream + } + if t.CoordBucket != "" && t.CoordBucket != DefaultNATSCoordBucket(t.Prefix) { + flags += " --coord-bucket " + t.CoordBucket + } + if domain != "" { + flags += " --js-domain " + domain + } + return flags +} diff --git a/internal/mq/nats_topology.go b/internal/mq/nats_topology.go new file mode 100644 index 000000000..252d772a4 --- /dev/null +++ b/internal/mq/nats_topology.go @@ -0,0 +1,817 @@ +package mq + +import ( + "context" + "errors" + "fmt" + "regexp" + "slices" + "strconv" + "strings" + "time" + + "github.com/nats-io/nats.go/jetstream" +) + +// NATSTopology is what WaveHouse needs of an operator-owned JetStream: N +// ingest partition streams with work-queue retention, each split into V +// shards by a subject token, with one pinned-client durable per shard, and +// each republishing every row it stores to the history +// stream (limits retention, for SSE replay and the live hub) under the +// row's subject without the partition; one dead-letter stream; and, +// for coord.backend=nats, a KV bucket holding the leases (Leases). The +// operator creates all of it (WriteNATSManifests renders it as nack CRs); +// WaveHouse only checks it (verifyNATSTopology) and never repairs it. +type NATSTopology struct { + // Prefix leads every subject: .ingest.

.…, .hist.… and + // .dlq.…. + Prefix string + // Partitions is N, the number of ingest partition streams. + Partitions int + // Shards is V, the shards of every partition: a table's rows are all in + // one shard, and each shard has a durable of its own. + Shards int + // IngestConsumer names the shard durables: shard s's on every partition + // is - (natsShardDurable). + IngestConsumer string + // HistoryStream names the history stream, which holds .hist.>. + HistoryStream string + // PublishTimeout bounds one publish attempt; a partition's duplicate + // window must cover every attempt (minDuplicateWindow), so a retried + // publish is not stored twice. + PublishTimeout time.Duration + // DedupeLease is how long a dedupe claim stays pending (dedupe.lease); 0 + // skips its rule. A publish whose outcome was unknown keeps its claim + // until the lease lapses, and the client's retry is published under the + // same idempotency key, so a partition's duplicate window must still hold + // the first copy then (dedupeDuplicateWindow). + DedupeLease time.Duration + // CoordBucket is the KV bucket this process holds its leases in; empty + // when it holds none there, and then the bucket is not checked. + CoordBucket string + // AckWait, MaxAckPending and Prefetch are what the ingest worker asks of + // the durable (internal/ingest/worker.go, which imports this package). + AckWait time.Duration + MaxAckPending int + Prefetch int +} + +// Defaults for a NATSTopology's zero fields. +const ( + DefaultNATSSubjectPrefix = "wh" + DefaultNATSPartitions = 1 + DefaultNATSShards = 32 + // MaxNATSShards bounds V: the wavehouse user's permissions name every + // shard's durable, and each shard is a consumer (a Raft group at R3). + MaxNATSShards = 256 + DefaultNATSIngestConsumer = "wh-ingest" + DefaultNATSPublishTimeout = 5 * time.Second + defaultNATSAckWait = 60 * time.Second + defaultNATSMaxAckPending = 10_000 + defaultNATSPrefetch = 500 +) + +// withDefaults fills t's zero fields. +func (t NATSTopology) withDefaults() NATSTopology { + if t.Prefix == "" { + t.Prefix = DefaultNATSSubjectPrefix + } + if t.Partitions == 0 { + t.Partitions = DefaultNATSPartitions + } + if t.Shards == 0 { + t.Shards = DefaultNATSShards + } + if t.IngestConsumer == "" { + t.IngestConsumer = DefaultNATSIngestConsumer + } + if t.HistoryStream == "" { + t.HistoryStream = t.streamName("HISTORY") + } + if t.PublishTimeout == 0 { + t.PublishTimeout = DefaultNATSPublishTimeout + } + if t.AckWait == 0 { + t.AckWait = defaultNATSAckWait + } + if t.MaxAckPending == 0 { + t.MaxAckPending = defaultNATSMaxAckPending + } + if t.Prefetch == 0 { + t.Prefetch = defaultNATSPrefetch + } + return t +} + +// validate reports a topology no deployment could satisfy. +func (t NATSTopology) validate() error { + if err := validSubjectPrefix(t.Prefix); err != nil { + return err + } + if t.Partitions < 1 { + return fmt.Errorf("partitions must be at least 1, got %d", t.Partitions) + } + if t.Shards < 1 || t.Shards > MaxNATSShards { + return fmt.Errorf("shards must be from 1 to %d, got %d", MaxNATSShards, t.Shards) + } + // Both are one token of a JetStream API subject. + if !natsBucketName.MatchString(t.IngestConsumer) { + return fmt.Errorf("ingest consumer %q must be a name of [a-zA-Z0-9_-]", t.IngestConsumer) + } + if !natsBucketName.MatchString(t.HistoryStream) { + return fmt.Errorf("history stream %q must be a name of [a-zA-Z0-9_-]", t.HistoryStream) + } + if !natsBucketName.MatchString(t.coordBucket()) { + return fmt.Errorf("coord bucket %q must be a KV bucket name of [a-zA-Z0-9_-]", t.coordBucket()) + } + return nil +} + +// natsBucketName is JetStream's grammar for a KV bucket name. +var natsBucketName = regexp.MustCompile(`^[a-zA-Z0-9_-]+$`) + +// DefaultNATSCoordBucket is the lease bucket's name for a subject prefix, as +// the generated manifests name it: one per prefix, so deployments sharing a +// NATS account under different prefixes never contend for one lease. +func DefaultNATSCoordBucket(prefix string) string { return prefix + "_coord" } + +// coordBucket is the lease bucket: the configured one, or the prefix's. +func (t NATSTopology) coordBucket() string { + if t.CoordBucket != "" { + return t.CoordBucket + } + return DefaultNATSCoordBucket(t.Prefix) +} + +// streamName is the name the generated manifests give a stream of kind. Only +// the history's is binding; the others are found by subject. +func (t NATSTopology) streamName(kind string) string { + return strings.ToUpper(t.Prefix) + "_" + kind +} + +// minDuplicateWindow is the shortest duplicate window that stores a publish +// once however many of its attempts were stored: ExternalNATS sends the last +// retry this long after the first attempt. +func (t NATSTopology) minDuplicateWindow() time.Duration { + return (publishRetries+1)*t.PublishTimeout + publishRetries*publishRetryWait +} + +// dedupeDuplicateWindow is how long after a claim its record's retry can +// still be published under the same idempotency key: the lease, the +// Retry-After a client obeys (the lease rounded up to a second), and the +// second a claim can outlive its lease — config's rule for the embedded +// queue's window, applied here to the operator's. +func (t NATSTopology) dedupeDuplicateWindow() time.Duration { + ceil := t.DedupeLease + if r := ceil % time.Second; r != 0 { + ceil += time.Second - r + } + return t.DedupeLease + ceil + time.Second +} + +// natsPriorityGroup is the one priority group of every shard durable: a +// worker pulls a shard in it, and the server delivers to the one pinned +// puller. +const natsPriorityGroup = "wavehouse" + +// natsPullExpiry is the most time between two pulls of a shard that is at +// its cap or halted, which are there to keep its pin (at the cap, taking one +// row each for a while). Every pull itself waits a +// second at most; boot still requires max_expires to allow a pull this long, +// as headroom for longer pulls. The server renews a pin only when its +// holder sends a new pull, so a live owner keeps its pin only while this is +// well under the durable's pinned TTL. +const natsPullExpiry = 5 * time.Second + +// minPinnedTTL is the shortest pinned TTL a shard durable may have: twice +// the time between pulls, so a live owner pulls at least once before its pin +// lapses. +const minPinnedTTL = 2 * natsPullExpiry + +// FindingSeverity says whether a finding stops WaveHouse from serving. +type FindingSeverity int + +const ( + // FindingRequired refuses boot. + FindingRequired FindingSeverity = iota + // FindingRecommended is logged as a warning. + FindingRecommended +) + +func (s FindingSeverity) String() string { + if s == FindingRequired { + return "required" + } + return "recommended" +} + +// Finding is one way the operator's topology differs from what WaveHouse +// needs. +type Finding struct { + Severity FindingSeverity + // Object is what the finding is about, e.g. "stream WH_INGEST_0". + Object string + // Field is the setting, e.g. "retention", in the stream or consumer + // config's own (JSON) names. + Field string + // Problem says what is wrong and what is needed. + Problem string +} + +func (f Finding) String() string { + return fmt.Sprintf("%s: %s: %s: %s", f.Severity, f.Object, f.Field, f.Problem) +} + +// ErrTopology is what a TopologyError matches. +var ErrTopology = errors.New("nats topology does not match what WaveHouse needs") + +// TopologyError lists every finding from the last check, required ones first. +type TopologyError struct { + Findings []Finding +} + +func (e *TopologyError) Error() string { + var b strings.Builder + b.WriteString(ErrTopology.Error()) + for _, f := range e.Findings { + b.WriteString("\n - ") + b.WriteString(f.String()) + } + return b.String() +} + +func (e *TopologyError) Unwrap() error { return ErrTopology } + +// hasRequired reports whether any finding refuses boot. +func hasRequired(findings []Finding) bool { + return slices.ContainsFunc(findings, func(f Finding) bool { return f.Severity == FindingRequired }) +} + +// minNATSServer is the oldest server whose features the topology relies on: +// pinned-client priority groups and unpinning (2.11) and resetting a consumer +// to its ack floor (2.14), besides stream metadata, subject-filtered stream +// info and discard_new_per_subject. +var minNATSServer = [3]int{2, 14, 0} + +// recommendedNATSMinor is the server line the embedded broker runs. +const recommendedNATSMinor = "2.14." + +// topologyVerifier accumulates the findings of one check. +type topologyVerifier struct { + js jetstream.JetStream + t NATSTopology + findings []Finding + // perms is the connection's refused publishes, nil to skip probing the + // user's permissions; probe is the shard durables found, to probe. + perms *natsPermissionWatch + probe []natsUnit +} + +func (v *topologyVerifier) add(sev FindingSeverity, object, field, format string, args ...any) { + v.findings = append(v.findings, Finding{Severity: sev, Object: object, Field: field, Problem: fmt.Sprintf(format, args...)}) +} + +// verifyNATSTopology checks the operator's JetStream against t and returns +// every finding at once. The error is for a check that could not run (the +// server unreachable, a request refused); a missing stream or consumer is a +// finding. With perms, the publishes the server refused js's connection, it +// also probes that the connecting user may pull from and unpin every shard +// durable (probePermissions). +func verifyNATSTopology(ctx context.Context, js jetstream.JetStream, t NATSTopology, perms *natsPermissionWatch) ([]Finding, error) { + t = t.withDefaults() + if err := t.validate(); err != nil { + return nil, err + } + v := &topologyVerifier{js: js, t: t, perms: perms} + v.serverVersion(js.Conn().ConnectedServerVersion()) + + partitions := make([]string, t.Partitions) + for p := range t.Partitions { + name, err := v.partition(ctx, p) + if err != nil { + return nil, err + } + if q := slices.Index(partitions[:p], name); name != "" && q >= 0 { + v.add(FindingRequired, "stream "+name, "subjects", "holds partitions %d and %d; each partition needs a stream of its own", q, p) + } + partitions[p] = name + } + if err := v.extras(ctx, partitions); err != nil { + return nil, err + } + if err := v.history(ctx); err != nil { + return nil, err + } + if err := v.dlq(ctx); err != nil { + return nil, err + } + if t.CoordBucket != "" { + if err := v.coordBucket(ctx); err != nil { + return nil, err + } + } + if err := v.probePermissions(ctx); err != nil { + return nil, err + } + slices.SortStableFunc(v.findings, func(a, b Finding) int { return int(a.Severity) - int(b.Severity) }) + return v.findings, nil +} + +// awaitNATSTopology repeats the check until nothing required is missing or +// wait runs out — on Kubernetes the operator's CRs roll out with the pods — +// and then returns the warnings, or a *TopologyError with every finding of +// the last check. A check that could not run is retried the same way. +func awaitNATSTopology(ctx context.Context, js jetstream.JetStream, t NATSTopology, perms *natsPermissionWatch, wait time.Duration) ([]Finding, error) { + deadline := time.Now().Add(wait) + backoff := 250 * time.Millisecond + for { + findings, err := verifyNATSTopology(ctx, js, t, perms) + if err == nil && !hasRequired(findings) { + return findings, nil + } + remaining := time.Until(deadline) + if remaining <= 0 { + if err != nil { + return nil, fmt.Errorf("check nats topology: %w", err) + } + return nil, &TopologyError{Findings: findings} + } + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-time.After(min(backoff, remaining)): + } + backoff = min(2*backoff, 5*time.Second) + } +} + +func (v *topologyVerifier) serverVersion(version string) { + const object, field = "server", "version" + var got [3]int + parts := strings.SplitN(strings.TrimPrefix(version, "v"), ".", 3) + parsed := len(parts) == 3 + for i := 0; parsed && i < 3; i++ { + // Only the leading digits: a pre-release suffix rides on the patch. + end := strings.IndexFunc(parts[i], func(r rune) bool { return r < '0' || r > '9' }) + if end < 0 { + end = len(parts[i]) + } + n, err := strconv.Atoi(parts[i][:end]) + parsed = err == nil + got[i] = n + } + switch { + case !parsed: + v.add(FindingRequired, object, field, "cannot read server version %q; need at least %d.%d.%d", version, minNATSServer[0], minNATSServer[1], minNATSServer[2]) + case slices.Compare(got[:], minNATSServer[:]) < 0: + v.add(FindingRequired, object, field, "is %s; need at least %d.%d.%d", version, minNATSServer[0], minNATSServer[1], minNATSServer[2]) + case !strings.HasPrefix(strings.TrimPrefix(version, "v"), recommendedNATSMinor): + v.add(FindingRecommended, object, field, "is %s; WaveHouse is tested against %sx", version, recommendedNATSMinor) + } +} + +// streamsHolding lists the streams whose subjects match subject. +func (v *topologyVerifier) streamsHolding(ctx context.Context, subject string) ([]string, error) { + lister := v.js.StreamNames(ctx, jetstream.WithStreamListSubject(subject)) + var names []string + for name := range lister.Name() { + names = append(names, name) + } + if err := lister.Err(); err != nil { + return nil, fmt.Errorf("list streams holding %s: %w", subject, err) + } + slices.Sort(names) + return names, nil +} + +// findBySubject finds the one stream holding probe, or adds a finding and +// returns nil. +func (v *topologyVerifier) findBySubject(ctx context.Context, object, probe string) (jetstream.Stream, error) { + names, err := v.streamsHolding(ctx, probe) + if err != nil { + return nil, err + } + switch len(names) { + case 0: + v.add(FindingRequired, object, "subjects", "no stream holds %s", probe) + return nil, nil + case 1: + default: + v.add(FindingRequired, object, "subjects", "streams %s all hold %s; exactly one may", strings.Join(names, ", "), probe) + return nil, nil + } + return v.stream(ctx, object, names[0]) +} + +// stream looks a stream up by name, adding a finding when it does not exist. +func (v *topologyVerifier) stream(ctx context.Context, object, name string) (jetstream.Stream, error) { + s, err := v.js.Stream(ctx, name) + if errors.Is(err, jetstream.ErrStreamNotFound) { + v.add(FindingRequired, object, "name", "stream %s does not exist", name) + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("stream %s: %w", name, err) + } + return s, nil +} + +// partition checks ingest partition p and its durable, returning the stream's +// name ("" when there is none to check). +func (v *topologyVerifier) partition(ctx context.Context, p int) (string, error) { + t := v.t + filter := natsIngestPartition(t.Prefix, p) + s, err := v.findBySubject(ctx, fmt.Sprintf("ingest partition %d", p), fmt.Sprintf("%s.ingest.%d.x", t.Prefix, p)) + if s == nil || err != nil { + return "", err + } + cfg := s.CachedInfo().Config + obj := "stream " + cfg.Name + req := func(field, format string, args ...any) { v.add(FindingRequired, obj, field, format, args...) } + rec := func(field, format string, args ...any) { v.add(FindingRecommended, obj, field, format, args...) } + + if !slices.Contains(cfg.Subjects, filter) { + req("subjects", "are %q; must include %q", cfg.Subjects, filter) + } + // Work queue, not interest: a row no consumer's filter covers yet is + // kept rather than dropped, and a row leaves once the worker acks it. + if cfg.Retention != jetstream.WorkQueuePolicy { + req("retention", "is %s; must be workqueue, so a row is kept until the ingest worker acks it, even while no consumer covers it", cfg.Retention) + } + v.republish(obj, cfg.RePublish, p) + if cfg.Discard != jetstream.DiscardNew { + req("discard", "is %s; must be new, so a full partition refuses rather than dropping unwritten rows", cfg.Discard) + } + if cfg.MaxBytes <= 0 { + req("max_bytes", "is unlimited; must be set, to bound the disk and signal backpressure") + } + if cfg.MaxAge != 0 { + req("max_age", "is %s; must be unset, since an age limit drops unwritten rows", cfg.MaxAge) + } + if cfg.Storage != jetstream.FileStorage { + req("storage", "is %s; must be file", cfg.Storage) + } + if cfg.Duplicates < t.minDuplicateWindow() { + req("duplicate_window", "is %s; must be at least %s (every attempt of a retried publish), so it is stored once", cfg.Duplicates, t.minDuplicateWindow()) + } + if t.DedupeLease > 0 && cfg.Duplicates < t.dedupeDuplicateWindow() { + req("duplicate_window", "is %s; must be at least %s for dedupe.lease %s (the lease, a Retry-After of it rounded up, and a second), so the retry of a publish whose outcome was unknown is stored once", cfg.Duplicates, t.dedupeDuplicateWindow(), t.DedupeLease) + } + if cfg.NoAck { + req("no_ack", "is set; publishes must be acknowledged") + } + if cfg.Sealed { + req("sealed", "is set; the partition must take publishes") + } + if cfg.Mirror != nil { + req("mirror", "is set; a partition must not be a mirror") + } + switch { + case cfg.MaxMsgsPerSubject > 0 && !cfg.DiscardNewPerSubject: + // Without it the server keeps the cap by evicting the topic's oldest rows. + req("discard_new_per_subject", "is unset while max_msgs_per_subject is %d; must be set, or a topic at its cap loses its oldest unwritten rows", cfg.MaxMsgsPerSubject) + case cfg.MaxMsgsPerSubject <= 0: + rec("max_msgs_per_subject", "set it with discard_new_per_subject, so one topic cannot fill the partition for every tenant in it") + } + if !cfg.DenyPurge || !cfg.DenyDelete { + rec("deny_purge", "set deny_purge and deny_delete; nothing should remove unwritten rows") + } + if cfg.PersistMode == jetstream.AsyncPersistMode { + req("persist_mode", "is async; must be default, or an ack precedes the write and a crash of the server process loses unwritten rows") + } + v.replicas(obj, cfg.Replicas) + gotP, hasP := cfg.Metadata["wavehouse.dev/partition"] + gotN, hasN := cfg.Metadata["wavehouse.dev/partitions"] + switch { + case !hasP || !hasN: + rec("metadata", "set wavehouse.dev/partition and wavehouse.dev/partitions, so a partition count mismatch is caught by name") + case gotP != strconv.Itoa(p) || gotN != strconv.Itoa(t.Partitions): + req("metadata", "says partition %s of %s; WaveHouse is configured for partition %d of %d (mq.nats.partitions must match the operator's)", gotP, gotN, p, t.Partitions) + } + + shards, hasV := cfg.Metadata["wavehouse.dev/shards"] + switch { + case !hasV: + rec("metadata", "set wavehouse.dev/shards, so a shard count mismatch is caught by name") + case shards != strconv.Itoa(t.Shards): + req("metadata", "says %s shards; WaveHouse is configured for %d (mq.nats.shards must match the operator's)", shards, t.Shards) + } + for sh := range t.Shards { + if err := v.durable(ctx, s, natsShardDurable(t.IngestConsumer, sh), natsShardFilter(t.Prefix, p, sh)); err != nil { + return "", err + } + } + return cfg.Name, nil +} + +// durable checks shard durable name on stream s, filtering on filter. +func (v *topologyVerifier) durable(ctx context.Context, s jetstream.Stream, name, filter string) error { + t := v.t + stream := s.CachedInfo().Config.Name + obj := "consumer " + stream + "/" + name + c, err := s.Consumer(ctx, name) + if errors.Is(err, jetstream.ErrConsumerNotFound) { + v.add(FindingRequired, obj, "durable_name", "does not exist") + return nil + } + if errors.Is(err, jetstream.ErrNotPullConsumer) { + v.add(FindingRequired, obj, "deliver_subject", "is set; must be a pull consumer") + return nil + } + if err != nil { + return fmt.Errorf("consumer %s/%s: %w", stream, name, err) + } + cfg := c.CachedInfo().Config + req := func(field, format string, args ...any) { v.add(FindingRequired, obj, field, format, args...) } + v.probe = append(v.probe, natsUnit{stream: stream, durable: name}) + + if cfg.AckPolicy != jetstream.AckExplicitPolicy { + req("ack_policy", "is %s; must be explicit", cfg.AckPolicy) + } + if cfg.AckWait < t.AckWait { + req("ack_wait", "is %s; must be at least %s, the ingest worker's", cfg.AckWait, t.AckWait) + } + if cfg.MaxDeliver != -1 { + req("max_deliver", "is %d; must be -1, or a row that is never written stays in the partition undelivered", cfg.MaxDeliver) + } + switch { + case cfg.MaxAckPending <= 0: + req("max_ack_pending", "is %d; must be set", cfg.MaxAckPending) + case cfg.MaxAckPending < t.MaxAckPending: + v.add(FindingRecommended, obj, "max_ack_pending", "is %d; the ingest worker expects %d", cfg.MaxAckPending, t.MaxAckPending) + } + if cfg.DeliverPolicy != jetstream.DeliverAllPolicy { + req("deliver_policy", "is %s; must be all", cfg.DeliverPolicy) + } + filters := cfg.FilterSubjects + if cfg.FilterSubject != "" { + filters = append(filters, cfg.FilterSubject) + } + if !slices.Equal(filters, []string{filter}) { + req("filter_subject", "is %q; must be %q, the shard's subjects", filters, filter) + } + if cfg.HeadersOnly { + req("headers_only", "is set; the worker needs the bodies, and acking an empty one deletes the row") + } + if cfg.ReplayPolicy != jetstream.ReplayInstantPolicy { + req("replay_policy", "is %s; must be instant, or a backlog drains at the rate it arrived", cfg.ReplayPolicy) + } + if cfg.InactiveThreshold != 0 { + req("inactive_threshold", "is %s; a durable must not expire", cfg.InactiveThreshold) + } + if cfg.MaxRequestExpires != 0 && cfg.MaxRequestExpires < natsPullExpiry { + req("max_expires", "is %s; must be unset or at least %s, or the server may refuse WaveHouse's pulls", cfg.MaxRequestExpires, natsPullExpiry) + } + if cfg.MaxRequestBatch != 0 && cfg.MaxRequestBatch < t.Prefetch { + req("max_request_batch", "is %d; must be 0 or at least %d, the prefetch of a worker that owns this one shard", cfg.MaxRequestBatch, t.Prefetch) + } + // One puller receives at a time: the one the server pinned. + if cfg.PriorityPolicy != jetstream.PriorityPolicyPinned { + req("priority_policy", "is %s; must be pinned_client, so one worker at a time receives the shard's rows", priorityPolicyName(cfg.PriorityPolicy)) + } + if !slices.Equal(cfg.PriorityGroups, []string{natsPriorityGroup}) { + req("priority_groups", "are %q; must be exactly [%q]", cfg.PriorityGroups, natsPriorityGroup) + } + switch { + case cfg.PriorityPolicy != jetstream.PriorityPolicyPinned: + case cfg.PinnedTTL < minPinnedTTL: + // The server renews a pin only on a new pull from its holder. + req("priority_timeout", "is %s; must be at least %s, twice the %s between WaveHouse's pulls of a shard at its cap, or a live owner loses its pin", cfg.PinnedTTL, minPinnedTTL, natsPullExpiry) + case cfg.PinnedTTL >= defaultLeaseDuration: + v.add(FindingRecommended, obj, "priority_timeout", "is %s; keep it under %s, the lease after which the other processes count a dead holder gone, so its pin has lapsed by then", cfg.PinnedTTL, defaultLeaseDuration) + } + return nil +} + +// priorityPolicyName is p as the consumer config spells it. +func priorityPolicyName(p jetstream.PriorityPolicy) string { + b, err := p.MarshalJSON() + if err != nil { + return strconv.Itoa(int(p)) + } + if name := strings.Trim(string(b), `"`); name != "" { + return name + } + return "none" +} + +// extras warns about shard durables outside the N×V the topology names: on +// a stream a lower partition count left behind, or for a shard past a lower +// shard count. The ingest workers drain each one (ExternalNATS.IngestUnits) +// until the operator deletes it; a stream without one has nothing to drain it. +func (v *topologyVerifier) extras(ctx context.Context, partitions []string) error { + units, err := findExtraUnits(ctx, v.js, v.t, partitions) + if err != nil { + return err + } + for _, u := range units { + if u.durable == "" { + v.add(FindingRecommended, "stream "+u.stream, "subjects", + "holds %s.ingest subjects outside partitions 0-%d and has no shard durable %s-, so nothing drains its %d rows; delete it", v.t.Prefix, v.t.Partitions-1, v.t.IngestConsumer, u.rows) + continue + } + v.add(FindingRecommended, "consumer "+u.id(), "durable_name", + "is outside the %d partition(s) of %d shard(s) WaveHouse is configured for; the ingest workers drain its %d rows: delete it once it holds none and no process runs the old counts", v.t.Partitions, v.t.Shards, u.rows) + } + return nil +} + +// natsUnit is one shard durable on one stream: what one worker at a time +// consumes. +type natsUnit struct { + stream, durable string + // rows is what the unit still holds: pending delivery or unacked, as + // last read (extras only). + rows uint64 +} + +// id is the unit's name, the same in every process. +func (u natsUnit) id() string { return u.stream + "/" + u.durable } + +// findExtraUnits lists the shard durables outside the topology's N×V, on any +// stream holding ingest subjects, and each such stream with none at all +// (durable empty). partitions is the configured partitions' streams. +func findExtraUnits(ctx context.Context, js jetstream.JetStream, t NATSTopology, partitions []string) ([]natsUnit, error) { + v := &topologyVerifier{js: js, t: t} + names, err := v.streamsHolding(ctx, t.Prefix+".ingest.>") + if err != nil { + return nil, err + } + var out []natsUnit + for _, name := range names { + configured := slices.Contains(partitions, name) + s, err := js.Stream(ctx, name) + if errors.Is(err, jetstream.ErrStreamNotFound) { + continue + } + if err != nil { + return nil, fmt.Errorf("stream %s: %w", name, err) + } + lister := s.ConsumerNames(ctx) + var shards []int + for c := range lister.Name() { + if sh, ok := parseShardDurable(t.IngestConsumer, c); ok && (!configured || sh >= t.Shards) { + shards = append(shards, sh) + } + } + if err := lister.Err(); err != nil { + return nil, fmt.Errorf("list consumers of %s: %w", name, err) + } + slices.Sort(shards) + if len(shards) == 0 && !configured { + out = append(out, natsUnit{stream: name, rows: s.CachedInfo().State.Msgs}) + } + for _, sh := range shards { + durable := natsShardDurable(t.IngestConsumer, sh) + c, err := s.Consumer(ctx, durable) + if errors.Is(err, jetstream.ErrConsumerNotFound) || errors.Is(err, jetstream.ErrNotPullConsumer) { + continue + } + if err != nil { + return nil, fmt.Errorf("consumer %s/%s: %w", name, durable, err) + } + info := c.CachedInfo() + out = append(out, natsUnit{stream: name, durable: durable, rows: info.NumPending + uint64(max(0, info.NumAckPending))}) + } + } + return out, nil +} + +func (v *topologyVerifier) history(ctx context.Context) error { + t := v.t + obj := "stream " + t.HistoryStream + s, err := v.stream(ctx, obj, t.HistoryStream) + if s == nil || err != nil { + return err + } + info := s.CachedInfo() + cfg := info.Config + req := func(field, format string, args ...any) { v.add(FindingRequired, obj, field, format, args...) } + + if subjects := natsHistorySubjects(t.Prefix); !slices.Equal(cfg.Subjects, []string{subjects}) { + req("subjects", "are %q; must be exactly %q, where the partitions republish every row", cfg.Subjects, subjects) + } + if len(cfg.Sources) > 0 || cfg.Mirror != nil { + req("sources", "are set; the history must have no sources or mirror, since the partitions' republish already stores every row there") + } + if cfg.Retention != jetstream.LimitsPolicy { + req("retention", "is %s; must be limits", cfg.Retention) + } + // A full history never holds up ingest (republish does not wait for it), + // but with discard new it stops taking the newest rows SSE wants. + if cfg.Discard != jetstream.DiscardOld { + v.add(FindingRecommended, obj, "discard", "is %s; old keeps the newest rows for SSE when the history is full", cfg.Discard) + } + if cfg.MaxAge <= 0 { + req("max_age", "is unlimited; must be set, at least the longest gap window a tenant replays") + } + if cfg.MaxBytes <= 0 { + v.add(FindingRecommended, obj, "max_bytes", "is unlimited; set it to bound the disk") + } + v.replicas(obj, cfg.Replicas) + return nil +} + +// republish checks that partition p republishes every row it stores to the +// history, with the partition token dropped from the subject. +func (v *topologyVerifier) republish(obj string, rp *jetstream.RePublish, p int) { + want := natsRepublish(v.t.Prefix, p) + switch { + case rp == nil: + v.add(FindingRequired, obj, "republish", "is unset; must be {src: %q, dest: %q}, or the history (SSE replay and the live hub) gets nothing", want.Source, want.Destination) + case rp.Source != want.Source || rp.Destination != want.Destination: + v.add(FindingRequired, obj, "republish", "is {src: %q, dest: %q}; must be {src: %q, dest: %q}", rp.Source, rp.Destination, want.Source, want.Destination) + case rp.HeadersOnly: + v.add(FindingRequired, obj, "republish", "is headers_only; the history needs the bodies") + } +} + +func (v *topologyVerifier) dlq(ctx context.Context) error { + s, err := v.findBySubject(ctx, "dead-letter stream", v.t.Prefix+".dlq.x") + if s == nil || err != nil { + return err + } + cfg := s.CachedInfo().Config + obj := "stream " + cfg.Name + req := func(field, format string, args ...any) { v.add(FindingRequired, obj, field, format, args...) } + + if filter := natsDLQSubjects(v.t.Prefix); !slices.Contains(cfg.Subjects, filter) { + req("subjects", "are %q; must include %q", cfg.Subjects, filter) + } + if cfg.Retention != jetstream.LimitsPolicy { + req("retention", "is %s; must be limits", cfg.Retention) + } + if cfg.Discard != jetstream.DiscardOld { + req("discard", "is %s; must be old", cfg.Discard) + } + if cfg.Storage != jetstream.FileStorage { + req("storage", "is %s; must be file", cfg.Storage) + } + if cfg.MaxBytes <= 0 { + req("max_bytes", "is unlimited; must be set") + } + if cfg.MaxMsgsPerSubject <= 0 { + v.add(FindingRecommended, obj, "max_msgs_per_subject", "set it, so one topic's parked rows evict only its own") + } + if cfg.PersistMode == jetstream.AsyncPersistMode { + v.add(FindingRecommended, obj, "persist_mode", "is async; a crash of the server process loses parked rows it acked") + } + v.replicas(obj, cfg.Replicas) + return nil +} + +// replicas recommends 3 replicas for a stream holding rows. WaveHouse does +// not require sync_always under nats: an R3 publish is acked once a quorum +// has stored it, so with one replica an ack rests on one server's disk. +func (v *topologyVerifier) replicas(obj string, n int) { + if problem, ok := replicasProblem(n); ok { + v.add(FindingRecommended, obj, "num_replicas", "%s", problem) + } +} + +func replicasProblem(n int) (string, bool) { + switch { + case n <= 1: + return fmt.Sprintf("is %d; an ack then rests on one server's disk, and a crash loses what it stored since its last sync (sync_interval); 3 across failure domains survives losing a server", n), true + case n < 3: + return fmt.Sprintf("is %d; 3 across failure domains survives losing a server", n), true + } + return "", false +} + +// coordBucket checks the KV bucket the leases live in (Leases). Its stream +// is KV_, which is how JetStream stores a bucket. +func (v *topologyVerifier) coordBucket(ctx context.Context) error { + name := v.t.CoordBucket + obj := "kv bucket " + name + s, err := v.js.Stream(ctx, "KV_"+name) + if errors.Is(err, jetstream.ErrStreamNotFound) { + v.add(FindingRequired, obj, "bucket", "does not exist; coord.backend=nats holds its leases there") + return nil + } + if err != nil { + return fmt.Errorf("kv bucket %s: %w", name, err) + } + cfg := s.CachedInfo().Config + req := func(field, format string, args ...any) { v.add(FindingRequired, obj, field, format, args...) } + + if cfg.MaxMsgsPerSubject < 1 { + req("history", "is unset; a KV bucket keeps at least one value per key") + } + // The wavehouse user may read a key only by direct get. + if !cfg.AllowDirect { + req("allow_direct", "is unset; WaveHouse reads leases by direct get (a bucket nack or `nats kv add` creates has it)") + } + // A candidate judges expiry on its own clock; a key the server expires + // would end a live lease early. + if cfg.MaxAge != 0 { + req("ttl", "is %s; must be unset, or a live lease expires under its holder", cfg.MaxAge) + } + if cfg.Storage != jetstream.FileStorage { + v.add(FindingRecommended, obj, "storage", "is %s; file survives a server restart without every lease starting over", cfg.Storage) + } + if cfg.Replicas < 3 { + v.add(FindingRecommended, obj, "num_replicas", "is %d; 3 survives losing a server", cfg.Replicas) + } + return nil +} diff --git a/internal/mq/nats_topology_test.go b/internal/mq/nats_topology_test.go new file mode 100644 index 000000000..0c0edb9bf --- /dev/null +++ b/internal/mq/nats_topology_test.go @@ -0,0 +1,464 @@ +package mq + +import ( + "bytes" + "context" + "errors" + "os" + "path/filepath" + "slices" + "strings" + "testing" + "time" + + "github.com/nats-io/nats.go/jetstream" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gopkg.in/yaml.v3" + + "github.com/Wave-RF/WaveHouse/internal/mq/natstest" +) + +// shippedSpec is the topology the shipped manifests are generated for, and +// coordSpec the same for a process holding its leases there. +var ( + shippedSpec = NATSTopology{Partitions: 4, Shards: 8, DedupeLease: 30 * time.Second} + coordSpec = NATSTopology{Partitions: 4, Shards: 8, CoordBucket: natstest.CoordBucket} +) + +// replicaWarnings are what the shipped manifests at one replica leave: one +// num_replicas recommendation per partition, the history and the DLQ. +func replicaWarnings(findings []Finding) bool { + for _, f := range findings { + if f.Severity != FindingRecommended || f.Field != "num_replicas" { + return false + } + } + return len(findings) == shippedSpec.Partitions+2 +} + +// The shipped manifests pass the verifier, run as the wavehouse user with +// exactly the shipped permissions. +func TestVerifyNATSTopology_ShippedManifestsPass(t *testing.T) { + t.Parallel() + f := newNATSFixture(t) + f.apply(t, shippedTopology(t)) + js := f.connect(t, "wavehouse") + findings, err := verifyNATSTopology(t.Context(), js, shippedSpec, nil) + require.NoError(t, err) + assert.True(t, replicaWarnings(findings), "findings: %v", findings) + + // With the lease bucket checked too: one more replica warning, its own. + findings, err = verifyNATSTopology(t.Context(), js, coordSpec, nil) + require.NoError(t, err) + require.Len(t, findings, shippedSpec.Partitions+3, "findings: %v", findings) + last := findings[len(findings)-1] + assert.Equal(t, "kv bucket wh_coord", last.Object) + assert.Equal(t, "num_replicas", last.Field) +} + +// Every rule the verifier holds the operator to, one mutation each. +func TestVerifyNATSTopology_Findings(t *testing.T) { //nolint:tparallel // its cases share one server, so they run in turn + t.Parallel() + const ( + p0 = "WH_INGEST_0" + history = "WH_HISTORY" + dlq = "WH_DLQ" + ) + // Four partitions, as shipped, of two shards: fewer consumers to create + // per case. + spec := NATSTopology{Partitions: 4, Shards: 2, DedupeLease: 30 * time.Second} + type want struct { + sev FindingSeverity + object string // a substring of Finding.Object + field string + // problem, when set, is a substring of Finding.Problem, for cases + // whose field has more than one rule. + problem string + } + req := func(object, field string) want { return want{FindingRequired, object, field, ""} } + rec := func(object, field string) want { return want{FindingRecommended, object, field, ""} } + stream := func(name string, mut func(*jetstream.StreamConfig)) func(*testing.T, *fixtureTopology) { + return func(t *testing.T, tp *fixtureTopology) { mut(tp.stream(t, name)) } + } + durable := func(mut func(*jetstream.ConsumerConfig)) func(*testing.T, *fixtureTopology) { + return func(t *testing.T, tp *fixtureTopology) { mut(tp.consumer(t, p0)) } + } + + cases := []struct { + name string + mutate func(*testing.T, *fixtureTopology) + spec NATSTopology + want want + }{ + // Ingest partitions. + {"partition missing", func(_ *testing.T, tp *fixtureTopology) { tp.drop(p0) }, spec, req("ingest partition 0", "subjects")}, + {"partition subjects", stream(p0, func(s *jetstream.StreamConfig) { s.Subjects = []string{"wh.ingest.0.*"} }), spec, req(p0, "subjects")}, + {"partition retention", stream(p0, func(s *jetstream.StreamConfig) { s.Retention = jetstream.LimitsPolicy }), spec, req(p0, "retention")}, + {"partition interest retention", stream(p0, func(s *jetstream.StreamConfig) { s.Retention = jetstream.InterestPolicy }), spec, req(p0, "retention")}, + {"partition republish missing", stream(p0, func(s *jetstream.StreamConfig) { s.RePublish = nil }), spec, want{FindingRequired, p0, "republish", "is unset"}}, + {"partition republish keeps the partition token", stream(p0, func(s *jetstream.StreamConfig) { + s.RePublish = &jetstream.RePublish{Source: "wh.ingest.>", Destination: "wh.hist.>"} + }), spec, want{FindingRequired, p0, "republish", "must be {src"}}, + {"partition republish keeps the shard token", stream(p0, func(s *jetstream.StreamConfig) { + s.RePublish = &jetstream.RePublish{Source: "wh.ingest.0.>", Destination: "wh.hist.>"} + }), spec, want{FindingRequired, p0, "republish", "must be {src"}}, + {"partition republish headers only", stream(p0, func(s *jetstream.StreamConfig) { s.RePublish.HeadersOnly = true }), spec, want{FindingRequired, p0, "republish", "headers_only"}}, + {"partition discard", stream(p0, func(s *jetstream.StreamConfig) { + s.Discard, s.DiscardNewPerSubject = jetstream.DiscardOld, false + }), spec, req(p0, "discard")}, + {"partition max_bytes", stream(p0, func(s *jetstream.StreamConfig) { s.MaxBytes = -1 }), spec, req(p0, "max_bytes")}, + {"partition max_age", stream(p0, func(s *jetstream.StreamConfig) { s.MaxAge = time.Hour }), spec, req(p0, "max_age")}, + {"partition storage", stream(p0, func(s *jetstream.StreamConfig) { s.Storage = jetstream.MemoryStorage }), spec, req(p0, "storage")}, + {"partition duplicate_window", stream(p0, func(s *jetstream.StreamConfig) { s.Duplicates = time.Second }), spec, req(p0, "duplicate_window")}, + {"duplicate window against the publish timeout", nil, NATSTopology{Partitions: 4, Shards: 2, PublishTimeout: 2 * time.Minute}, req(p0, "duplicate_window")}, + // 59.5s + 60s + 1s = 2m0.5s, just over the shipped 2m. + {"duplicate window against the dedupe lease", nil, NATSTopology{Partitions: 4, Shards: 2, DedupeLease: 59500 * time.Millisecond}, want{FindingRequired, p0, "duplicate_window", "dedupe.lease"}}, + {"partition no_ack", stream(p0, func(s *jetstream.StreamConfig) { s.NoAck = true }), spec, req(p0, "no_ack")}, + {"partition per-subject cap", stream(p0, func(s *jetstream.StreamConfig) { + s.MaxMsgsPerSubject, s.DiscardNewPerSubject = 0, false + }), spec, rec(p0, "max_msgs_per_subject")}, + {"partition per-subject cap evicting", stream(p0, func(s *jetstream.StreamConfig) { + s.MaxMsgsPerSubject, s.DiscardNewPerSubject = 1000, false + }), spec, req(p0, "discard_new_per_subject")}, + {"one stream for two partitions", func(t *testing.T, tp *fixtureTopology) { + tp.drop("WH_INGEST_1") + s := tp.stream(t, p0) + s.Subjects = append(s.Subjects, "wh.ingest.1.>") + s.Metadata = nil + }, spec, req(p0, "subjects")}, + {"partition deny_purge", stream(p0, func(s *jetstream.StreamConfig) { s.DenyPurge = false }), spec, rec(p0, "deny_purge")}, + {"partition at one replica", nil, spec, want{FindingRecommended, p0, "num_replicas", "sync_interval"}}, + {"partition persist_mode async", stream(p0, func(s *jetstream.StreamConfig) { s.PersistMode = jetstream.AsyncPersistMode }), spec, req(p0, "persist_mode")}, + {"partition metadata missing", stream(p0, func(s *jetstream.StreamConfig) { s.Metadata = nil }), spec, rec(p0, "metadata")}, + {"partition metadata mismatch", stream(p0, func(s *jetstream.StreamConfig) { + s.Metadata = map[string]string{"wavehouse.dev/partition": "3", "wavehouse.dev/partitions": "4"} + }), spec, req(p0, "metadata")}, + {"partition count mismatch caught by metadata", nil, NATSTopology{Partitions: 2, Shards: 2}, req(p0, "metadata")}, + { + "partitions beyond N are drained", nil, + NATSTopology{Partitions: 2, Shards: 2}, + want{FindingRecommended, "WH_INGEST_3/wh-ingest-1", "durable_name", "the ingest workers drain its 0 rows"}, + }, + { + "partitions beyond N without the durables", func(_ *testing.T, tp *fixtureTopology) { delete(tp.Consumers, "WH_INGEST_3") }, + NATSTopology{Partitions: 2, Shards: 2}, + want{FindingRecommended, "WH_INGEST_3", "subjects", "has no shard durable wh-ingest-, so nothing drains"}, + }, + { + "shards beyond V are drained", nil, + NATSTopology{Partitions: 4, Shards: 1}, + want{FindingRecommended, p0 + "/wh-ingest-1", "durable_name", "the ingest workers drain its 0 rows"}, + }, + {"shard count mismatch caught by metadata", nil, NATSTopology{Partitions: 4, Shards: 1}, req(p0, "metadata")}, + {"shard durable missing", func(_ *testing.T, tp *fixtureTopology) { + tp.Consumers[p0] = slices.DeleteFunc(tp.Consumers[p0], func(c jetstream.ConsumerConfig) bool { return c.Durable == "wh-ingest-1" }) + }, spec, req(p0+"/wh-ingest-1", "durable_name")}, + + // The wh-ingest durable. + {"durable missing", func(_ *testing.T, tp *fixtureTopology) { delete(tp.Consumers, p0) }, spec, req(p0+"/wh-ingest-0", "durable_name")}, + {"durable is push", durable(func(c *jetstream.ConsumerConfig) { + c.DeliverSubject = "deliver.here" + c.MaxAckPending = 0 + c.PriorityPolicy, c.PriorityGroups, c.PinnedTTL = jetstream.PriorityPolicyNone, nil, 0 + }), spec, req(p0+"/wh-ingest-0", "deliver_subject")}, + {"durable ack_policy", func(t *testing.T, tp *fixtureTopology) { + tp.stream(t, p0).Retention = jetstream.LimitsPolicy + c := tp.consumer(t, p0) + c.AckPolicy, c.MaxAckPending = jetstream.AckNonePolicy, 0 + }, spec, req(p0+"/wh-ingest-0", "ack_policy")}, + {"durable ack_wait", durable(func(c *jetstream.ConsumerConfig) { c.AckWait = 30 * time.Second }), spec, req(p0+"/wh-ingest-0", "ack_wait")}, + {"durable max_deliver", durable(func(c *jetstream.ConsumerConfig) { c.MaxDeliver = 5 }), spec, req(p0+"/wh-ingest-0", "max_deliver")}, + {"durable max_ack_pending unlimited", durable(func(c *jetstream.ConsumerConfig) { c.MaxAckPending = -1 }), spec, req(p0+"/wh-ingest-0", "max_ack_pending")}, + {"durable max_ack_pending low", durable(func(c *jetstream.ConsumerConfig) { c.MaxAckPending = 100 }), spec, rec(p0+"/wh-ingest-0", "max_ack_pending")}, + // A work queue refuses such a durable itself; on a partition of the + // wrong retention the verifier still names it. + {"durable deliver_policy", func(t *testing.T, tp *fixtureTopology) { + tp.stream(t, p0).Retention = jetstream.LimitsPolicy + tp.consumer(t, p0).DeliverPolicy = jetstream.DeliverNewPolicy + }, spec, req(p0+"/wh-ingest-0", "deliver_policy")}, + {"durable filter", durable(func(c *jetstream.ConsumerConfig) { c.FilterSubject = "wh.ingest.0.0.acme.>" }), spec, req(p0+"/wh-ingest-0", "filter_subject")}, + {"durable headers_only", durable(func(c *jetstream.ConsumerConfig) { c.HeadersOnly = true }), spec, req(p0+"/wh-ingest-0", "headers_only")}, + {"durable replay_policy", durable(func(c *jetstream.ConsumerConfig) { c.ReplayPolicy = jetstream.ReplayOriginalPolicy }), spec, req(p0+"/wh-ingest-0", "replay_policy")}, + {"durable inactive_threshold", durable(func(c *jetstream.ConsumerConfig) { c.InactiveThreshold = time.Hour }), spec, req(p0+"/wh-ingest-0", "inactive_threshold")}, + {"durable max_expires", durable(func(c *jetstream.ConsumerConfig) { c.MaxRequestExpires = time.Second }), spec, req(p0+"/wh-ingest-0", "max_expires")}, + {"durable max_request_batch", durable(func(c *jetstream.ConsumerConfig) { c.MaxRequestBatch = 10 }), spec, req(p0+"/wh-ingest-0", "max_request_batch")}, + {"durable priority_policy", durable(func(c *jetstream.ConsumerConfig) { + c.PriorityPolicy, c.PriorityGroups, c.PinnedTTL = jetstream.PriorityPolicyNone, nil, 0 + }), spec, want{FindingRequired, p0 + "/wh-ingest-0", "priority_policy", "is none; must be pinned_client"}}, + {"durable priority_groups", durable(func(c *jetstream.ConsumerConfig) { c.PriorityGroups = []string{"workers"} }), spec, req(p0+"/wh-ingest-0", "priority_groups")}, + {"durable pinned ttl under two pull expiries", durable(func(c *jetstream.ConsumerConfig) { c.PinnedTTL = 9 * time.Second }), spec, req(p0+"/wh-ingest-0", "priority_timeout")}, + {"durable pinned ttl over the membership lease", durable(func(c *jetstream.ConsumerConfig) { c.PinnedTTL = 20 * time.Second }), spec, rec(p0+"/wh-ingest-0", "priority_timeout")}, + + // The history. + {"history missing", func(_ *testing.T, tp *fixtureTopology) { tp.drop(history) }, spec, req(history, "name")}, + {"history subjects", stream(history, func(s *jetstream.StreamConfig) { s.Subjects = []string{"history.>"} }), spec, req(history, "subjects")}, + {"history subjects wider", stream(history, func(s *jetstream.StreamConfig) { s.Subjects = []string{"wh.hist.>", "wh.other.>"} }), spec, req(history, "subjects")}, + {"history sources a partition", stream(history, func(s *jetstream.StreamConfig) { + s.Sources = []*jetstream.StreamSource{{Name: p0}} + }), spec, req(history, "sources")}, + {"history retention", stream(history, func(s *jetstream.StreamConfig) { s.Retention = jetstream.InterestPolicy }), spec, req(history, "retention")}, + {"history discard", stream(history, func(s *jetstream.StreamConfig) { s.Discard = jetstream.DiscardNew }), spec, rec(history, "discard")}, + {"history max_age", stream(history, func(s *jetstream.StreamConfig) { s.MaxAge = 0 }), spec, req(history, "max_age")}, + {"history max_bytes", stream(history, func(s *jetstream.StreamConfig) { s.MaxBytes = -1 }), spec, rec(history, "max_bytes")}, + {"history at one replica", nil, spec, want{FindingRecommended, history, "num_replicas", "sync_interval"}}, + {"history named elsewhere", nil, NATSTopology{Partitions: 4, Shards: 2, HistoryStream: "OTHER"}, req("OTHER", "name")}, + + // The dead-letter stream. + {"dlq missing", func(_ *testing.T, tp *fixtureTopology) { tp.drop(dlq) }, spec, req("dead-letter stream", "subjects")}, + {"dlq subjects", stream(dlq, func(s *jetstream.StreamConfig) { s.Subjects = []string{"wh.dlq.x", "wh.dlq.acme.>"} }), spec, req(dlq, "subjects")}, + {"dlq retention", stream(dlq, func(s *jetstream.StreamConfig) { s.Retention = jetstream.InterestPolicy }), spec, req(dlq, "retention")}, + {"dlq discard", stream(dlq, func(s *jetstream.StreamConfig) { s.Discard = jetstream.DiscardNew }), spec, req(dlq, "discard")}, + {"dlq storage", stream(dlq, func(s *jetstream.StreamConfig) { s.Storage = jetstream.MemoryStorage }), spec, req(dlq, "storage")}, + {"dlq max_bytes", stream(dlq, func(s *jetstream.StreamConfig) { s.MaxBytes = -1 }), spec, req(dlq, "max_bytes")}, + {"dlq at one replica", nil, spec, want{FindingRecommended, dlq, "num_replicas", "sync_interval"}}, + {"dlq persist_mode async", stream(dlq, func(s *jetstream.StreamConfig) { s.PersistMode = jetstream.AsyncPersistMode }), spec, rec(dlq, "persist_mode")}, + {"dlq per-subject cap", stream(dlq, func(s *jetstream.StreamConfig) { s.MaxMsgsPerSubject = 0 }), spec, rec(dlq, "max_msgs_per_subject")}, + } + // One server for every case, emptied between them: a server per case + // costs more than the unit suite's per-package timeout can spare. + f := newNATSFixture(t) + js := f.connect(t, "wavehouse") + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + f.reset(t) + tp := smallTopology(t) + tp.KeyValues = nil // no case here checks the lease bucket + if tc.mutate != nil { + tc.mutate(t, tp) + } + require.NoError(t, f.create(t.Context(), tp)) + findings, err := verifyNATSTopology(t.Context(), js, tc.spec, nil) + require.NoError(t, err) + found := false + for _, got := range findings { + if got.Severity == tc.want.sev && got.Field == tc.want.field && strings.Contains(got.Object, tc.want.object) && + strings.Contains(got.Problem, tc.want.problem) { + found = true + } + } + assert.True(t, found, "want %s %s/%s among %v", tc.want.sev, tc.want.object, tc.want.field, findings) + if tc.want.sev == FindingRequired { + assert.Equal(t, FindingRequired, findings[0].Severity, "required findings sort first") + } + }) + } +} + +// Fewer than 3 replicas is recommended against, never required: a one-server +// dev cluster boots. At one replica nothing but the server's sync interval +// stands behind an ack, since WaveHouse does not require sync_always. +func TestReplicasProblem(t *testing.T) { + t.Parallel() + for n, want := range map[int]string{0: "sync_interval", 1: "sync_interval", 2: "3 across failure domains"} { + got, ok := replicasProblem(n) + assert.True(t, ok, "replicas %d", n) + assert.Contains(t, got, want, "replicas %d", n) + } + got, _ := replicasProblem(2) + assert.NotContains(t, got, "sync_interval", "a quorum of two does not rest on one disk") + for _, n := range []int{3, 5} { + _, ok := replicasProblem(n) + assert.False(t, ok, "replicas %d", n) + } +} + +// WaveHouse does not require sync_always under nats, so the shipped Helm +// values set no sync option anywhere. natstest.ServerConfig reads only +// config.merge, so this reads the file itself. +func TestShippedValues_SetNoSync(t *testing.T) { + t.Parallel() + raw, err := os.ReadFile(natstest.ShippedValues()) + require.NoError(t, err) + var values any + require.NoError(t, yaml.Unmarshal(raw, &values)) + var walk func(path string, v any) + walk = func(path string, v any) { + switch v := v.(type) { + case map[string]any: + for k, child := range v { + assert.NotContains(t, strings.ToLower(k), "sync", "values.yaml sets %s.%s", path, k) + walk(path+"."+k, child) + } + case []any: + for _, child := range v { + walk(path, child) + } + } + } + walk("", values) +} + +// Boot waits for the operator's resources, which on Kubernetes roll out with +// the pods, and passes once they are there. +func TestAwaitNATSTopology_WaitsForTheOperator(t *testing.T) { + t.Parallel() + f := newNATSFixture(t) + js := f.connect(t, "wavehouse") + tp := shippedTopology(t) + created := make(chan error, 1) + go func() { + time.Sleep(time.Second) + created <- f.create(t.Context(), tp) + }() + start := time.Now() + findings, err := awaitNATSTopology(t.Context(), js, shippedSpec, nil, 20*time.Second) + require.NoError(t, <-created) + require.NoError(t, err) + assert.True(t, replicaWarnings(findings), "findings: %v", findings) + assert.GreaterOrEqual(t, time.Since(start), time.Second) +} + +// When the wait runs out, one error lists every finding at once. +func TestAwaitNATSTopology_ListsEveryFinding(t *testing.T) { + t.Parallel() + f := newNATSFixture(t) + tp := shippedTopology(t) + tp.drop("WH_DLQ") + tp.stream(t, "WH_INGEST_1").Retention = jetstream.LimitsPolicy + delete(tp.Consumers, "WH_INGEST_2") + f.apply(t, tp) + + _, err := awaitNATSTopology(t.Context(), f.connect(t, "wavehouse"), shippedSpec, nil, 300*time.Millisecond) + require.ErrorIs(t, err, ErrTopology) + var terr *TopologyError + require.True(t, errors.As(err, &terr)) + msg := err.Error() + for _, want := range []string{"dead-letter stream", "stream WH_INGEST_1: retention", "consumer WH_INGEST_2/wh-ingest-0", "consumer WH_INGEST_2/wh-ingest-7"} { + assert.Contains(t, msg, want) + } + assert.Equal(t, 2+8, countRequired(terr.Findings), "one finding per missing shard durable") +} + +func countRequired(findings []Finding) int { + n := 0 + for _, f := range findings { + if f.Severity == FindingRequired { + n++ + } + } + return n +} + +// A check that cannot run is an error, not a finding, and await gives up on +// its context. +func TestAwaitNATSTopology_ContextEnds(t *testing.T) { + t.Parallel() + f := newNATSFixture(t) + js := f.connect(t, "wavehouse") + ctx, cancel := context.WithTimeout(t.Context(), 200*time.Millisecond) + defer cancel() + _, err := awaitNATSTopology(ctx, js, shippedSpec, nil, time.Minute) + require.ErrorIs(t, err, context.DeadlineExceeded) +} + +func TestVerifyNATSTopology_ServerVersion(t *testing.T) { + t.Parallel() + cases := map[string]*FindingSeverity{ + "2.14.6": nil, + "v2.14.0-beta": nil, + "2.10.0": new(FindingRequired), + "2.13.9": new(FindingRequired), + "2.15.1": new(FindingRecommended), + "2.9.25": new(FindingRequired), + "1.4.1": new(FindingRequired), + "garbage": new(FindingRequired), + } + for version, want := range cases { + v := &topologyVerifier{} + v.serverVersion(version) + if want == nil { + assert.Empty(t, v.findings, version) + continue + } + require.Len(t, v.findings, 1, version) + assert.Equal(t, *want, v.findings[0].Severity, version) + } +} + +func TestVerifyNATSTopology_RefusesAnImpossibleSpec(t *testing.T) { + t.Parallel() + f := newNATSFixture(t) + js := f.connect(t, "wavehouse") + _, err := verifyNATSTopology(t.Context(), js, NATSTopology{Prefix: "Bad.Prefix"}, nil) + require.Error(t, err) + _, err = verifyNATSTopology(t.Context(), js, NATSTopology{Partitions: -1}, nil) + require.Error(t, err) +} + +// The shipped Helm values give the wavehouse user exactly natsPermissions. +func TestNATSPermissions_MatchShippedValues(t *testing.T) { + t.Parallel() + raw, err := os.ReadFile(natstest.ShippedValues()) + require.NoError(t, err) + var values struct { + Config struct { + Merge struct { + Accounts map[string]struct { + Users []struct { + User string `yaml:"user"` + Permissions struct { + Publish struct{ Allow, Deny []string } `yaml:"publish"` + Subscribe struct{ Allow []string } `yaml:"subscribe"` + } `yaml:"permissions"` + } `yaml:"users"` + } `yaml:"accounts"` + } `yaml:"merge"` + } `yaml:"config"` + } + require.NoError(t, yaml.Unmarshal(raw, &values)) + want := natsPermissions(NATSTopology{Shards: 8}, "") + found := false + for _, acc := range values.Config.Merge.Accounts { + for _, u := range acc.Users { + if u.User != "wavehouse" { + continue + } + found = true + assert.Equal(t, want.PublishAllow, u.Permissions.Publish.Allow) + assert.Equal(t, want.PublishDeny, u.Permissions.Publish.Deny) + assert.Equal(t, want.SubscribeAllow, u.Permissions.Subscribe.Allow) + } + } + assert.True(t, found, "no wavehouse user in %s", natstest.ShippedValues()) +} + +// The generated manifests round-trip through the fixture's parser into the +// configs the verifier accepts, at any N and prefix. +func TestWriteNATSManifests_RoundTrip(t *testing.T) { + t.Parallel() + spec := NATSTopology{Prefix: "acme-wh", Partitions: 3} + path := t.TempDir() + "/m.yaml" + out, err := os.Create(path) //nolint:gosec // G304: path is rooted in t.TempDir() + require.NoError(t, err) + require.NoError(t, WriteNATSManifests(out, NATSManifestOptions{Topology: spec, Replicas: 1})) + require.NoError(t, out.Close()) + + f := newNATSFixture(t) + f.apply(t, loadNATSManifests(t, path)) + spec.CoordBucket = DefaultNATSCoordBucket(spec.Prefix) + findings, err := verifyNATSTopology(t.Context(), f.admin, spec, nil) + require.NoError(t, err) + for _, got := range findings { + assert.Equal(t, "num_replicas", got.Field, "unexpected finding %v", got) + } +} + +func TestWriteNATSManifests_RefusesAnImpossibleSpec(t *testing.T) { + t.Parallel() + var b strings.Builder + require.Error(t, WriteNATSManifests(&b, NATSManifestOptions{Topology: NATSTopology{Prefix: "a.b"}})) + require.Error(t, WriteNATSManifests(&b, NATSManifestOptions{Topology: NATSTopology{Partitions: -2}})) + require.Error(t, WriteNATSManifests(&b, NATSManifestOptions{Topology: NATSTopology{CoordBucket: "a.b"}})) +} + +// smallTopology is `wavehouse mq manifests --partitions 4 --shards 2` at +// one replica. +func smallTopology(t *testing.T) *fixtureTopology { + t.Helper() + var buf bytes.Buffer + require.NoError(t, WriteNATSManifests(&buf, NATSManifestOptions{Topology: NATSTopology{Partitions: 4, Shards: 2}, Replicas: 1})) + path := filepath.Join(t.TempDir(), "manifests.yaml") + require.NoError(t, os.WriteFile(path, buf.Bytes(), 0o600)) + return loadNATSManifests(t, path) +} diff --git a/internal/mq/natsspike/pinned_test.go b/internal/mq/natsspike/pinned_test.go new file mode 100644 index 000000000..ff366c43b --- /dev/null +++ b/internal/mq/natsspike/pinned_test.go @@ -0,0 +1,391 @@ +//go:build integration + +package natsspike + +import ( + "context" + "fmt" + "os" + "path/filepath" + "sync" + "sync/atomic" + "testing" + "time" + + natsserver "github.com/nats-io/nats-server/v2/server" + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// The tests in this file pin how a shard durable with the pinned_client +// priority policy behaves, which the shard ownership rests on: one puller at +// a time receives, a pin lapses on its own once its holder stops pulling, +// UNPIN hands it over at once, and RESET redelivers a dead holder's unacked +// rows without replaying acked ones — but also the ones the caller holds. + +const group = "g" + +// pinnedShard is one work-queue partition with one pinned-client durable, +// "s0", on its only shard. +func pinnedShard(t *testing.T, ttl, ackWait time.Duration) (*natsserver.Server, jetstream.JetStream) { + t.Helper() + s := server(t) + js := connect(t, s) + _, err := js.CreateStream(t.Context(), jetstream.StreamConfig{Name: "IN", Subjects: []string{"wh.ingest.>"}, Retention: jetstream.WorkQueuePolicy}) + require.NoError(t, err) + _, err = js.CreateConsumer(t.Context(), "IN", jetstream.ConsumerConfig{ + Durable: "s0", AckPolicy: jetstream.AckExplicitPolicy, AckWait: ackWait, MaxDeliver: -1, + FilterSubject: "wh.ingest.0.0.>", + PriorityPolicy: jetstream.PriorityPolicyPinned, PinnedTTL: ttl, PriorityGroups: []string{group}, + }) + require.NoError(t, err) + return s, js +} + +type delivery struct { + worker string + seq uint64 + pin string + at time.Time +} + +type recorder struct { + mu sync.Mutex + all []delivery +} + +func (r *recorder) add(d delivery) { r.mu.Lock(); r.all = append(r.all, d); r.mu.Unlock() } + +func (r *recorder) since(from time.Time) map[string]int { + r.mu.Lock() + defer r.mu.Unlock() + m := map[string]int{} + for _, d := range r.all { + if !d.at.Before(from) { + m[d.worker]++ + } + } + return m +} + +func (r *recorder) of(worker string) []delivery { + r.mu.Lock() + defer r.mu.Unlock() + var out []delivery + for _, d := range r.all { + if d.worker == worker { + out = append(out, d) + } + } + return out +} + +// puller is a process pulling s0 in the group on a connection of its own. +type puller struct { + nc *nats.Conn + cc jetstream.ConsumeContext + hold atomic.Bool // leave rows unacked +} + +func pull(t *testing.T, s *natsserver.Server, name string, rec *recorder, expiry time.Duration) *puller { + t.Helper() + nc, err := nats.Connect(s.ClientURL(), nats.Name(name)) + require.NoError(t, err) + t.Cleanup(nc.Close) + js, err := jetstream.New(nc) + require.NoError(t, err) + c, err := js.Consumer(t.Context(), "IN", "s0") + require.NoError(t, err) + p := &puller{nc: nc} + p.cc, err = c.Consume(func(m jetstream.Msg) { + md, _ := m.Metadata() + rec.add(delivery{worker: name, seq: md.Sequence.Stream, pin: m.Headers().Get("Nats-Pin-Id"), at: time.Now()}) + if !p.hold.Load() { + _ = m.Ack() + } + }, jetstream.PullPriorityGroup(group), jetstream.PullExpiry(expiry), + jetstream.ConsumeErrHandler(func(jetstream.ConsumeContext, error) {})) + require.NoError(t, err) + return p +} + +func publishEvery(t *testing.T, js jetstream.JetStream, every time.Duration) (stop func()) { + t.Helper() + done := make(chan struct{}) + var wg sync.WaitGroup + wg.Go(func() { + tk := time.NewTicker(every) + defer tk.Stop() + for i := 0; ; i++ { + select { + case <-done: + return + case <-tk.C: + _, _ = js.Publish(context.Background(), "wh.ingest.0.0.acme.t1", fmt.Appendf(nil, "%d", i)) + } + } + }) + return func() { close(done); wg.Wait() } +} + +func pinned(t *testing.T, js jetstream.JetStream) string { + t.Helper() + c, err := js.Consumer(t.Context(), "IN", "s0") + require.NoError(t, err) + for _, g := range c.CachedInfo().PriorityGroups { + if g.Group == group { + return g.PinnedClientID + } + } + return "" +} + +// Two processes pull one shard; only the first one pinned receives. The pin +// id the server reports is the one its rows carry. +func TestS2_PinnedOneReceiverAtATime(t *testing.T) { + t.Parallel() + s, js := pinnedShard(t, 10*time.Second, time.Minute) + rec := &recorder{} + pull(t, s, "A", rec, time.Second) + time.Sleep(100 * time.Millisecond) // A's pull arrives first + pull(t, s, "B", rec, time.Second) + stop := publishEvery(t, js, 10*time.Millisecond) + time.Sleep(time.Second) + stop() + time.Sleep(100 * time.Millisecond) + got := rec.since(time.Time{}) + assert.Positive(t, got["A"]) + assert.Zero(t, got["B"]) + a := rec.of("A") + require.NotEmpty(t, a) + assert.NotEmpty(t, a[0].pin, "a pinned delivery carries the pin id") + assert.Equal(t, a[0].pin, pinned(t, js), "the consumer info names the same pin") +} + +// A holder whose connection dies stops pulling, and its pin lapses within +// the TTL of its last pull; the other puller receives from then on. Rows the +// dead holder held unacked come back only after ack_wait. +func TestS2_PinLapsesWithinTTLOfADeadHolder(t *testing.T) { + t.Parallel() + const ttl, ackWait = 2 * time.Second, 5 * time.Second + s, js := pinnedShard(t, ttl, ackWait) + rec := &recorder{} + a := pull(t, s, "A", rec, time.Second) + time.Sleep(100 * time.Millisecond) + pull(t, s, "B", rec, time.Second) + stop := publishEvery(t, js, 20*time.Millisecond) + defer stop() + time.Sleep(500 * time.Millisecond) + a.hold.Store(true) + time.Sleep(300 * time.Millisecond) + a.nc.Close() + killed := time.Now() + require.Eventually(t, func() bool { return rec.since(killed)["B"] > 0 }, ttl+2*time.Second, 20*time.Millisecond) + first := rec.of("B")[0].at.Sub(killed) + t.Logf("B's first row %s after A died (pinned ttl %s)", first.Round(10*time.Millisecond), ttl) + assert.Less(t, first, ttl+time.Second) +} + +// UNPIN by the holder, after it stopped pulling, hands the shard to the next +// puller at once, well before the TTL. +func TestS2_UnpinHandsOverAtOnce(t *testing.T) { + t.Parallel() + const ttl = 30 * time.Second + s, js := pinnedShard(t, ttl, time.Minute) + rec := &recorder{} + a := pull(t, s, "A", rec, time.Second) + time.Sleep(100 * time.Millisecond) + pull(t, s, "B", rec, time.Second) + stop := publishEvery(t, js, 20*time.Millisecond) + defer stop() + time.Sleep(500 * time.Millisecond) + a.cc.Drain() + <-a.cc.Closed() + st, err := js.Stream(t.Context(), "IN") + require.NoError(t, err) + unpinned := time.Now() + require.NoError(t, st.UnpinConsumer(t.Context(), "s0", group)) + require.Eventually(t, func() bool { return rec.since(unpinned)["B"] > 0 }, 5*time.Second, 10*time.Millisecond) + took := rec.of("B")[0].at.Sub(unpinned) + t.Logf("B's first row %s after the unpin (pinned ttl %s)", took.Round(time.Millisecond), ttl) + assert.Less(t, took, 2*time.Second) + assert.Zero(t, rec.since(time.Time{})["B"]-rec.since(unpinned)["B"], "nothing reached B before the unpin") +} + +// A holder that stops pulling past the TTL is fenced: once another puller +// holds the pin, the old holder gets nothing though it pulls again. +func TestS2_StalledHolderIsFenced(t *testing.T) { + t.Parallel() + s, js := pinnedShard(t, 2*time.Second, time.Minute) + rec := &recorder{} + a := pull(t, s, "A", rec, time.Second) + time.Sleep(100 * time.Millisecond) + pull(t, s, "B", rec, time.Second) + stop := publishEvery(t, js, 20*time.Millisecond) + defer stop() + time.Sleep(500 * time.Millisecond) + a.cc.Stop() + <-a.cc.Closed() + time.Sleep(3 * time.Second) // past the TTL: B holds the pin + resumed := time.Now() + pull(t, s, "A", rec, time.Second) // the same process pulls again + time.Sleep(2 * time.Second) + got := rec.since(resumed) + assert.Positive(t, got["B"]) + assert.Zero(t, got["A"], "the stalled holder is fenced") +} + +// A live idle holder keeps its pin only while its pull expiry is under the +// TTL: the server renews a pin on a new pull, not on acks or heartbeats. +func TestS2_ExpiryOverTTLLosesAnIdlePin(t *testing.T) { + t.Parallel() + for _, tc := range []struct { + expiry time.Duration + flaps bool + }{{time.Second, false}, {10 * time.Second, true}} { + t.Run(tc.expiry.String(), func(t *testing.T) { + t.Parallel() + s, js := pinnedShard(t, 2*time.Second, time.Minute) + rec := &recorder{} + pull(t, s, "A", rec, tc.expiry) + time.Sleep(100 * time.Millisecond) + pull(t, s, "B", rec, tc.expiry) + var owners []string + for range 3 { + from := time.Now() + _, err := js.Publish(t.Context(), "wh.ingest.0.0.acme.t1", []byte("x")) + require.NoError(t, err) + require.Eventually(t, func() bool { return len(rec.since(from)) == 1 }, 5*time.Second, 10*time.Millisecond) + for w := range rec.since(from) { + owners = append(owners, w) + } + time.Sleep(3 * time.Second) // idle past the TTL + } + t.Logf("expiry %s: owner of each row after an idle TTL: %v", tc.expiry, owners) + if tc.flaps { + assert.NotEqual(t, []string{"A", "A", "A"}, owners, "the pin moves while its holder is idle") + } else { + assert.Equal(t, []string{"A", "A", "A"}, owners) + } + }) + } +} + +// RESET to the ack floor redelivers a dead holder's unacked rows at once and +// replays none of its acked ones. +func TestS2_ResetRedeliversOnlyTheUnacked(t *testing.T) { + t.Parallel() + s, js := pinnedShard(t, 2*time.Second, time.Minute) + rec := &recorder{} + a := pull(t, s, "A", rec, time.Second) + for range 10 { + _, err := js.Publish(t.Context(), "wh.ingest.0.0.acme.t1", []byte("acked")) + require.NoError(t, err) + } + require.Eventually(t, func() bool { return len(rec.of("A")) == 10 }, 5*time.Second, 10*time.Millisecond) + a.hold.Store(true) + for range 5 { + _, err := js.Publish(t.Context(), "wh.ingest.0.0.acme.t1", []byte("held")) + require.NoError(t, err) + } + require.Eventually(t, func() bool { return len(rec.of("A")) == 15 }, 5*time.Second, 10*time.Millisecond) + a.nc.Close() + require.Eventually(t, func() bool { return pinned(t, js) == "" }, 5*time.Second, 50*time.Millisecond, "the dead holder's pin lapses") + + // Reset first, then pull: the new holder receives the five at once. + _, err := js.ResetConsumer(t.Context(), "IN", "s0") + require.NoError(t, err) + reset := time.Now() + pull(t, s, "B", rec, time.Second) + require.Eventually(t, func() bool { return len(rec.of("B")) == 5 }, 2*time.Second, 10*time.Millisecond, "well before ack_wait (1m)") + time.Sleep(300 * time.Millisecond) + held := map[uint64]bool{} + for _, d := range rec.of("A")[10:] { + held[d.seq] = true + } + for _, d := range rec.of("B") { + assert.True(t, held[d.seq], "row %d was acked by A and replayed", d.seq) + } + t.Logf("5 held rows redelivered within %s of the reset", rec.of("B")[4].at.Sub(reset).Round(time.Millisecond)) +} + +// A reset also redelivers what the caller itself holds unacked, and leaves +// the pin in place: why WaveHouse resets a shard before it starts pulling, +// never while it holds rows of it. +func TestS2_ResetRedeliversTheCallersOwnRows(t *testing.T) { + t.Parallel() + s, js := pinnedShard(t, 10*time.Second, time.Minute) + rec := &recorder{} + b := pull(t, s, "B", rec, time.Second) + b.hold.Store(true) + for range 3 { + _, err := js.Publish(t.Context(), "wh.ingest.0.0.acme.t1", []byte("x")) + require.NoError(t, err) + } + require.Eventually(t, func() bool { return len(rec.of("B")) == 3 }, 5*time.Second, 10*time.Millisecond) + pin := pinned(t, js) + require.NotEmpty(t, pin) + _, err := js.ResetConsumer(t.Context(), "IN", "s0") + require.NoError(t, err) + require.Eventually(t, func() bool { return len(rec.of("B")) == 6 }, 5*time.Second, 10*time.Millisecond, "B receives its own three again") + assert.Equal(t, pin, pinned(t, js), "the reset leaves the pin") +} + +// A NATS permission wildcard is a whole token: wh-ingest-* names no durable, +// so the per-shard durables are listed one by one. +func TestS2_PartialTokenPermissionIsLiteral(t *testing.T) { + t.Parallel() + dir := t.TempDir() + conf := filepath.Join(dir, "nats.conf") + require.NoError(t, os.WriteFile(conf, fmt.Appendf(nil, ` +jetstream { store_dir: %q } +accounts { A { jetstream: enabled, users: [ + {user: admin, password: a}, + {user: partial, password: p, permissions: {publish: {allow: ["$JS.API.INFO", "$JS.API.CONSUMER.INFO.>", "$JS.API.CONSUMER.MSG.NEXT.IN.wh-ingest-*"]}, subscribe: {allow: ["_INBOX.>"]}}}, + {user: exact, password: e, permissions: {publish: {allow: ["$JS.API.INFO", "$JS.API.CONSUMER.INFO.>", "$JS.API.CONSUMER.MSG.NEXT.*.wh-ingest-0"]}, subscribe: {allow: ["_INBOX.>"]}}} +]}} +`, filepath.Join(dir, "js")), 0o600)) + opts, err := natsserver.ProcessConfigFile(conf) + require.NoError(t, err) + opts.Host, opts.Port, opts.NoSigs, opts.NoLog = "127.0.0.1", -1, true, true + s, err := natsserver.NewServer(opts) + require.NoError(t, err) + s.Start() + t.Cleanup(s.Shutdown) + require.True(t, s.ReadyForConnections(10*time.Second)) + + as := func(user, pw string) jetstream.JetStream { + nc, err := nats.Connect(s.ClientURL(), nats.UserInfo(user, pw), nats.ErrorHandler(func(*nats.Conn, *nats.Subscription, error) {})) + require.NoError(t, err) + t.Cleanup(nc.Close) + js, err := jetstream.New(nc) + require.NoError(t, err) + return js + } + admin := as("admin", "a") + _, err = admin.CreateStream(t.Context(), jetstream.StreamConfig{Name: "IN", Subjects: []string{"wh.ingest.>"}, Retention: jetstream.WorkQueuePolicy}) + require.NoError(t, err) + _, err = admin.CreateConsumer(t.Context(), "IN", jetstream.ConsumerConfig{Durable: "wh-ingest-0", AckPolicy: jetstream.AckExplicitPolicy}) + require.NoError(t, err) + _, err = admin.Publish(t.Context(), "wh.ingest.0.0.acme.t1", []byte("x")) + require.NoError(t, err) + + fetch := func(js jetstream.JetStream) int { + c, err := js.Consumer(t.Context(), "IN", "wh-ingest-0") + require.NoError(t, err) + b, err := c.Fetch(1, jetstream.FetchMaxWait(time.Second)) + if err != nil { + return 0 + } + n := 0 + for m := range b.Messages() { + n++ + _ = m.Nak() + } + return n + } + assert.Zero(t, fetch(as("partial", "p")), "wh-ingest-* does not match wh-ingest-0") + assert.Equal(t, 1, fetch(as("exact", "e"))) +} diff --git a/internal/mq/natsspike/workqueue_test.go b/internal/mq/natsspike/workqueue_test.go new file mode 100644 index 000000000..3cf9a2037 --- /dev/null +++ b/internal/mq/natsspike/workqueue_test.go @@ -0,0 +1,249 @@ +//go:build integration + +// Package natsspike pins the nats-server behavior the external-NATS topology +// rests on (#613). It runs in `make test-integration` (seconds per test, more +// than the unit suite's per-package timeout spares), and lives under +// internal/mq because only internal/mq may import NATS. +package natsspike + +import ( + "fmt" + "testing" + "time" + + natsserver "github.com/nats-io/nats-server/v2/server" + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// The tests in this file pin why an ingest partition has work-queue +// retention and feeds the history by republish: a row no consumer covers is +// kept, not dropped; the server refuses a second consumer on a subject; and a +// history that is gone or full never holds up ingest. + +// server runs an in-process JetStream server on a random TCP port, shut +// down by the test framework. +func server(t *testing.T) *natsserver.Server { + t.Helper() + s, err := natsserver.NewServer(&natsserver.Options{ + Host: "127.0.0.1", Port: -1, JetStream: true, StoreDir: t.TempDir(), NoSigs: true, NoLog: true, + }) + require.NoError(t, err) + s.Start() + require.True(t, s.ReadyForConnections(10*time.Second), "server not ready") + t.Cleanup(s.Shutdown) + return s +} + +func connect(t *testing.T, s *natsserver.Server) jetstream.JetStream { + t.Helper() + nc, err := nats.Connect(s.ClientURL()) + require.NoError(t, err) + t.Cleanup(nc.Close) + js, err := jetstream.New(nc) + require.NoError(t, err) + return js +} + +func stream(t *testing.T, js jetstream.JetStream, name string) jetstream.Stream { + t.Helper() + s, err := js.Stream(t.Context(), name) + require.NoError(t, err) + return s +} + +func msgs(t *testing.T, js jetstream.JetStream, name string) uint64 { + t.Helper() + info, err := stream(t, js, name).Info(t.Context()) + require.NoError(t, err) + return info.State.Msgs +} + +func subjectMsgs(t *testing.T, js jetstream.JetStream, name, filter string) uint64 { + t.Helper() + info, err := stream(t, js, name).Info(t.Context(), jetstream.WithSubjectFilter(filter)) + require.NoError(t, err) + var n uint64 + for _, c := range info.State.Subjects { + n += c + } + return n +} + +func publish(t *testing.T, js jetstream.JetStream, subject string, n int) { + t.Helper() + for i := range n { + _, err := js.Publish(t.Context(), subject, fmt.Appendf(nil, "%d", i)) + require.NoError(t, err, "publish %s", subject) + } +} + +func durable(t *testing.T, js jetstream.JetStream, streamName, name string, filters ...string) error { + t.Helper() + _, err := js.CreateOrUpdateConsumer(t.Context(), streamName, jetstream.ConsumerConfig{ + Durable: name, AckPolicy: jetstream.AckExplicitPolicy, MaxDeliver: -1, FilterSubjects: filters, + }) + return err +} + +// drain fetches and acks every row the durable delivers until a pull comes +// back empty. +func drain(t *testing.T, js jetstream.JetStream, streamName, name string) int { + t.Helper() + c, err := js.Consumer(t.Context(), streamName, name) + require.NoError(t, err) + n := 0 + for { + b, err := c.Fetch(100, jetstream.FetchMaxWait(300*time.Millisecond)) + require.NoError(t, err) + got := 0 + for m := range b.Messages() { + require.NoError(t, m.DoubleAck(t.Context())) + got++ + } + n += got + if got == 0 { + return n + } + } +} + +// A row whose subject no consumer covers: interest retention acks the publish +// and stores nothing; a work queue keeps it for a consumer to come. +func TestS2_WorkQueueKeepsARowNoConsumerCovers(t *testing.T) { + t.Parallel() + for _, tc := range []struct { + retention jetstream.RetentionPolicy + kept uint64 + }{{jetstream.InterestPolicy, 0}, {jetstream.WorkQueuePolicy, 5}} { + t.Run(tc.retention.String(), func(t *testing.T) { + t.Parallel() + js := connect(t, server(t)) + _, err := js.CreateStream(t.Context(), jetstream.StreamConfig{Name: "IN", Subjects: []string{"wh.ingest.0.>"}, Retention: tc.retention}) + require.NoError(t, err) + require.NoError(t, durable(t, js, "IN", "x", "wh.ingest.0.acme.t1")) + publish(t, js, "wh.ingest.0.acme.t2", 5) // acked either way + assert.Equal(t, tc.kept, subjectMsgs(t, js, "IN", "wh.ingest.0.acme.t2")) + }) + } +} + +// A work queue refuses a second consumer whose filter overlaps one it has, +// created or updated into the overlap, and a catch-all beside a filtered one; +// a consumer on a disjoint subject is allowed. +func TestS2_WorkQueueRefusesOverlap(t *testing.T) { + t.Parallel() + js := connect(t, server(t)) + _, err := js.CreateStream(t.Context(), jetstream.StreamConfig{Name: "IN", Subjects: []string{"wh.ingest.0.>"}, Retention: jetstream.WorkQueuePolicy}) + require.NoError(t, err) + require.NoError(t, durable(t, js, "IN", "s0", "wh.ingest.0.0.>")) + var apiErr *jetstream.APIError + require.ErrorAs(t, durable(t, js, "IN", "other", "wh.ingest.0.0.>"), &apiErr, "the same subject") + assert.Equal(t, jetstream.ErrorCode(10100), apiErr.ErrorCode) + require.Error(t, durable(t, js, "IN", "all", "wh.ingest.0.>"), "a catch-all") + require.NoError(t, durable(t, js, "IN", "s1", "wh.ingest.0.1.>"), "a disjoint subject") + require.Error(t, durable(t, js, "IN", "s0", "wh.ingest.0.0.>", "wh.ingest.0.1.>"), "an update into the overlap") +} + +// republishSetup is one work-queue partition of 200 rows republishing to a +// history, with a durable on the partition. +func republishSetup(t *testing.T, history jetstream.StreamConfig) jetstream.JetStream { + t.Helper() + js := connect(t, server(t)) + _, err := js.CreateStream(t.Context(), jetstream.StreamConfig{ + Name: "WH_INGEST_0", Subjects: []string{"wh.ingest.0.>"}, Retention: jetstream.WorkQueuePolicy, + MaxMsgs: 200, Discard: jetstream.DiscardNew, + RePublish: &jetstream.RePublish{Source: "wh.ingest.0.>", Destination: "wh.hist.>"}, + }) + require.NoError(t, err) + require.NoError(t, durable(t, js, "WH_INGEST_0", "wh-ingest", "wh.ingest.0.>")) + history.Name, history.Subjects = "WH_HISTORY", []string{"wh.hist.>"} + _, err = js.CreateStream(t.Context(), history) + require.NoError(t, err) + return js +} + +// The history gets every row the partition stores, under the subject with +// the partition token dropped, and the ack empties the partition only. +func TestS2_RepublishFeedsTheHistory(t *testing.T) { + t.Parallel() + js := republishSetup(t, jetstream.StreamConfig{MaxAge: time.Hour}) + publish(t, js, "wh.ingest.0.acme.orders", 30) + publish(t, js, "wh.ingest.0.acme.orders.scope1", 20) + assert.Equal(t, 50, drain(t, js, "WH_INGEST_0", "wh-ingest")) + assert.Zero(t, msgs(t, js, "WH_INGEST_0"), "acked rows leave the partition at once") + require.Eventually(t, func() bool { return msgs(t, js, "WH_HISTORY") == 50 }, 5*time.Second, 10*time.Millisecond) + assert.Equal(t, uint64(30), subjectMsgs(t, js, "WH_HISTORY", "wh.hist.acme.orders")) + assert.Equal(t, uint64(20), subjectMsgs(t, js, "WH_HISTORY", "wh.hist.acme.orders.scope1")) +} + +// A history that is deleted, or full and refusing, never holds up ingest: +// every publish is stored and every acked row leaves the partition. +func TestS2_HistoryOutageNeverHoldsIngest(t *testing.T) { + t.Parallel() + t.Run("deleted", func(t *testing.T) { + t.Parallel() + js := republishSetup(t, jetstream.StreamConfig{MaxAge: time.Hour}) + require.NoError(t, js.DeleteStream(t.Context(), "WH_HISTORY")) + publish(t, js, "wh.ingest.0.acme.t1", 50) + assert.Equal(t, 50, drain(t, js, "WH_INGEST_0", "wh-ingest")) + assert.Zero(t, msgs(t, js, "WH_INGEST_0")) + }) + t.Run("full", func(t *testing.T) { + t.Parallel() + js := republishSetup(t, jetstream.StreamConfig{MaxMsgs: 10, Discard: jetstream.DiscardNew}) + for range 20 { // 400 rows through a 200-row partition, beside a 10-row history + publish(t, js, "wh.ingest.0.acme.t1", 20) + drain(t, js, "WH_INGEST_0", "wh-ingest") + } + assert.Zero(t, msgs(t, js, "WH_INGEST_0")) + assert.Equal(t, uint64(10), msgs(t, js, "WH_HISTORY")) + }) +} + +// The republished copy carries the publish's headers. A publish that expects +// its stream by name therefore reaches the history expecting the partition, +// and the history refuses it: the reason ExternalNATS publishes ingest rows +// without that expectation and checks the ack's stream instead. +func TestS2_RepublishCarriesTheExpectedStream(t *testing.T) { + t.Parallel() + js := republishSetup(t, jetstream.StreamConfig{MaxAge: time.Hour}) + _, err := js.Publish(t.Context(), "wh.ingest.0.acme.a", []byte("x"), jetstream.WithExpectStream("WH_INGEST_0")) + require.NoError(t, err) + ack, err := js.Publish(t.Context(), "wh.ingest.0.acme.b", []byte("x"), jetstream.WithMsgID("id-1")) + require.NoError(t, err) + assert.Equal(t, "WH_INGEST_0", ack.Stream, "the ack names the stream that stored it") + require.Eventually(t, func() bool { return msgs(t, js, "WH_HISTORY") == 1 }, 5*time.Second, 10*time.Millisecond) + time.Sleep(200 * time.Millisecond) + assert.Zero(t, subjectMsgs(t, js, "WH_HISTORY", "wh.hist.acme.a"), "the copy expecting WH_INGEST_0 is refused") + m, err := stream(t, js, "WH_HISTORY").GetLastMsgForSubject(t.Context(), "wh.hist.acme.b") + require.NoError(t, err) + assert.Equal(t, "id-1", m.Header.Get(jetstream.MsgIDHeader), "the copy keeps the publish's headers") + assert.Equal(t, "WH_INGEST_0", m.Header.Get("Nats-Stream")) +} + +// A republish destination the stream's own subjects would capture is a +// cycle, refused at create. +func TestS2_RepublishRefusesACycle(t *testing.T) { + t.Parallel() + js := connect(t, server(t)) + _, err := js.CreateStream(t.Context(), jetstream.StreamConfig{ + Name: "BAD", Subjects: []string{"x.>"}, RePublish: &jetstream.RePublish{Source: "x.>", Destination: "x.y.>"}, + }) + require.Error(t, err) +} + +// A work-queue partition keeps its last row's timestamp after the row is +// acked away, which the history gauge compares against. +func TestS2_WorkQueueKeepsTheLastTimestamp(t *testing.T) { + t.Parallel() + js := republishSetup(t, jetstream.StreamConfig{MaxAge: time.Hour}) + publish(t, js, "wh.ingest.0.acme.t1", 3) + require.Equal(t, 3, drain(t, js, "WH_INGEST_0", "wh-ingest")) + info, err := stream(t, js, "WH_INGEST_0").Info(t.Context()) + require.NoError(t, err) + assert.Zero(t, info.State.Msgs) + assert.WithinDuration(t, time.Now(), info.State.LastTime, 10*time.Second) +} diff --git a/internal/mq/natstest/natstest.go b/internal/mq/natstest/natstest.go new file mode 100644 index 000000000..448580022 --- /dev/null +++ b/internal/mq/natstest/natstest.go @@ -0,0 +1,556 @@ +// Package natstest stands up NATS the way an operator deploys it for +// mq.backend: nats: the shipped Helm values' accounts, users and permissions +// (deployments/nats/values.yaml) and the shipped nack manifests +// (deployments/nats/jetstream.yaml). internal/mq's own fixture builds on it, +// and so do tests outside internal/mq, which may not import NATS themselves +// (depguard's mq boundary): they get a server URL and the two users' +// passwords, and act on the topology only through this package. +// +// It is test code that lives outside *_test.go so those tests can import it, +// like mqtest; nothing in the binary imports it. +package natstest + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "regexp" + "runtime" + "testing" + "time" + + natsserver "github.com/nats-io/nats-server/v2/server" + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + "gopkg.in/yaml.v3" +) + +// The shipped values' two users: nack, the operator's JetStream controller +// with full access, and wavehouse, WaveHouse with exactly the permissions it +// needs. +const ( + OperatorUser = "nack" + WaveHouseUser = "wavehouse" +) + +// Password is the password every user gets in place of the Helm values' +// Secret reference. +func Password(user string) string { return "pw-" + user } + +// repoFile is path under the repository root. +func repoFile(path string) string { + _, file, _, _ := runtime.Caller(0) + return filepath.Join(filepath.Dir(file), "..", "..", "..", path) +} + +// ShippedValues and ShippedManifests are the files an operator deploys. +func ShippedValues() string { return repoFile("deployments/nats/values.yaml") } +func ShippedManifests() string { return repoFile("deployments/nats/jetstream.yaml") } + +// helmVariable matches the chart's `<< $VAR >>` unquoted config variable. +var helmVariable = regexp.MustCompile(`^<< *\$[A-Za-z0-9_]+ *>>$`) + +// ServerConfig renders the Helm values' config.merge block as a nats.conf +// (the chart writes it as JSON too), each Secret-referenced password set to +// Password(user), and JetStream on file storage under storeDir with the +// max_file_store the chart would set (FileStore). A server reserves what its +// streams' max_bytes add up to against that, without writing it, so the +// manifests that do not fit the shipped volume do not fit here either. +func ServerConfig(valuesPath, storeDir string) ([]byte, error) { + raw, err := os.ReadFile(valuesPath) //nolint:gosec // G304: a shipped file or one a test wrote + if err != nil { + return nil, err + } + var values chartValues + if err := yaml.Unmarshal(raw, &values); err != nil { + return nil, fmt.Errorf("%s: %w", valuesPath, err) + } + store, err := values.fileStore() + if err != nil { + return nil, fmt.Errorf("%s: %w", valuesPath, err) + } + merge := values.Config.Merge + accounts, _ := merge["accounts"].(map[string]any) + if len(accounts) == 0 { + return nil, fmt.Errorf("%s: config.merge has no accounts", valuesPath) + } + for _, acc := range accounts { + users, _ := acc.(map[string]any)["users"].([]any) + for _, u := range users { + user, _ := u.(map[string]any) + if pw, _ := user["password"].(string); helmVariable.MatchString(pw) { + name, _ := user["user"].(string) + user["password"] = Password(name) + } + } + } + conf := map[string]any{"jetstream": map[string]any{ + "store_dir": storeDir, "max_file_store": fileStorePlaceholder, "max_memory_store": int64(1) << 40, + }} + for k, v := range merge { + conf[k] = v + } + // NATS config strings take no \u escapes, which json.Marshal writes for + // the '>' of every wildcard. + var buf bytes.Buffer + enc := json.NewEncoder(&buf) + enc.SetEscapeHTML(false) + if err := enc.Encode(conf); err != nil { + return nil, err + } + // Unquoted, as the chart writes it, so the server reads the size's suffix. + return bytes.Replace(buf.Bytes(), []byte(`"`+fileStorePlaceholder+`"`), []byte(store), 1), nil +} + +const fileStorePlaceholder = "@max_file_store@" + +// chartValues is what ServerConfig reads of a NATS Helm values file. +type chartValues struct { + Config struct { + JetStream struct { + FileStore struct { + MaxSize string `yaml:"maxSize"` + PVC struct { + Size string `yaml:"size"` + } `yaml:"pvc"` + } `yaml:"fileStore"` + } `yaml:"jetstream"` + Merge map[string]any `yaml:"merge"` + } `yaml:"config"` +} + +// storeSize is a Kubernetes quantity the NATS config parser reads the same. +var storeSize = regexp.MustCompile(`^[0-9]+(|k|M|G|T|Ki|Mi|Gi|Ti)$`) + +// fileStore is the max_file_store the chart renders: fileStore.maxSize, else +// the PVC's size. +func (v chartValues) fileStore() (string, error) { + fs := v.Config.JetStream.FileStore + size := fs.MaxSize + if size == "" { + size = fs.PVC.Size + } + if !storeSize.MatchString(size) { + return "", fmt.Errorf("config.jetstream.fileStore: size %q is not a quantity the test server reads", size) + } + return size, nil +} + +// FileStore is the max_file_store the NATS chart sets from the values at +// valuesPath, as written there (100Gi). +func FileStore(valuesPath string) (string, error) { + raw, err := os.ReadFile(valuesPath) //nolint:gosec // G304: a shipped file or one a test wrote + if err != nil { + return "", err + } + var values chartValues + if err := yaml.Unmarshal(raw, &values); err != nil { + return "", fmt.Errorf("%s: %w", valuesPath, err) + } + return values.fileStore() +} + +// Manifests is a set of nack Stream, Consumer and KeyValue resources as the +// JetStream configs nack would create from them, in manifest order. +type Manifests struct { + Streams []jetstream.StreamConfig + Consumers map[string][]jetstream.ConsumerConfig // by stream name + KeyValues []jetstream.KeyValueConfig +} + +// The nack (jetstream.nats.io/v1beta2) fields the shipped manifests use. +// Decoding is strict, so a field the generator starts writing fails here +// rather than being dropped from every fixture. +type nackStream struct { + Name string `yaml:"name"` + Subjects []string `yaml:"subjects"` + Retention string `yaml:"retention"` + Discard string `yaml:"discard"` + DiscardPerSubject bool `yaml:"discardPerSubject"` + MaxBytes int64 `yaml:"maxBytes"` + MaxAge string `yaml:"maxAge"` + MaxMsgsPerSubject int64 `yaml:"maxMsgsPerSubject"` + Storage string `yaml:"storage"` + Replicas int `yaml:"replicas"` + DuplicateWindow string `yaml:"duplicateWindow"` + DenyPurge bool `yaml:"denyPurge"` + DenyDelete bool `yaml:"denyDelete"` + Metadata map[string]string `yaml:"metadata"` + PreventDelete bool `yaml:"preventDelete"` + Republish *struct { + Source string `yaml:"source"` + Destination string `yaml:"destination"` + } `yaml:"republish"` +} + +type nackConsumer struct { + PriorityPolicy string `yaml:"priorityPolicy"` + PriorityGroups []string `yaml:"priorityGroups"` + PinnedTTL string `yaml:"pinnedTtl"` + StreamName string `yaml:"streamName"` + DurableName string `yaml:"durableName"` + DeliverPolicy string `yaml:"deliverPolicy"` + AckPolicy string `yaml:"ackPolicy"` + AckWait string `yaml:"ackWait"` + MaxDeliver int `yaml:"maxDeliver"` + MaxAckPending int `yaml:"maxAckPending"` + FilterSubject string `yaml:"filterSubject"` + PreventDelete bool `yaml:"preventDelete"` +} + +type nackKeyValue struct { + Bucket string `yaml:"bucket"` + History int `yaml:"history"` + Storage string `yaml:"storage"` + Replicas int `yaml:"replicas"` +} + +// LoadManifests parses the nack resources at path. +func LoadManifests(path string) (*Manifests, error) { + f, err := os.Open(path) //nolint:gosec // G304: a shipped manifest or one a test wrote + if err != nil { + return nil, err + } + defer func() { _ = f.Close() }() + m := &Manifests{Consumers: map[string][]jetstream.ConsumerConfig{}} + dec := yaml.NewDecoder(f) + for { + var doc struct { + Kind string `yaml:"kind"` + Spec yaml.Node `yaml:"spec"` + } + if err := dec.Decode(&doc); err != nil { + if errors.Is(err, io.EOF) { + return m, nil + } + return nil, fmt.Errorf("%s: %w", path, err) + } + switch doc.Kind { + case "Stream": + var s nackStream + if err := decodeStrict(&doc.Spec, &s); err != nil { + return nil, fmt.Errorf("%s: stream: %w", path, err) + } + cfg, err := streamConfig(s) + if err != nil { + return nil, fmt.Errorf("%s: stream %s: %w", path, s.Name, err) + } + m.Streams = append(m.Streams, cfg) + case "Consumer": + var c nackConsumer + if err := decodeStrict(&doc.Spec, &c); err != nil { + return nil, fmt.Errorf("%s: consumer: %w", path, err) + } + cfg, err := consumerConfig(c) + if err != nil { + return nil, fmt.Errorf("%s: consumer %s/%s: %w", path, c.StreamName, c.DurableName, err) + } + m.Consumers[c.StreamName] = append(m.Consumers[c.StreamName], cfg) + case "KeyValue": + var kv nackKeyValue + if err := decodeStrict(&doc.Spec, &kv); err != nil { + return nil, fmt.Errorf("%s: keyvalue: %w", path, err) + } + storage, err := enum("storage", kv.Storage, map[string]jetstream.StorageType{ + "file": jetstream.FileStorage, "memory": jetstream.MemoryStorage, + }) + if err != nil { + return nil, fmt.Errorf("%s: keyvalue %s: %w", path, kv.Bucket, err) + } + m.KeyValues = append(m.KeyValues, jetstream.KeyValueConfig{ + Bucket: kv.Bucket, History: uint8(min(kv.History, 64)), Storage: storage, Replicas: kv.Replicas, //nolint:gosec // G115: clamped to JetStream's own cap + }) + default: + return nil, fmt.Errorf("%s: unexpected kind %q", path, doc.Kind) + } + } +} + +// decodeStrict decodes node into v, refusing a field v does not declare. +func decodeStrict(node *yaml.Node, v any) error { + raw, err := yaml.Marshal(node) + if err != nil { + return err + } + dec := yaml.NewDecoder(bytes.NewReader(raw)) + dec.KnownFields(true) + return dec.Decode(v) +} + +func duration(s string) (time.Duration, error) { + if s == "" { + return 0, nil + } + return time.ParseDuration(s) +} + +func enum[T any](field, value string, values map[string]T) (T, error) { + v, ok := values[value] + if !ok { + return v, fmt.Errorf("%s: unknown value %q", field, value) + } + return v, nil +} + +func streamConfig(s nackStream) (jetstream.StreamConfig, error) { + cfg := jetstream.StreamConfig{ + Name: s.Name, + Subjects: s.Subjects, + DiscardNewPerSubject: s.DiscardPerSubject, + MaxBytes: s.MaxBytes, + MaxMsgsPerSubject: s.MaxMsgsPerSubject, + Replicas: s.Replicas, + DenyPurge: s.DenyPurge, + DenyDelete: s.DenyDelete, + Metadata: s.Metadata, + } + var err error + var errs []error + cfg.Retention, err = enum("retention", s.Retention, map[string]jetstream.RetentionPolicy{ + "limits": jetstream.LimitsPolicy, "interest": jetstream.InterestPolicy, "workqueue": jetstream.WorkQueuePolicy, + }) + errs = append(errs, err) + cfg.Discard, err = enum("discard", s.Discard, map[string]jetstream.DiscardPolicy{ + "old": jetstream.DiscardOld, "new": jetstream.DiscardNew, + }) + errs = append(errs, err) + cfg.Storage, err = enum("storage", s.Storage, map[string]jetstream.StorageType{ + "file": jetstream.FileStorage, "memory": jetstream.MemoryStorage, + }) + errs = append(errs, err) + cfg.MaxAge, err = duration(s.MaxAge) + errs = append(errs, err) + cfg.Duplicates, err = duration(s.DuplicateWindow) + errs = append(errs, err) + if r := s.Republish; r != nil { + cfg.RePublish = &jetstream.RePublish{Source: r.Source, Destination: r.Destination} + } + return cfg, errors.Join(errs...) +} + +func consumerConfig(c nackConsumer) (jetstream.ConsumerConfig, error) { + cfg := jetstream.ConsumerConfig{ + Durable: c.DurableName, + MaxDeliver: c.MaxDeliver, + MaxAckPending: c.MaxAckPending, + FilterSubject: c.FilterSubject, + PriorityGroups: c.PriorityGroups, + } + var err error + var errs []error + cfg.PriorityPolicy, err = enum("priorityPolicy", c.PriorityPolicy, map[string]jetstream.PriorityPolicy{ + "": jetstream.PriorityPolicyNone, "none": jetstream.PriorityPolicyNone, "pinned_client": jetstream.PriorityPolicyPinned, + "overflow": jetstream.PriorityPolicyOverflow, "prioritized": jetstream.PriorityPolicyPrioritized, + }) + errs = append(errs, err) + cfg.PinnedTTL, err = duration(c.PinnedTTL) + errs = append(errs, err) + cfg.DeliverPolicy, err = enum("deliverPolicy", c.DeliverPolicy, map[string]jetstream.DeliverPolicy{ + "all": jetstream.DeliverAllPolicy, "last": jetstream.DeliverLastPolicy, "new": jetstream.DeliverNewPolicy, + }) + errs = append(errs, err) + cfg.AckPolicy, err = enum("ackPolicy", c.AckPolicy, map[string]jetstream.AckPolicy{ + "none": jetstream.AckNonePolicy, "all": jetstream.AckAllPolicy, "explicit": jetstream.AckExplicitPolicy, + }) + errs = append(errs, err) + cfg.AckWait, err = duration(c.AckWait) + errs = append(errs, err) + return cfg, errors.Join(errs...) +} + +// SingleReplica sets every stream to one replica, which is all a single +// server can hold. +func (m *Manifests) SingleReplica() { + for i := range m.Streams { + m.Streams[i].Replicas = 1 + } + for i := range m.KeyValues { + m.KeyValues[i].Replicas = 1 + } +} + +// Create creates m's streams and each one's consumers, in order, then its +// KV buckets the way nack does (CreateKeyValue), without waiting for anything. +func (m *Manifests) Create(ctx context.Context, js jetstream.JetStream) error { + for _, cfg := range m.Streams { + s, err := js.CreateStream(ctx, cfg) + if err != nil { + return fmt.Errorf("create stream %s: %w", cfg.Name, err) + } + for _, c := range m.Consumers[cfg.Name] { + if _, err := s.CreateConsumer(ctx, c); err != nil { + return fmt.Errorf("create consumer %s/%s: %w", cfg.Name, c.Durable, err) + } + } + } + for _, cfg := range m.KeyValues { + if _, err := js.CreateKeyValue(ctx, cfg); err != nil { + return fmt.Errorf("create kv bucket %s: %w", cfg.Bucket, err) + } + } + return nil +} + +// Apply creates or updates m's streams and each one's consumers, in order, +// as nack applying changed manifests would. What m leaves out is kept: the +// generated resources set preventDelete. +func (m *Manifests) Apply(ctx context.Context, js jetstream.JetStream) error { + for _, cfg := range m.Streams { + s, err := js.CreateOrUpdateStream(ctx, cfg) + if err != nil { + return fmt.Errorf("apply stream %s: %w", cfg.Name, err) + } + for _, c := range m.Consumers[cfg.Name] { + if _, err := s.CreateOrUpdateConsumer(ctx, c); err != nil { + return fmt.Errorf("apply consumer %s/%s: %w", cfg.Name, c.Durable, err) + } + } + } + return nil +} + +// Operator is the operator's hand on a running server: nack's user. +type Operator struct { + nc *nats.Conn + js jetstream.JetStream +} + +// Connect connects to url as OperatorUser. +func Connect(url string) (*Operator, error) { + nc, err := nats.Connect(url, nats.UserInfo(OperatorUser, Password(OperatorUser))) + if err != nil { + return nil, err + } + js, err := jetstream.New(nc) + if err != nil { + nc.Close() + return nil, err + } + return &Operator{nc: nc, js: js}, nil +} + +// JetStream is the operator's JetStream context. +func (o *Operator) JetStream() jetstream.JetStream { return o.js } + +// Close closes the connection. +func (o *Operator) Close() { o.nc.Close() } + +// ApplyShipped creates the shipped manifests at one replica, as nack would. +func (o *Operator) ApplyShipped(ctx context.Context) error { + m, err := LoadManifests(ShippedManifests()) + if err != nil { + return err + } + m.SingleReplica() + return m.Create(ctx, o.js) +} + +// DeleteIngestDurables deletes every shard durable of the shipped +// manifests, as an operator could while WaveHouse consumes them. +func (o *Operator) DeleteIngestDurables(ctx context.Context) error { + m, err := LoadManifests(ShippedManifests()) + if err != nil { + return err + } + for stream, consumers := range m.Consumers { + for _, c := range consumers { + if err := o.js.DeleteConsumer(ctx, stream, c.Durable); err != nil { + return fmt.Errorf("delete %s/%s: %w", stream, c.Durable, err) + } + } + } + return nil +} + +// CoordBucket is the lease bucket in the shipped manifests. +const CoordBucket = "wh_coord" + +// DeleteBucket deletes a KV bucket, as an operator could. +func (o *Operator) DeleteBucket(ctx context.Context, bucket string) error { + return o.js.DeleteKeyValue(ctx, bucket) +} + +// LeaseHolder is the holder the named lease's key in bucket names, "" when +// nobody holds it. +func (o *Operator) LeaseHolder(ctx context.Context, bucket, name string) (string, error) { + kv, err := o.js.KeyValue(ctx, bucket) + if err != nil { + return "", err + } + e, err := kv.Get(ctx, "lease."+name) + if errors.Is(err, jetstream.ErrKeyNotFound) { + return "", nil + } + if err != nil { + return "", err + } + var v struct { + Holder string `json:"holder"` + } + if err := json.Unmarshal(e.Value(), &v); err != nil { + return "", err + } + return v.Holder, nil +} + +// StreamMsgs is how many messages the named stream holds. +func (o *Operator) StreamMsgs(ctx context.Context, stream string) (uint64, error) { + s, err := o.js.Stream(ctx, stream) + if err != nil { + return 0, err + } + return s.CachedInfo().State.Msgs, nil +} + +// Server is an in-process NATS server configured from the shipped Helm +// values, listening on TCP, with the shipped manifests applied. +type Server struct { + s *natsserver.Server + // Operator is connected as OperatorUser, closed with the test. + Operator *Operator +} + +// Start starts a Server, shut down by the test framework. +func Start(t testing.TB) *Server { + t.Helper() + dir := t.TempDir() + conf, err := ServerConfig(ShippedValues(), dir) + if err != nil { + t.Fatal(err) + } + confPath := filepath.Join(dir, "nats.conf") + if err := os.WriteFile(confPath, conf, 0o600); err != nil { + t.Fatal(err) + } + opts, err := natsserver.ProcessConfigFile(confPath) + if err != nil { + t.Fatal(err) + } + opts.Host, opts.Port, opts.NoSigs, opts.NoLog = "127.0.0.1", -1, true, true + s, err := natsserver.NewServer(opts) + if err != nil { + t.Fatal(err) + } + s.Start() + t.Cleanup(s.Shutdown) + if !s.ReadyForConnections(10 * time.Second) { + t.Fatal("nats server not ready") + } + op, err := Connect(s.ClientURL()) + if err != nil { + t.Fatal(err) + } + t.Cleanup(op.Close) + if err := op.ApplyShipped(context.Background()); err != nil { + t.Fatal(err) + } + return &Server{s: s, Operator: op} +} + +// URL is the server's client URL. +func (s *Server) URL() string { return s.s.ClientURL() } diff --git a/internal/mq/settle.go b/internal/mq/settle.go new file mode 100644 index 000000000..d91868e05 --- /dev/null +++ b/internal/mq/settle.go @@ -0,0 +1,47 @@ +// Message.OnSettled, apart from mq.go so the e2e coverage gate, whose +// stack runs the embedded broker, can leave it to the suites that consume a +// sharded queue (.testcoverage.yml). + +package mq + +import ( + "context" + "sync" + "time" +) + +// OnSettled arranges for fn to run once, after the first DoubleAck, Ack, +// Nak or NakWithDelay, whether or not the broker confirms it: the consumer +// has let go of the message either way, and one it failed to settle comes +// back from the broker as a new delivery. A consumer handing a shard on to +// another process waits for it, and one that caps the messages it holds +// frees a slot. Call it before the message is shared with another goroutine. +func (m *Message) OnSettled(fn func()) { + var once sync.Once + settle := func(err error) error { + once.Do(fn) + return err + } + doubleAck, ack, nak, nakDelay := m.doubleAckFn, m.ackFn, m.nakFn, m.nakDelayFn + m.doubleAckFn = func(ctx context.Context) error { + if doubleAck == nil { + return settle(nil) + } + return settle(doubleAck(ctx)) + } + m.ackFn = func() error { + if ack == nil { + return settle(nil) + } + return settle(ack()) + } + m.nakFn = func() error { + if nak == nil { + return settle(nil) + } + return settle(nak()) + } + if nakDelay != nil { + m.nakDelayFn = func(d time.Duration) error { return settle(nakDelay(d)) } + } +} diff --git a/internal/mq/subject_nats.go b/internal/mq/subject_nats.go new file mode 100644 index 000000000..b0c17d191 --- /dev/null +++ b/internal/mq/subject_nats.go @@ -0,0 +1,162 @@ +package mq + +import ( + "fmt" + "hash/fnv" + "regexp" + "strconv" + "strings" + + "github.com/Wave-RF/WaveHouse/internal/keyenc" + "github.com/nats-io/nats.go/jetstream" +) + +// The external broker's subjects, on streams every tenant shares: +// +// .ingest.

...

[.] (p, s) = natsRoute(topic, N, V) +// .hist..
[.] republished by each partition +// .dlq..
[.] not partitioned (low volume) +// +// The tail after the shard (or after hist or dlq) is the topic key, the same +// one the embedded broker writes, so parking a message is still a prefix swap. + +// subjectPrefixPattern is the grammar of a subject prefix: one lowercase +// token, so it can neither split nor wildcard a subject, and uppercased it is +// still a valid stream name. +var subjectPrefixPattern = regexp.MustCompile(`^[a-z0-9_-]+$`) + +// validSubjectPrefix reports why prefix cannot lead the external subjects. +func validSubjectPrefix(prefix string) error { + if !subjectPrefixPattern.MatchString(prefix) { + return fmt.Errorf("subject prefix %q must be one token of [a-z0-9_-]", prefix) + } + return nil +} + +// natsRoute is where topic t's table lives among n partitions of v shards +// each: jump consistent hash (Lamping and Veach, 2014) of FNV-1a 64 over the +// tenant id and the table's subject token for the partition, and of the same +// hash through the splitmix64 finalizer for the shard. The second hash has +// to be independent of the first: with the same key, jump(h,2)=1 would force +// jump(h,4)≠0, leaving shards of some partitions empty. The scope is left +// out, so a table's scopes share one shard and one owner. Consistent: growing +// n (or v) by one moves about 1/(n+1) of the tables, each into the new +// partition (or shard), where a modulus would move nearly all. This is the +// one place a table is mapped. +func natsRoute(t Topic, n, v int) (p, s int) { + h := fnv.New64a() + _, _ = h.Write([]byte(t.Tenant)) + _, _ = h.Write([]byte{'.'}) + _, _ = h.Write([]byte(keyenc.Escape(t.Table))) + key := h.Sum64() + return jumpHash(key, n), jumpHash(mix64(key), v) +} + +// jumpHash maps key to one of n buckets such that growing n to n+1 moves a +// key only into bucket n, and each key with probability 1/(n+1). +func jumpHash(key uint64, n int) int { + var b, j int64 = -1, 0 + for j < int64(n) { + b = j + key = key*2862933555777941757 + 1 + j = int64(float64(b+1) * (float64(int64(1)<<31) / float64((key>>33)+1))) + } + return int(b) +} + +// mix64 is splitmix64's finalizer. +func mix64(z uint64) uint64 { + z = (z ^ (z >> 30)) * 0xbf58476d1ce4e5b9 + z = (z ^ (z >> 27)) * 0x94d049bb133111eb + return z ^ (z >> 31) +} + +// natsIngestPartition is every subject of ingest partition p: what its +// stream holds. +func natsIngestPartition(prefix string, p int) string { + return prefix + ".ingest." + strconv.Itoa(p) + ".>" +} + +// natsShardFilter is every subject of shard s of partition p: what its +// durable filters on. +func natsShardFilter(prefix string, p, s int) string { + return prefix + ".ingest." + strconv.Itoa(p) + "." + strconv.Itoa(s) + ".>" +} + +// natsShardDurable is shard s's durable on every partition. It is not +// zero-padded, so a durable keeps its name when the shard count grows. +func natsShardDurable(consumerPrefix string, s int) string { + return consumerPrefix + "-" + strconv.Itoa(s) +} + +// parseShardDurable is the shard a durable named by natsShardDurable +// serves, false for any other name. +func parseShardDurable(consumerPrefix, name string) (int, bool) { + rest, ok := strings.CutPrefix(name, consumerPrefix+"-") + if !ok || rest == "" || (len(rest) > 1 && rest[0] == '0') { + return 0, false + } + s, err := strconv.ParseUint(rest, 10, 31) + return int(s), err == nil +} + +// natsHistorySubjects is every subject of the history stream. +func natsHistorySubjects(prefix string) string { return prefix + ".hist.>" } + +// natsHistorySubject is where topic t's rows are republished. +func natsHistorySubject(prefix string, t Topic) (string, error) { + return subject(prefix+".hist.", t) +} + +// natsRepublish is how partition p republishes each row it stores to the +// history: the same subject with the partition and shard tokens dropped. +func natsRepublish(prefix string, p int) jetstream.RePublish { + return jetstream.RePublish{Source: prefix + ".ingest." + strconv.Itoa(p) + ".*.>", Destination: natsHistorySubjects(prefix)} +} + +// natsDLQSubjects is every subject of the shared dead-letter stream. +func natsDLQSubjects(prefix string) string { return prefix + ".dlq.>" } + +// natsIngestSubject is where topic t is published among n partitions of v +// shards, and the partition that holds it. +func natsIngestSubject(prefix string, n, v int, t Topic) (string, int, error) { + p, s := natsRoute(t, n, v) + subj, err := subject(prefix+".ingest."+strconv.Itoa(p)+"."+strconv.Itoa(s)+".", t) + return subj, p, err +} + +// natsDLQSubject is where a message on topic t is parked. +func natsDLQSubject(prefix string, t Topic) (string, error) { + return subject(prefix+".dlq.", t) +} + +// natsTopicKey is the topic key a subject under prefix carries — the +// partition and shard, the hist or the dlq token stripped — false for a +// subject of none of those kinds. +func natsTopicKey(prefix, subj string) (string, bool) { + rest, ok := strings.CutPrefix(subj, prefix+".") + if !ok { + return "", false + } + if key, ok := strings.CutPrefix(rest, "dlq."); ok { + return key, key != "" + } + if key, ok := strings.CutPrefix(rest, "hist."); ok { + return key, key != "" + } + rest, ok = strings.CutPrefix(rest, "ingest.") + if !ok { + return "", false + } + for range 2 { // the partition, then the shard + var n string + n, rest, ok = strings.Cut(rest, ".") + if !ok { + return "", false + } + if _, err := strconv.ParseUint(n, 10, 31); err != nil { + return "", false + } + } + return rest, rest != "" +} diff --git a/internal/mq/subject_nats_test.go b/internal/mq/subject_nats_test.go new file mode 100644 index 000000000..0b932f520 --- /dev/null +++ b/internal/mq/subject_nats_test.go @@ -0,0 +1,168 @@ +package mq + +import ( + "strconv" + "strings" + "testing" + + "github.com/Wave-RF/WaveHouse/internal/tenant" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// The table-to-(partition, shard) mapping is pinned: a change moves tables +// on upgrade, so it must be deliberate. The values were derived apart from +// this code (the paper's algorithm, FNV-1a 64 and splitmix64's finalizer, +// in another language), not by running natsRoute. +func TestNATSRoute_Golden(t *testing.T) { + t.Parallel() + assert.Equal(t, []int{0, 6, 87, 520}, []int{jumpHash(0, 1), jumpHash(1, 10), jumpHash(0xdeadbeef, 128), jumpHash(256, 1024)}) + assert.Equal(t, []uint64{0, 0x5692161d100b05e5, 0x4e062702ec929eea}, []uint64{mix64(0), mix64(1), mix64(0xdeadbeef)}) + ns, vs := []int{1, 2, 4, 7, 16, 64}, []int{1, 8, 32, 33, 100} + for _, tc := range []struct { + topic Topic + partitions []int + shards []int + }{ + {Topic{Tenant: "0", Table: "events"}, []int{0, 1, 2, 2, 2, 2}, []int{0, 4, 29, 29, 84}}, + {Topic{Tenant: "acme", Table: "events"}, []int{0, 0, 2, 2, 11, 11}, []int{0, 0, 22, 22, 42}}, + {Topic{Tenant: "acme", Table: "clicks"}, []int{0, 1, 2, 6, 15, 41}, []int{0, 6, 16, 16, 33}}, + {Topic{Tenant: "globex", Table: "a.b *>% c"}, []int{0, 1, 3, 3, 3, 63}, []int{0, 3, 3, 3, 89}}, + {Topic{Tenant: "t-1_x", Table: "users", Scope: "ignored"}, []int{0, 0, 0, 0, 15, 27}, []int{0, 0, 22, 22, 22}}, + } { + got := make([]int, len(ns)) + for i, n := range ns { + got[i], _ = natsRoute(tc.topic, n, 1) + } + assert.Equal(t, tc.partitions, got, "partitions of %+v", tc.topic) + got = make([]int, len(vs)) + for i, v := range vs { + _, got[i] = natsRoute(tc.topic, 1, v) + } + assert.Equal(t, tc.shards, got, "shards of %+v", tc.topic) + } +} + +// Consistent hashing, for partitions and shards alike: growing the count by +// one moves a table only into the new bucket, and about 1/(n+1) of the +// tables; the tables spread evenly. The shard does not depend on the +// partition, so every partition's shards fill. +func TestNATSRoute_Consistent(t *testing.T) { + t.Parallel() + const tables = 20_000 + topics := make([]Topic, tables) + for i := range topics { + topics[i] = Topic{Tenant: tenant.ID("t" + strconv.Itoa(i%97)), Table: "table_" + strconv.Itoa(i)} + } + for _, n := range []int{1, 3, 4, 8, 15} { + movedP, movedS := 0, 0 + countsP, countsS := make([]int, n+1), make([]int, n+1) + for _, topic := range topics { + p0, s0 := natsRoute(topic, n, n) + p1, s1 := natsRoute(topic, n+1, n+1) + countsP[p1]++ + countsS[s1]++ + if p0 != p1 { + movedP++ + require.Equal(t, n, p1, "%+v moved to an old partition", topic) + } + if s0 != s1 { + movedS++ + require.Equal(t, n, s1, "%+v moved to an old shard", topic) + } + } + want := float64(tables) / float64(n+1) + assert.InDelta(t, want, float64(movedP), want*0.1, "partitions %d→%d", n, n+1) + assert.InDelta(t, want, float64(movedS), want*0.1, "shards %d→%d", n, n+1) + for b := range n + 1 { + assert.InDelta(t, want, float64(countsP[b]), want*0.1, "partition %d of %d", b, n+1) + assert.InDelta(t, want, float64(countsS[b]), want*0.1, "shard %d of %d", b, n+1) + } + } + // Four shards in each of two partitions: every one of the eight holds tables. + cells := map[[2]int]int{} + for _, topic := range topics { + p, s := natsRoute(topic, 2, 4) + cells[[2]int{p, s}]++ + } + assert.Len(t, cells, 8) + for cell, n := range cells { + assert.InDelta(t, tables/8, n, tables/8*0.1, "cell %v", cell) + } +} + +func TestNATSShardDurable(t *testing.T) { + t.Parallel() + assert.Equal(t, "wh-ingest-0", natsShardDurable("wh-ingest", 0)) + assert.Equal(t, "wh-ingest-31", natsShardDurable("wh-ingest", 31)) + for name, want := range map[string]int{"wh-ingest-0": 0, "wh-ingest-7": 7, "wh-ingest-255": 255} { + got, ok := parseShardDurable("wh-ingest", name) + assert.True(t, ok, name) + assert.Equal(t, want, got, name) + } + for _, name := range []string{"wh-ingest", "wh-ingest-", "wh-ingest-07", "wh-ingest-x", "wh-ingest-1-2", "other-1", "wh-ingest--1"} { + _, ok := parseShardDurable("wh-ingest", name) + assert.False(t, ok, name) + } +} + +func TestNATSSubjects_RoundTrip(t *testing.T) { + t.Parallel() + topics := []Topic{ + {Tenant: "acme", Table: "events"}, + {Tenant: "globex", Table: "a.b *>% c", Scope: "s.1"}, + } + for _, topic := range topics { + subj, p, err := natsIngestSubject("wh", 4, 8, topic) + require.NoError(t, err) + wantP, wantS := natsRoute(topic, 4, 8) + assert.Equal(t, wantP, p) + assert.True(t, strings.HasPrefix(subj, "wh.ingest."+strconv.Itoa(p)+"."+strconv.Itoa(wantS)+"."+string(topic.Tenant)+"."), subj) + key, ok := natsTopicKey("wh", subj) + require.True(t, ok, subj) + assert.Equal(t, topic, parseTopicKey(key)) + + hist, err := natsHistorySubject("wh", topic) + require.NoError(t, err) + assert.Equal(t, "wh.hist."+topic.key(), hist) + key, ok = natsTopicKey("wh", hist) + require.True(t, ok, hist) + assert.Equal(t, topic, parseTopicKey(key)) + + dlq, err := natsDLQSubject("wh", topic) + require.NoError(t, err) + assert.Equal(t, "wh.dlq."+topic.key(), dlq) + key, ok = natsTopicKey("wh", dlq) + require.True(t, ok, dlq) + assert.Equal(t, topic, parseTopicKey(key)) + } +} + +func TestNATSSubjects_RefuseATopicWithoutATenant(t *testing.T) { + t.Parallel() + _, _, err := natsIngestSubject("wh", 4, 8, Topic{Table: "events"}) + require.Error(t, err) + _, err = natsDLQSubject("wh", Topic{Tenant: "a.b", Table: "events"}) + require.Error(t, err) +} + +func TestNATSTopicKey_OtherSubjects(t *testing.T) { + t.Parallel() + for _, subj := range []string{ + "other.ingest.0.1.acme.events", "wh.ingest.acme.events", "wh.ingest.x.1.acme.events", "wh.ingest.0.x.acme.events", + "wh.ingest.0.1.", "wh.ingest.0.1", "wh.ingest.0", "wh.dlq.", "wh.hist.", "wh.history.acme.events", "wh", "", + } { + _, ok := natsTopicKey("wh", subj) + assert.False(t, ok, subj) + } +} + +func TestValidSubjectPrefix(t *testing.T) { + t.Parallel() + for _, ok := range []string{"wh", "acme-wh", "wh_2"} { + require.NoError(t, validSubjectPrefix(ok), ok) + } + for _, bad := range []string{"", "Wh", "a.b", "a*", "a>", "a b"} { + require.Error(t, validSubjectPrefix(bad), bad) + } +} diff --git a/internal/settings/store.go b/internal/settings/store.go index a03546411..9d4a19853 100644 --- a/internal/settings/store.go +++ b/internal/settings/store.go @@ -114,6 +114,23 @@ func (s *Store) DedupeFor(table string) Dedupe { return out } +// DedupeRetentions is the retention of the default (key "") and of every +// table override that names one, from one snapshot; 0 is forever. +func (s *Store) DedupeRetentions() map[string]time.Duration { + d := s.doc().Config.Dedupe + out := map[string]time.Duration{"": 0} + // Validate has parsed them already. + if d.Retention != nil { + out[""], _ = time.ParseDuration(*d.Retention) + } + for table, td := range d.Tables { + if td.Retention != nil { + out[table], _ = time.ParseDuration(*td.Retention) + } + } + return out +} + // ClickHouse is the adopted connection wiring, resolved as one value from // one snapshot so a reconnect never mixes the address of one document with // the database of another. The password is not here — it is boot config. diff --git a/internal/settings/store_test.go b/internal/settings/store_test.go index 3487ba505..02edf842b 100644 --- a/internal/settings/store_test.go +++ b/internal/settings/store_test.go @@ -65,6 +65,8 @@ func TestStore_DedupeFor_Cascade(t *testing.T) { assert.Equal(t, tt.want, s.DedupeFor(tt.table)) }) } + // Only the overrides that name a retention are listed; "" is the default. + assert.Equal(t, map[string]time.Duration{"": 720 * time.Hour, "views": 24 * time.Hour, "audit": 0}, s.DedupeRetentions()) } // A config.json without dedupe.retention keeps ids forever, and its table @@ -81,6 +83,7 @@ func TestStore_DedupeFor_RetentionMissing(t *testing.T) { assert.Equal(t, Dedupe{IDField: "event_id"}, s.DedupeFor("other"), "forever") assert.Equal(t, Dedupe{IDField: "click_id"}, s.DedupeFor("clicks"), "inherits forever") assert.Equal(t, Dedupe{IDField: "event_id", Retention: 24 * time.Hour}, s.DedupeFor("views")) + assert.Equal(t, map[string]time.Duration{"": 0, "views": 24 * time.Hour}, s.DedupeRetentions()) } // TestStore_SeedIsValid pins that the shipped starter directory passes its diff --git a/tests/integration/coord_nats_test.go b/tests/integration/coord_nats_test.go new file mode 100644 index 000000000..0ca60ed85 --- /dev/null +++ b/tests/integration/coord_nats_test.go @@ -0,0 +1,105 @@ +//go:build integration + +package tests + +import ( + "context" + "os" + "path/filepath" + "strconv" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/Wave-RF/WaveHouse/internal/app" + "github.com/Wave-RF/WaveHouse/internal/config" + "github.com/Wave-RF/WaveHouse/internal/mq" + "github.com/Wave-RF/WaveHouse/internal/mq/natstest" +) + +// Two full replicas on one NATS, through the shipped lease bucket as the +// restricted wavehouse user, each hold a shard claim membership lease, and +// take no sweeper lease: under nats retention is the operator's streams', so +// no sweeper is wired. When one stops, its membership lease is resigned. +func TestCoordNATS_ReplicasShareTheShards(t *testing.T) { + e := env(t) + ctx := context.Background() + natsURL := startNATS(t) + op, err := natstest.Connect(natsURL) + require.NoError(t, err) + t.Cleanup(op.Close) + require.NoError(t, op.ApplyShipped(ctx)) + root, err := writeTestSettings(e.ch) + require.NoError(t, err) + + replicas := map[string]*natsProcess{} + for range 2 { + p := bootNATSProcess(t, natsURL, root, config.AllRoles()...) + replicas[p.id] = p + } + members := func() map[string]bool { + out := map[string]bool{} + for j := range 32 { + h, err := op.LeaseHolder(ctx, natstest.CoordBucket, "ingest.m"+strconv.Itoa(j)) + require.NoError(t, err) + if h != "" { + out[h] = true + } + } + return out + } + require.Eventually(t, func() bool { m := members(); return len(m) == 2 }, 15*time.Second, 50*time.Millisecond, "each replica is a member: %v", members()) + for id := range members() { + assert.Contains(t, replicas, id) + } + sweeper, err := op.LeaseHolder(ctx, natstest.CoordBucket, "sweeper") + require.NoError(t, err) + assert.Empty(t, sweeper, "no sweeper runs under nats") + + var first string + for id := range replicas { + first = id + break + } + replicas[first].stop() + select { + case err := <-replicas[first].runDone: + require.NoError(t, err) + case <-time.After(15 * time.Second): + t.Fatal("the replica did not stop") + } + require.Eventually(t, func() bool { m := members(); return len(m) == 1 && !m[first] }, 10*time.Second, 50*time.Millisecond, + "the stopped replica resigned its membership: %v", members()) +} + +// The lease bucket is the operator's: boot waits for it with the rest of the +// topology and then refuses, naming it. +func TestCoordNATS_MissingBucketRefusesBoot(t *testing.T) { + srv := natstest.Start(t) + require.NoError(t, srv.Operator.DeleteBucket(t.Context(), natstest.CoordBucket)) + pw := filepath.Join(t.TempDir(), "nats-password") + require.NoError(t, os.WriteFile(pw, []byte(natstest.Password(natstest.WaveHouseUser)), 0o600)) + root, err := writeTestSettings(env(t).ch) + require.NoError(t, err) + cfg := &config.Config{ + DataDir: t.TempDir(), + Server: config.Server{Port: 1, ShutdownTimeout: 1}, + MQ: config.MQ{Backend: config.MQNATS, NATS: config.MQNATSConfig{ + URLs: []string{srv.URL()}, User: natstest.WaveHouseUser, PasswordFile: pw, + SubjectPrefix: "wh", Partitions: 4, Shards: 8, IngestConsumer: "wh-ingest", + ConnectTimeout: 5 * time.Second, PublishTimeout: 5 * time.Second, TopologyWait: 300 * time.Millisecond, + }}, + Cache: config.Cache{Backend: config.CacheLocal, L1MaxCost: 1 << 20}, + Dedupe: config.Dedupe{Backend: config.DedupePebble, Lease: 30 * time.Second, ReserveConcurrency: 64}, + Coord: config.Coord{Backend: config.CoordNATS}, + Roles: config.AllRoles(), + InstanceID: "boot", + Settings: config.Settings{Dir: root}, + } + require.NoError(t, cfg.Validate()) + _, err = app.New(t.Context(), app.Options{Config: cfg}) + require.ErrorIs(t, err, mq.ErrTopology) + assert.ErrorContains(t, err, "kv bucket wh_coord") +} diff --git a/tests/integration/mq_nats_test.go b/tests/integration/mq_nats_test.go new file mode 100644 index 000000000..4e98ac19c --- /dev/null +++ b/tests/integration/mq_nats_test.go @@ -0,0 +1,273 @@ +//go:build integration + +package tests + +import ( + "bufio" + "context" + "encoding/json" + "fmt" + "io" + "net" + "net/http" + "net/url" + "os" + "path/filepath" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/Wave-RF/WaveHouse/internal/app" + "github.com/Wave-RF/WaveHouse/internal/config" + "github.com/Wave-RF/WaveHouse/internal/ingest" + "github.com/Wave-RF/WaveHouse/internal/mq" + "github.com/Wave-RF/WaveHouse/internal/mq/natstest" + "github.com/Wave-RF/WaveHouse/internal/tenant" +) + +const natsOperatorKey = "it-nats-operator-key" + +// natsProcess is one WaveHouse process booted on mq.backend: nats. +type natsProcess struct { + app *app.App + id string // its instance_id, the holder its leases name + baseURL string + runDone chan error + stop context.CancelFunc // ends Run; runDone then says how +} + +// natsProcesses numbers the processes the tests boot, for their instance_id. +var natsProcesses atomic.Int32 + +// bootNATSProcess boots the real wiring with roles over the nested settings +// directory root, on the NATS at natsURL as the shipped wavehouse user with +// its leases in the shipped bucket, and runs it until the test ends (or until +// it fails on its own: runDone). +func bootNATSProcess(t *testing.T, natsURL, root string, roles ...config.Role) *natsProcess { + t.Helper() + ctx := context.Background() + pw := filepath.Join(t.TempDir(), "nats-password") + require.NoError(t, os.WriteFile(pw, []byte(natstest.Password(natstest.WaveHouseUser)+"\n"), 0o600)) + var lc net.ListenConfig + ln, err := lc.Listen(ctx, "tcp", "127.0.0.1:0") + require.NoError(t, err) + cfg := &config.Config{ + DataDir: t.TempDir(), + Server: config.Server{Port: ln.Addr().(*net.TCPAddr).Port, ShutdownTimeout: 10}, + ClickHouse: config.ClickHouse{Password: testCHPassword}, + Auth: config.Auth{OperatorKey: natsOperatorKey}, + MQ: config.MQ{Backend: config.MQNATS, NATS: config.MQNATSConfig{ + URLs: []string{natsURL}, User: natstest.WaveHouseUser, PasswordFile: pw, + SubjectPrefix: "wh", Partitions: 4, Shards: 8, IngestConsumer: "wh-ingest", + ConnectTimeout: 5 * time.Second, PublishTimeout: 5 * time.Second, TopologyWait: 30 * time.Second, + }}, + Cache: config.Cache{Backend: config.CacheLocal, L1MaxCost: 1 << 20}, + Dedupe: config.Dedupe{Backend: config.DedupePebble, Lease: 30 * time.Second, ReserveConcurrency: 64}, + Coord: config.Coord{Backend: config.CoordNATS}, + Roles: roles, + InstanceID: fmt.Sprintf("proc-%d", natsProcesses.Add(1)), + Settings: config.Settings{Dir: root}, + } + require.NoError(t, cfg.Validate(), "the split boots on a shared queue") + a, err := app.New(ctx, app.Options{Config: cfg, Listener: ln}) + require.NoError(t, err) + runCtx, stop := context.WithCancel(ctx) + p := &natsProcess{app: a, id: cfg.InstanceID, baseURL: "http://" + ln.Addr().String(), runDone: make(chan error, 1), stop: stop} + go func() { p.runDone <- a.Run(runCtx) }() + t.Cleanup(func() { + stop() + closeCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + assert.NoError(t, a.Close(closeCtx)) + }) + require.NoError(t, waitForLive(ctx, p.baseURL, 30*time.Second)) + return p +} + +func (p *natsProcess) do(t *testing.T, method, path string, headers map[string]string, body string) (int, string) { + t.Helper() + req, err := http.NewRequestWithContext(t.Context(), method, p.baseURL+path, strings.NewReader(body)) + require.NoError(t, err) + for k, v := range headers { + req.Header.Set(k, v) + } + resp, err := http.DefaultClient.Do(req) + require.NoError(t, err) + defer func() { _ = resp.Body.Close() }() + b, err := io.ReadAll(resp.Body) + require.NoError(t, err) + return resp.StatusCode, string(b) +} + +// sse opens GET /v1/stream on p for tenant id with query, and returns its +// lines as they arrive, once the stream is open. +func (p *natsProcess) sse(t *testing.T, id tenant.ID, query string) <-chan string { + t.Helper() + ctx, cancel := context.WithCancel(t.Context()) + t.Cleanup(cancel) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, p.baseURL+"/v1/stream?"+query, nil) + require.NoError(t, err) + req.Header.Set(tenant.Header, id.String()) + resp, err := http.DefaultClient.Do(req) //nolint:bodyclose // closed by the reader below, on cancel + require.NoError(t, err) + require.Equal(t, http.StatusOK, resp.StatusCode) + lines := make(chan string, 256) + connected := make(chan struct{}) + go func() { + defer func() { _ = resp.Body.Close() }() + defer close(lines) + sc := bufio.NewScanner(resp.Body) + for sc.Scan() { + if sc.Text() == ": connected" { + close(connected) + continue + } + lines <- sc.Text() + } + }() + select { + case <-connected: + case <-time.After(10 * time.Second): + t.Fatal("the stream never opened") + } + return lines +} + +// awaitEvent reads lines until a data line contains want. +func awaitEvent(t *testing.T, lines <-chan string, want string) { + t.Helper() + deadline := time.After(30 * time.Second) + for { + select { + case line, ok := <-lines: + require.True(t, ok, "the stream ended before an event with %q", want) + if strings.HasPrefix(line, "data:") && strings.Contains(line, want) { + return + } + case <-deadline: + t.Fatalf("no event with %q within 30s", want) + } + } +} + +// TestNATSBackend_EndToEnd runs WaveHouse on mq.backend: nats against a +// real NATS set up the way an operator would: the Helm values' accounts and +// permissions, then the nack manifests (deployments/nats), applied before +// WaveHouse starts publishing, as the deployment guide says. Two processes +// share it, a split the embedded MQ cannot serve: A runs every role, B runs +// api and ingest. Over a nested directory with two tenants it shows ingest +// reaching each tenant's own ClickHouse database exactly once whichever +// worker takes the row, live SSE events reaching the API process that did not +// ingest them (every hub reads the history), SSE replay from the history, +// dead-letter counts kept per tenant on the one shared dead-letter stream, and +// the operator deleting the ingest durable ending every worker, and so every +// process. +func TestNATSBackend_EndToEnd(t *testing.T) { + e := env(t) + ctx := context.Background() + natsURL := startNATS(t) + op, err := natstest.Connect(natsURL) + require.NoError(t, err) + t.Cleanup(op.Close) + require.NoError(t, op.ApplyShipped(ctx)) + + databases := map[tenant.ID]string{} + root := t.TempDir() + for _, id := range []tenant.ID{"acme", "globex"} { + db := "it_nats_" + id.String() + databases[id] = db + require.NoError(t, e.chConn.Exec(ctx, "CREATE DATABASE IF NOT EXISTS "+db)) + t.Cleanup(func() { _ = e.chConn.Exec(context.Background(), "DROP DATABASE IF EXISTS "+db) }) + require.NoError(t, e.chConn.Exec(ctx, fmt.Sprintf("CREATE TABLE %s.events (id String, page String) ENGINE = MergeTree() ORDER BY id", db))) + files, err := tenantSettings(e.ch, db) + require.NoError(t, err) + require.NoError(t, writeSettingsFiles(filepath.Join(root, id.String()), files)) + } + + a := bootNATSProcess(t, natsURL, root, config.AllRoles()...) + b := bootNATSProcess(t, natsURL, root, config.RoleAPI, config.RoleIngest) + operator := map[string]string{"X-Operator-Key": natsOperatorKey} + for _, p := range []*natsProcess{a, b} { + for id := range databases { + require.Eventually(t, func() bool { + status, _ := p.do(t, http.MethodGet, "/v1/ops/schema?table=events&tenant="+id.String(), operator, "") + return status == http.StatusOK + }, 30*time.Second, 200*time.Millisecond, "tenant %s never discovered its schema", id) + } + } + + since := time.Now().UTC() + liveA := a.sse(t, "acme", "table=events") + liveB := b.sse(t, "acme", "table=events") + for id, row := range map[tenant.ID]string{"acme": `{"id":"a1","page":"home"}`, "globex": `{"id":"g1","page":"cart"}`} { + status, body := a.do(t, http.MethodPost, "/v1/ingest?table=events", map[string]string{tenant.Header: id.String(), "Content-Type": "application/json"}, row) + require.Equal(t, http.StatusOK, status, body) + } + + // Every API process's hub sees every event, whichever took the publish. + awaitEvent(t, liveA, `"a1"`) + awaitEvent(t, liveB, `"a1"`) + + // Each row reaches its own tenant's database, once, and no other's. + count := func(db, id string) uint64 { + var n uint64 + require.NoError(t, e.chConn.QueryRow(ctx, fmt.Sprintf("SELECT count() FROM %s.events WHERE id = '%s'", db, id)).Scan(&n)) + return n + } + require.Eventually(t, func() bool { + return count(databases["acme"], "a1") == 1 && count(databases["globex"], "g1") == 1 + }, 30*time.Second, 250*time.Millisecond, "each tenant's row reaches its own ClickHouse") + assert.Zero(t, count(databases["acme"], "g1")) + assert.Zero(t, count(databases["globex"], "a1")) + + // A reconnect replays from the history stream. + replay := b.sse(t, "acme", "table=events&since="+url.QueryEscape(since.Format(time.RFC3339Nano))) + awaitEvent(t, replay, `"a1"`) + + // A row no insert can take is parked on the shared dead-letter stream, + // and counted for its tenant alone. + payload, err := json.Marshal(ingest.EventMessage{ + TableName: "missing", + ReceivedTimestamp: time.Now().UTC().Format(time.RFC3339Nano), + Format: ingest.FormatJSONCompactEachRow, + Columns: []string{"id"}, + Row: json.RawMessage(`["x"]`), + }) + require.NoError(t, err) + require.NoError(t, a.app.MQ().Publish(ctx, mq.Topic{Tenant: "globex", Table: "missing"}, payload)) + dlq := func(p *natsProcess, id tenant.ID) (total float64, tables map[string]any) { + status, body := p.do(t, http.MethodGet, "/v1/ops/dlq/stats?tenant="+id.String(), operator, "") + require.Equal(t, http.StatusOK, status, body) + var stats struct { + Total float64 `json:"total"` + Tables map[string]any `json:"tables"` + } + require.NoError(t, json.Unmarshal([]byte(body), &stats)) + return stats.Total, stats.Tables + } + require.Eventually(t, func() bool { + _, tables := dlq(b, "globex") + _, ok := tables["missing"] + return ok + }, 30*time.Second, 250*time.Millisecond, "the unwritable row is parked under its tenant") + total, tables := dlq(a, "acme") + assert.Zero(t, total, "another tenant's parked rows are not counted for acme") + assert.Empty(t, tables) + + // The operator deleting the durable ends every worker, and with it every + // process: nothing can write what the API would go on accepting. + require.NoError(t, op.DeleteIngestDurables(ctx)) + for name, p := range map[string]*natsProcess{"A": a, "B": b} { + select { + case err := <-p.runDone: + require.ErrorIs(t, err, mq.ErrDeliveryEnded, "process %s", name) + assert.True(t, strings.HasPrefix(err.Error(), "ingest worker: "), "process %s: %v", name, err) + case <-time.After(30 * time.Second): + t.Fatalf("process %s kept running without its durable", name) + } + } +} diff --git a/tests/integration/roles_test.go b/tests/integration/roles_test.go new file mode 100644 index 000000000..4691cd63b --- /dev/null +++ b/tests/integration/roles_test.go @@ -0,0 +1,599 @@ +//go:build integration + +package tests + +import ( + "bufio" + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net" + "net/http" + "net/url" + "os" + "os/exec" + "path/filepath" + "runtime" + "strconv" + "strings" + "sync" + "syscall" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/Wave-RF/WaveHouse/internal/mq/natstest" + "github.com/Wave-RF/WaveHouse/internal/settings" +) + +// The binary under test is built once per run, with coverage when the suite +// collects it: a child inherits GOCOVERDIR and writes its counters there. +// TestMain starts the build alongside the containers, so it overlaps their +// startup and stays outside -timeout, which counts from m.Run; a cold cover +// build is most of a minute on a CI runner. binaryDir is removed by TestMain +// after the run (removeRolesBinary). +var ( + binaryBuilt = make(chan struct{}) + binaryDir string + binaryPath string + errBinary error +) + +// buildWavehouseBinary builds the binary and closes binaryBuilt. TestMain +// calls it once. +func buildWavehouseBinary() { + defer close(binaryBuilt) + dir, err := os.MkdirTemp("", "wh-roles-bin-") + if err != nil { + errBinary = err + return + } + binaryDir = dir + binaryPath = filepath.Join(dir, "wavehouse") + args := []string{"build", "-o", binaryPath} + if os.Getenv("GOCOVERDIR") != "" { + args = append(args, "-cover", "-coverpkg=./...") + } + _, file, _, _ := runtime.Caller(0) + cmd := exec.Command("go", append(args, "./cmd/wavehouse")...) //nolint:gosec // G204: fixed arguments + cmd.Dir = filepath.Join(filepath.Dir(file), "..", "..") + if out, err := cmd.CombinedOutput(); err != nil { + errBinary = fmt.Errorf("go build: %w\n%s", err, out) + } +} + +// removeRolesBinary deletes the binary buildWavehouseBinary built, if any. +func removeRolesBinary() { + <-binaryBuilt + if binaryDir != "" { + _ = os.RemoveAll(binaryDir) + } +} + +func wavehouseBinary(t *testing.T) string { + t.Helper() + <-binaryBuilt + require.NoError(t, errBinary) + return binaryPath +} + +// whProcess is one wavehouse process, configured by WH_* variables alone, +// as a Deployment would be. +type whProcess struct { + name string + baseURL string + cmd *exec.Cmd + log *lockedWriter + exited chan struct{} +} + +// freePort returns a port that was free a moment ago. +func freePort(t *testing.T) int { + t.Helper() + var lc net.ListenConfig + ln, err := lc.Listen(context.Background(), "tcp", "127.0.0.1:0") + require.NoError(t, err) + port := ln.Addr().(*net.TCPAddr).Port + require.NoError(t, ln.Close()) + return port +} + +// childEnv is the parent's environment without any WH_* variable (boot +// refuses an unbound one) or AWS setting, plus vars. +func childEnv(vars map[string]string) []string { + var out []string + for _, kv := range os.Environ() { + if strings.HasPrefix(kv, "WH_") || strings.HasPrefix(kv, "AWS_") { + continue + } + out = append(out, kv) + } + for k, v := range vars { + out = append(out, k+"="+v) + } + return out +} + +// startProcess starts the binary as instance name with vars. It is stopped +// at cleanup, and its log printed if the test failed. +func startProcess(t *testing.T, name string, vars map[string]string) *whProcess { + t.Helper() + port := freePort(t) + env := map[string]string{ + "WH_SERVER_PORT": strconv.Itoa(port), + "WH_INSTANCE_ID": name, + "WH_CONFIG": filepath.Join(t.TempDir(), "absent.yaml"), + "WH_DATA_DIR": t.TempDir(), + } + for k, v := range vars { + env[k] = v + } + cmd := exec.Command(wavehouseBinary(t)) //nolint:gosec // G204: the binary this test built + cmd.Env = childEnv(env) + w := &lockedWriter{w: &bytes.Buffer{}} + cmd.Stdout, cmd.Stderr = w, w + p := &whProcess{name: name, baseURL: "http://127.0.0.1:" + strconv.Itoa(port), cmd: cmd, log: w, exited: make(chan struct{})} + require.NoError(t, cmd.Start()) + go func() { _ = cmd.Wait(); close(p.exited) }() + t.Cleanup(func() { + p.stop() + if t.Failed() { + t.Logf("---- %s log ----\n%s", name, w.String()) + } + }) + return p +} + +// stop sends SIGTERM and waits, killing the process after 15s. +func (p *whProcess) stop() { + select { + case <-p.exited: + return + default: + } + _ = p.cmd.Process.Signal(syscall.SIGTERM) + select { + case <-p.exited: + case <-time.After(15 * time.Second): + _ = p.cmd.Process.Kill() + <-p.exited + } +} + +// kill ends the process at once: no resign, no drain. +func (p *whProcess) kill(t *testing.T) { + t.Helper() + require.NoError(t, p.cmd.Process.Kill()) + <-p.exited +} + +// awaitLive waits for /livez, failing at once if the process exits. +func (p *whProcess) awaitLive(t *testing.T) { + t.Helper() + deadline := time.Now().Add(90 * time.Second) + for time.Now().Before(deadline) { + select { + case <-p.exited: + t.Fatalf("%s exited before it was live:\n%s", p.name, p.log.String()) + default: + } + if status, _, _, _ := p.request(http.MethodGet, "/livez", nil, ""); status == http.StatusOK { + return + } + time.Sleep(200 * time.Millisecond) + } + t.Fatalf("%s never went live:\n%s", p.name, p.log.String()) +} + +// awaitExit waits for the process to exit on its own and returns its log. +func (p *whProcess) awaitExit(t *testing.T, within time.Duration) string { + t.Helper() + select { + case <-p.exited: + case <-time.After(within): + t.Fatalf("%s still running after %s:\n%s", p.name, within, p.log.String()) + } + return p.log.String() +} + +func (p *whProcess) request(method, path string, headers map[string]string, body string) (int, http.Header, string, error) { + req, err := http.NewRequestWithContext(context.Background(), method, p.baseURL+path, strings.NewReader(body)) + if err != nil { + return 0, nil, "", err + } + for k, v := range headers { + req.Header.Set(k, v) + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + return 0, nil, "", err + } + defer func() { _ = resp.Body.Close() }() + b, err := io.ReadAll(resp.Body) + return resp.StatusCode, resp.Header, string(b), err +} + +// ingest posts body to p for table. +func (p *whProcess) ingest(t *testing.T, table, contentType, body string) (int, string) { + t.Helper() + status, _, resp, err := p.request(http.MethodPost, "/v1/ingest?table="+url.QueryEscape(table), map[string]string{"Content-Type": contentType}, body) + require.NoError(t, err) + return status, strings.TrimSpace(resp) +} + +// query runs a select-all structured query on p: status, X-Cache, body. +func (p *whProcess) query(t *testing.T, table string) (int, string, string) { + t.Helper() + status, h, body, err := p.request(http.MethodPost, "/v1/query?table="+url.QueryEscape(table), map[string]string{"Content-Type": "application/json"}, `{"select_all":true}`) + require.NoError(t, err) + return status, h.Get("X-Cache"), body +} + +// sse opens GET /v1/stream on p for table and returns its lines as they +// arrive, once the stream is open. +func (p *whProcess) sse(t *testing.T, table string) <-chan string { + t.Helper() + ctx, cancel := context.WithCancel(t.Context()) + t.Cleanup(cancel) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, p.baseURL+"/v1/stream?table="+url.QueryEscape(table), nil) + require.NoError(t, err) + resp, err := http.DefaultClient.Do(req) //nolint:bodyclose // closed by the reader below, on cancel + require.NoError(t, err) + require.Equal(t, http.StatusOK, resp.StatusCode) + lines := make(chan string, 1024) + connected := make(chan struct{}) + go func() { + defer func() { _ = resp.Body.Close() }() + defer close(lines) + sc := bufio.NewScanner(resp.Body) + for sc.Scan() { + if sc.Text() == ": connected" { + close(connected) + continue + } + lines <- sc.Text() + } + }() + select { + case <-connected: + case <-time.After(10 * time.Second): + t.Fatalf("%s: the stream never opened", p.name) + } + return lines +} + +type lockedWriter struct { + mu sync.Mutex + w *bytes.Buffer +} + +func (l *lockedWriter) Write(b []byte) (int, error) { + l.mu.Lock() + defer l.mu.Unlock() + return l.w.Write(b) +} + +func (l *lockedWriter) String() string { + l.mu.Lock() + defer l.mu.Unlock() + return l.w.String() +} + +// TestRoles_SeparateProcesses runs the split #613's workstream C2 describes, as +// separate OS processes of the real binary that share nothing but the +// backends: A and E serve the API (roles=api), B and C write the queue to +// ClickHouse (roles=ingest); no process sweeps, since under nats the +// streams' retention does. The queue is an operator-provisioned NATS (the +// shipped Helm values and nack manifests, four partitions of eight shards, +// the coordination bucket included), the cache Redis, dedupe a +// dynamodb-local table, over the suite's ClickHouse. It checks that B and C +// split the 32 shards evenly with none unowned, that ingest through the API +// lands in ClickHouse exactly once, that B's and C's inserts invalidate what +// A cached, that an SSE client on either API process sees every event, that +// a duplicate sent to both API processes is accepted once, that a third +// ingest process F takes its share by handover, and that killing C hands +// its shards to B and F about one lease later, with nothing lost or doubled. +func TestRoles_SeparateProcesses(t *testing.T) { + t.Parallel() + e := env(t) + ctx := context.Background() + natsURL := startNATS(t) + op, err := natstest.Connect(natsURL) + require.NoError(t, err) + t.Cleanup(op.Close) + require.NoError(t, op.ApplyShipped(ctx)) + _, redisAddr := startRedis(t) + + table := createTable(t, "event_id String, page String", "ORDER BY event_id") + files, err := tenantSettings(e.ch, testCHDatabase) + require.NoError(t, err) + var doc map[string]json.RawMessage + require.NoError(t, json.Unmarshal(files[settings.FileConfig], &doc)) + doc["dedupe"] = json.RawMessage(`{"enabled": true, "id_field": "event_id", "require_id": true, "retention": "0", "tables": {}}`) + files[settings.FileConfig], err = json.Marshal(doc) + require.NoError(t, err) + settingsDir := filepath.Join(t.TempDir(), "settings") + require.NoError(t, writeSettingsFiles(settingsDir, files)) + + pw := filepath.Join(t.TempDir(), "nats-password") + require.NoError(t, os.WriteFile(pw, []byte(natstest.Password(natstest.WaveHouseUser)+"\n"), 0o600)) + none := filepath.Join(t.TempDir(), "none") + shared := map[string]string{ + "WH_SETTINGS_DIR": settingsDir, + "WH_CH_PASSWORD": testCHPassword, + "WH_AUTH_OPERATOR_KEY": natsOperatorKey, + + "WH_MQ_BACKEND": "nats", + "WH_MQ_NATS_URLS": natsURL, + "WH_MQ_NATS_USER": natstest.WaveHouseUser, + "WH_MQ_NATS_PASSWORD_FILE": pw, + "WH_MQ_NATS_PARTITIONS": "4", + "WH_MQ_NATS_SHARDS": "8", + "WH_PROMETHEUS_ENABLED": "true", // the metrics path on server.port + "WH_COORD_BACKEND": "nats", + + "WH_CACHE_BACKEND": "redis", + "WH_CACHE_REDIS_ADDRS": redisAddr, + "WH_CACHE_REDIS_KEY_PREFIX": fmt.Sprintf("roles%d", cachePrefixes.Add(1)), + "WH_CACHE_REDIS_TIMEOUT": "1s", + "WH_DEDUPE_BACKEND": "dynamodb", + "WH_DEDUPE_DYNAMODB_TABLE": newDynamoTable(), + "WH_DEDUPE_DYNAMODB_REGION": "us-east-1", + "WH_DEDUPE_DYNAMODB_ENDPOINT": e.dynamoEndpoint, + "WH_DEDUPE_DYNAMODB_TIMEOUT": "5s", + "WH_DEDUPE_DYNAMODB_CREATE_TABLE": "true", + // The SDK's default chain, as in production; never the developer's files. + "AWS_ACCESS_KEY_ID": "local", "AWS_SECRET_ACCESS_KEY": "local", + "AWS_CONFIG_FILE": none, "AWS_SHARED_CREDENTIALS_FILE": none, "AWS_EC2_METADATA_DISABLED": "true", + } + with := func(roles string) map[string]string { + vars := map[string]string{"WH_ROLES": roles} + for k, v := range shared { + vars[k] = v + } + return vars + } + + // A first: it creates the dedupe table, which E then finds. + a := startProcess(t, "api-a", with("api")) + a.awaitLive(t) + procs := []*whProcess{ + startProcess(t, "api-e", with("api")), + startProcess(t, "ingest-b", with("ingest")), + startProcess(t, "ingest-c", with("ingest")), + } + for _, p := range procs { + p.awaitLive(t) + } + apiE, ingestB, ingestC := procs[0], procs[1], procs[2] + + // B and C split the shards: each at most half, none left without an owner. + require.Eventually(t, func() bool { return shardsSplit(t, ingestB, ingestC) }, 60*time.Second, 250*time.Millisecond, + "B and C never split the %d shards", shippedShards) + + // Every API process sees every event: the hub is per process (#613). + liveA, liveE := a.sse(t, table), apiE.sse(t, table) + + // A fills the cache, then a batch through A is written by B or C, the + // only processes with a worker, and A's next answer is a miss carrying it: + // an invalidation, not an expiry, since the fill is younger than the + // shortest TTL. + status, xc, body := a.query(t, table) + require.Equal(t, http.StatusOK, status, body) + require.Equal(t, "MISS", xc) + require.Eventually(t, func() bool { _, x, _ := a.query(t, table); return x == "HIT" }, 10*time.Second, 100*time.Millisecond, "A's fill never reached the shared cache") + filled := time.Now() + + const n = 50 + var batch strings.Builder + for i := range n { + fmt.Fprintf(&batch, `{"event_id":"e%03d","page":"p%d"}`+"\n", i, i%7) + } + status, resp := a.ingest(t, table, "application/x-ndjson", batch.String()) + require.Equal(t, http.StatusOK, status, resp) + + count := func() (uint64, uint64) { + var c, u uint64 + require.NoError(t, e.chConn.QueryRow(ctx, "SELECT count(), uniqExact(event_id) FROM "+table).Scan(&c, &u)) + return c, u + } + require.Eventually(t, func() bool { c, _ := count(); return c >= n }, 60*time.Second, 200*time.Millisecond, "the batch never reached ClickHouse") + var invalidated bool + require.Eventually(t, func() bool { + st, x, b := a.query(t, table) + invalidated = st == http.StatusOK && x == "MISS" && strings.Contains(b, "e049") + return invalidated || x == "HIT" && time.Since(filled) > minCacheTTL + }, 30*time.Second, 100*time.Millisecond) + require.True(t, invalidated, "A served its pre-insert fill: B's and C's inserts did not reach A's cache") + assert.Less(t, time.Since(filled), 3*minCacheTTL) + + for i := range n { + id := fmt.Sprintf("e%03d", i) + awaitEvent(t, liveA, id) + awaitEvent(t, liveE, id) + } + + // Exactly once, and it stays so: nothing is redelivered and inserted again. + c, u := count() + assert.Equal(t, uint64(n), c) + assert.Equal(t, uint64(n), u) + assert.Never(t, func() bool { c, _ := count(); return c != n }, 5*time.Second, 500*time.Millisecond, "a row was inserted twice") + + // One id sent to both API processes at once is accepted by one of them. + type answer struct { + status int + body string + } + answers := make(chan answer, 2) + for _, p := range []*whProcess{a, apiE} { + go func() { + st, _, b, err := p.request(http.MethodPost, "/v1/ingest?table="+url.QueryEscape(table), map[string]string{"Content-Type": "application/json"}, `{"event_id":"dup","page":"x"}`) + assert.NoError(t, err) + answers <- answer{st, strings.TrimSpace(b)} + }() + } + var ok, dup, inflight int + for range 2 { + ans := <-answers + switch { + case ans.status == http.StatusOK && strings.Contains(ans.body, `"ok":true`): + ok++ + case ans.status == http.StatusOK && strings.Contains(ans.body, `"duplicate":true`): + dup++ + case ans.status == http.StatusServiceUnavailable: + inflight++ // the other's claim was still pending + default: + t.Errorf("unexpected answer %d %s", ans.status, ans.body) + } + } + assert.Equal(t, 1, ok, "exactly one process accepts the id") + assert.Equal(t, 1, dup+inflight) + status, resp = apiE.ingest(t, table, "application/json", `{"event_id":"dup","page":"y"}`) + require.Equal(t, http.StatusOK, status, resp) + assert.JSONEq(t, `{"duplicate":true}`, resp, "the id is committed for every process") + status, resp = a.ingest(t, table, "application/json", `{"event_id":"dup","page":"z"}`) + require.Equal(t, http.StatusOK, status, resp) + assert.JSONEq(t, `{"duplicate":true}`, resp) + require.Eventually(t, func() bool { c, _ := count(); return c == n+1 }, 60*time.Second, 200*time.Millisecond) + assert.Never(t, func() bool { c, _ := count(); return c != n+1 }, 3*time.Second, 500*time.Millisecond, "the duplicate was inserted") + + // A third ingest process takes its share, handed over (released, not + // waited out), and a batch is still written exactly once. + ingestF := startProcess(t, "ingest-f", with("ingest")) + ingestF.awaitLive(t) + joined := time.Now() + require.Eventually(t, func() bool { return shardsSplit(t, ingestB, ingestC, ingestF) }, 60*time.Second, 250*time.Millisecond, + "B, C and F never split the shards") + t.Logf("three ingest processes split the shards %s after F went live", time.Since(joined).Round(100*time.Millisecond)) + released := metricSum(t, ingestB, `wavehouse_ingest_shard_events_total{event="released"`) + metricSum(t, ingestC, `wavehouse_ingest_shard_events_total{event="released"`) + assert.Positive(t, released, "B and C handed shards to F") + ingestBatch(t, a, table, "f", n) + require.Eventually(t, func() bool { c, _ := count(); return c == 2*n+1 }, 60*time.Second, 200*time.Millisecond) + + // Killed without releasing or resigning, C's shards go to B and F once + // its membership lease has gone unrenewed, and a batch published across + // the handover is written exactly once. + ingestC.kill(t) + killed := time.Now() + require.Eventually(t, func() bool { return shardsSplit(t, ingestB, ingestF) }, membershipLease+30*time.Second, 250*time.Millisecond, + "B and F never took C's shards") + took := time.Since(killed) + t.Logf("C's shards moved %s after it was killed (membership lease %s)", took.Round(100*time.Millisecond), membershipLease) + assert.GreaterOrEqual(t, took, membershipLease-3*time.Second, "C's shards moved before its lease could have lapsed") + assert.Less(t, took, membershipLease+20*time.Second) + ingestBatch(t, a, table, "k", n) + require.Eventually(t, func() bool { c, _ := count(); return c == 3*n+1 }, 60*time.Second, 200*time.Millisecond) + c, u = count() + assert.Equal(t, c, u, "no row inserted twice") + assert.Never(t, func() bool { c, _ := count(); return c != 3*n+1 }, 3*time.Second, 500*time.Millisecond) +} + +// TestRoles_BootRefusesWhatTheBackendsCannotServe drives the boot rules 1–5 +// through the real binary and its environment: each +// combination exits non-zero before dialing anything, naming the fix. +func TestRoles_BootRefusesWhatTheBackendsCannotServe(t *testing.T) { + t.Parallel() + nats := map[string]string{"WH_MQ_BACKEND": "nats", "WH_MQ_NATS_URLS": "nats://127.0.0.1:1", "WH_COORD_BACKEND": "nats"} + cases := []struct { + name string + vars map[string]string + want string + }{ + {"1: unknown backend", map[string]string{"WH_CACHE_BACKEND": "memcached"}, "is not a backend this build has; valid: local, redis"}, + {"1: unknown role", map[string]string{"WH_ROLES": "api,reader"}, "is not a role; valid: api,ingest,sweeper"}, + {"1: repeated role", map[string]string{"WH_ROLES": "api,api"}, "api\\\" twice"}, + {"2: a split over the embedded queue", map[string]string{"WH_ROLES": "api,ingest"}, "the embedded MQ lives inside this process"}, + {"3: nats leases without the nats queue", map[string]string{"WH_COORD_BACKEND": "nats"}, "the NATS leases ride mq.nats"}, + {"4: ingest on the nats queue with local leases", merge(nats, map[string]string{"WH_COORD_BACKEND": "local", "WH_ROLES": "api,ingest"}), "in a process running ingest"}, + {"a sweeper-only process on the nats queue runs nothing", merge(nats, map[string]string{"WH_ROLES": "sweeper"}), "would run nothing"}, + {"5: api without ingest over a local cache", merge(nats, map[string]string{"WH_ROLES": "api"}), "would never reach the API's cache"}, + {"5: ingest without api over a local cache", merge(nats, map[string]string{"WH_ROLES": "ingest,sweeper"}), "would never reach the API's cache"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + p := startProcess(t, "refused", merge(tc.vars, map[string]string{"WH_SETTINGS_DIR": t.TempDir()})) + log := p.awaitExit(t, 30*time.Second) + assert.NotZero(t, p.cmd.ProcessState.ExitCode()) + assert.Contains(t, log, tc.want) + }) + } +} + +// merge returns a copy of a with b's entries over it. +func merge(a, b map[string]string) map[string]string { + out := map[string]string{} + for k, v := range a { + out[k] = v + } + for k, v := range b { + out[k] = v + } + return out +} + +// membershipLease is how long an ingest process that stopped renewing keeps +// its membership lease (internal/mq's default; the others wait it out on +// their own clocks). +const membershipLease = 15 * time.Second + +// shippedShards is the units of the shipped manifests: four partitions of +// eight shards. +const shippedShards = 32 + +// metricSum sums the samples of p's metrics whose line starts with prefix: +// a metric name, or a name and the start of its labels. +func metricSum(t *testing.T, p *whProcess, prefix string) float64 { + t.Helper() + status, _, body, err := p.request(http.MethodGet, "/metrics", nil, "") + require.NoError(t, err) + require.Equal(t, http.StatusOK, status) + sum := 0.0 + for line := range strings.Lines(body) { + line = strings.TrimSpace(line) + if !strings.HasPrefix(line, prefix) || strings.HasPrefix(line, "#") { + continue + } + if rest := line[len(prefix):]; !strings.Contains(prefix, "{") && !strings.HasPrefix(rest, " ") && !strings.HasPrefix(rest, "{") { + continue // another metric whose name the prefix begins + } + fields := strings.Fields(line) + v, err := strconv.ParseFloat(fields[len(fields)-1], 64) + require.NoError(t, err, line) + sum += v + } + return sum +} + +// shardsSplit reports whether procs own every shipped shard between them, +// none above an even share, and whichever of them counts the shards without +// an owner reads none. +func shardsSplit(t *testing.T, procs ...*whProcess) bool { + t.Helper() + limit := float64((shippedShards + len(procs) - 1) / len(procs)) + total, unowned := 0.0, 0.0 + for _, p := range procs { + owned := metricSum(t, p, "wavehouse_ingest_shards_owned") + if owned > limit { + return false + } + total += owned + unowned = max(unowned, metricSum(t, p, "wavehouse_ingest_shards_unowned")) + } + return total == shippedShards && unowned == 0 +} + +// ingestBatch posts n rows through p, their ids led by prefix. +func ingestBatch(t *testing.T, p *whProcess, table, prefix string, n int) { + t.Helper() + var batch strings.Builder + for i := range n { + fmt.Fprintf(&batch, `{"event_id":"%s%03d","page":"p%d"}`+"\n", prefix, i, i%7) + } + status, resp := p.ingest(t, table, "application/x-ndjson", batch.String()) + require.Equal(t, http.StatusOK, status, resp) +} diff --git a/tests/integration/setup_test.go b/tests/integration/setup_test.go index 240417b69..48f5b52ec 100644 --- a/tests/integration/setup_test.go +++ b/tests/integration/setup_test.go @@ -10,6 +10,7 @@ package tests import ( + "bytes" "context" "encoding/json" "errors" @@ -34,6 +35,7 @@ import ( "github.com/Wave-RF/WaveHouse/internal/config" "github.com/Wave-RF/WaveHouse/internal/discovery" "github.com/Wave-RF/WaveHouse/internal/mq" + "github.com/Wave-RF/WaveHouse/internal/mq/natstest" "github.com/Wave-RF/WaveHouse/internal/settings" ) @@ -108,14 +110,18 @@ func createTable(t *testing.T, columns, tableOpts string) string { } func TestMain(m *testing.M) { + go buildWavehouseBinary() code, cleanup := setup() if code != 0 { cleanup() + removeRolesBinary() os.Exit(code) } + <-binaryBuilt exit := m.Run() cleanup() + removeRolesBinary() os.Exit(exit) } @@ -399,6 +405,51 @@ func startClickHouse(ctx context.Context) (*chInstance, error) { return ch, nil } +// natsImage is the server line WaveHouse embeds, and the one the shipped +// Helm values pin. +const natsImage = "nats:2.14.6-alpine" + +// startNATS starts NATS as deployments/nats/values.yaml configures it — its +// accounts, users and permissions, JetStream on file storage — and returns +// its client URL. Nothing is created on it: that is the operator's step. +// The store is a tmpfs, so the container leaves nothing behind. +func startNATS(t *testing.T) string { + t.Helper() + ctx := context.Background() + conf, err := natstest.ServerConfig(natstest.ShippedValues(), "/data") + if err != nil { + t.Fatalf("nats config: %v", err) + } + container, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ + ContainerRequest: testcontainers.ContainerRequest{ + Image: natsImage, + ExposedPorts: []string{"4222/tcp"}, + Tmpfs: map[string]string{"/data": "rw"}, + Files: []testcontainers.ContainerFile{{ + Reader: bytes.NewReader(conf), ContainerFilePath: "/etc/nats/nats-server.conf", FileMode: 0o644, + }}, + // The log line alone can precede the host port's forwarding when + // several containers start at once. + WaitingFor: wait.ForAll( + wait.ForLog("Server is ready"), + wait.ForListeningPort("4222/tcp"), + ).WithDeadline(60 * time.Second), + }, + Started: true, + }) + if container != nil { + t.Cleanup(func() { _ = container.Terminate(context.Background()) }) + } + if err != nil { + t.Fatalf("start nats: %v", err) + } + endpoint, err := container.PortEndpoint(ctx, "4222/tcp", "nats") + if err != nil { + t.Fatalf("nats endpoint: %v", err) + } + return endpoint +} + // startDynamoDBLocal starts dynamodb-local in memory (no volume) and // returns it with its endpoint URL. func startDynamoDBLocal(ctx context.Context) (testcontainers.Container, string, error) { diff --git a/tests/integration/shard_claims_test.go b/tests/integration/shard_claims_test.go new file mode 100644 index 000000000..2ceb4a1b3 --- /dev/null +++ b/tests/integration/shard_claims_test.go @@ -0,0 +1,344 @@ +//go:build integration + +package tests + +import ( + "context" + "errors" + "io" + "net" + "os" + "path/filepath" + "slices" + "strconv" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/Wave-RF/WaveHouse/internal/ingest" + "github.com/Wave-RF/WaveHouse/internal/mq" + "github.com/Wave-RF/WaveHouse/internal/mq/natstest" + "github.com/Wave-RF/WaveHouse/internal/tenant" +) + +// The shipped topology's partitions and shards: 32 units. +const ( + claimPartitions = 4 + claimShards = 8 +) + +// shardLog records, per topic, the processes that received its rows. +type shardLog struct { + mu sync.Mutex + rows map[mq.Topic]map[string]int + at map[mq.Topic]time.Time // the last row's arrival +} + +func (l *shardLog) add(topic mq.Topic, proc string) { + l.mu.Lock() + defer l.mu.Unlock() + if l.rows == nil { + l.rows, l.at = map[mq.Topic]map[string]int{}, map[mq.Topic]time.Time{} + } + if l.rows[topic] == nil { + l.rows[topic] = map[string]int{} + } + l.rows[topic][proc]++ + l.at[topic] = time.Now() +} + +func (l *shardLog) count(topic mq.Topic) int { + l.mu.Lock() + defer l.mu.Unlock() + n := 0 + for _, c := range l.rows[topic] { + n += c + } + return n +} + +func (l *shardLog) countBy(topic mq.Topic, proc string) int { + l.mu.Lock() + defer l.mu.Unlock() + return l.rows[topic][proc] +} + +func (l *shardLog) writers(topic mq.Topic) []string { + l.mu.Lock() + defer l.mu.Unlock() + var out []string + for p := range l.rows[topic] { + out = append(out, p) + } + return out +} + +func (l *shardLog) reset() { + l.mu.Lock() + defer l.mu.Unlock() + l.rows, l.at = nil, nil +} + +// shardProc is one ingest process's claims over a shared NATS, as the +// restricted wavehouse user, with a lease of its own timings. +type shardProc struct { + id string + broker *mq.ExternalNATS + stop func() + failed <-chan error + hold atomic.Bool // leave rows unacked, as a stuck insert would + held atomic.Int64 +} + +func shardBroker(t *testing.T, url string) *mq.ExternalNATS { + t.Helper() + pw := filepath.Join(t.TempDir(), "nats-password") + require.NoError(t, os.WriteFile(pw, []byte(natstest.Password(natstest.WaveHouseUser)), 0o600)) + broker, err := mq.NewNATS(t.Context(), mq.NATSConfig{ + URLs: []string{url}, User: natstest.WaveHouseUser, PasswordFile: pw, + Topology: mq.NATSTopology{ + Prefix: "wh", Partitions: claimPartitions, Shards: claimShards, IngestConsumer: "wh-ingest", + PublishTimeout: 5 * time.Second, CoordBucket: natstest.CoordBucket, + }, + TopologyWait: 10 * time.Second, + }) + require.NoError(t, err) + t.Cleanup(func() { _ = broker.Close() }) + return broker +} + +func startShardProc(t *testing.T, url, id string, lease time.Duration, log *shardLog) *shardProc { + t.Helper() + ctx := t.Context() + p := &shardProc{id: id, broker: shardBroker(t, url)} + leases, err := p.broker.Leases(ctx, natstest.CoordBucket, id, mq.WithLeaseTimings(lease, lease*7/10, 100*time.Millisecond)) + require.NoError(t, err) + q, err := ingest.ClaimShards(p.broker, leases, ingest.ClaimConfig{Every: 100 * time.Millisecond, Handover: 2 * time.Second}) + require.NoError(t, err) + cons, err := q.CreateConsumer(ctx, mq.ConsumerConfig{Durable: ingest.BufferConsumerName, AckWait: 60 * time.Second, MaxAckPending: 10_000}) + require.NoError(t, err) + stop, failed, err := cons.Consume(func(m *mq.Message) { + log.add(m.Topic(), id) + if p.hold.Load() { + p.held.Add(1) + return + } + _ = m.DoubleAck(context.Background()) + }, 100) + require.NoError(t, err) + t.Cleanup(stop) + p.stop, p.failed = stop, failed + return p +} + +func shardTopics(id tenant.ID, n int) []mq.Topic { + out := make([]mq.Topic, n) + for i := range out { + out[i] = mq.Topic{Tenant: id, Table: "t" + strconv.Itoa(i)} + } + return out +} + +func publishRows(t *testing.T, b mq.Broker, topics []mq.Topic, rows int) { + t.Helper() + for _, topic := range topics { + for i := range rows { + require.NoError(t, b.Publish(t.Context(), topic, []byte(strconv.Itoa(i)))) + } + } +} + +// oneWriterEach publishes rows to topics and reports, once every row +// arrived, whether each topic's rows reached exactly one of procs. +func oneWriterEach(t *testing.T, b mq.Broker, log *shardLog, topics []mq.Topic, procs ...string) { + t.Helper() + log.reset() + publishRows(t, b, topics, 3) + require.Eventually(t, func() bool { + for _, topic := range topics { + if log.count(topic) < 3 { + return false + } + } + return true + }, 30*time.Second, 20*time.Millisecond, "every row arrives") + per := map[string]int{} + for _, topic := range topics { + w := log.writers(topic) + require.Len(t, w, 1, "%v written by %v", topic, w) + assert.Contains(t, procs, w[0]) + assert.Equal(t, 3, log.count(topic), "%v: no row twice", topic) + per[w[0]]++ + } + t.Logf("tables per process: %v", per) +} + +func pinnedUnits(t *testing.T, srv *natstest.Server) int { + t.Helper() + n := 0 + for p := range claimPartitions { + for s := range claimShards { + c, err := srv.Operator.JetStream().Consumer(t.Context(), "WH_INGEST_"+strconv.Itoa(p), "wh-ingest-"+strconv.Itoa(s)) + require.NoError(t, err) + for _, g := range c.CachedInfo().PriorityGroups { + if g.PinnedClientID != "" { + n++ + } + } + } + } + return n +} + +// Scaling 1 → 3 → 2 processes: every table is written by one process at +// every step, a clean stop hands its shards on well inside the lease, and no +// row is received twice. +func TestShardClaims_ScaleOneThreeTwo(t *testing.T) { + srv := natstest.Start(t) + log := &shardLog{} + const lease = 3 * time.Second + topics := append(shardTopics("acme", 30), shardTopics("globex", 30)...) + + a := startShardProc(t, srv.URL(), "proc-a", lease, log) + oneWriterEach(t, a.broker, log, topics, "proc-a") + assert.Positive(t, pinnedUnits(t, srv)) + + startShardProc(t, srv.URL(), "proc-b", lease, log) + c := startShardProc(t, srv.URL(), "proc-c", lease, log) + time.Sleep(3 * time.Second) // membership and handovers settle + oneWriterEach(t, a.broker, log, topics, "proc-a", "proc-b", "proc-c") + + stopped := time.Now() + c.stop() + oneWriterEach(t, a.broker, log, topics, "proc-a", "proc-b") + took := time.Since(stopped) + t.Logf("after a clean stop every table was written again within %s (lease %s)", took.Round(10*time.Millisecond), lease) + assert.Less(t, took, 3*lease, "a clean stop releases at once") + noShardFailure(t, a) +} + +// A process whose connection dies holding rows unacked: once its lease runs +// out and its pins lapse, a survivor takes its shards over, resets them, and +// receives the held rows at once, not after ack_wait (a minute here), and +// none of the rows it acked before. +func TestShardClaims_CrashWithRowsInFlight(t *testing.T) { + srv := natstest.Start(t) + log := &shardLog{} + const lease = 3 * time.Second + proxy := startTCPProxy(t, srv.URL()) + + victim := startShardProc(t, proxy.url, "proc-a", lease, log) + topics := shardTopics("acme", 40) + oneWriterEach(t, victim.broker, log, topics, "proc-a") + + survivor := startShardProc(t, srv.URL(), "proc-b", lease, log) + time.Sleep(3 * time.Second) + oneWriterEach(t, survivor.broker, log, topics, "proc-a", "proc-b") + + // The victim holds what it receives now, then dies. + victim.hold.Store(true) + log.reset() + publishRows(t, survivor.broker, topics, 1) + require.Eventually(t, func() bool { + n := 0 + for _, topic := range topics { + n += log.count(topic) + } + return n == len(topics) + }, 10*time.Second, 20*time.Millisecond) + held := victim.held.Load() + require.Positive(t, held, "the victim holds some rows") + killed := time.Now() + proxy.kill() + + require.Eventually(t, func() bool { + for _, topic := range topics { + if !slices.Contains(log.writers(topic), "proc-b") { + return false + } + } + return true + }, 45*time.Second, 50*time.Millisecond, "the survivor receives every table's row") + took := time.Since(killed) + t.Logf("the victim held %d rows; the survivor had all of them %s after the kill (lease %s, pinned ttl 10s, ack_wait 60s)", held, took.Round(100*time.Millisecond), lease) + assert.Less(t, took, 30*time.Second, "at takeover, not after ack_wait") + for _, topic := range topics { + assert.LessOrEqual(t, log.countBy(topic, "proc-b"), 1, "%v: an acked row replayed", topic) + } + noShardFailure(t, survivor) +} + +func noShardFailure(t *testing.T, procs ...*shardProc) { + t.Helper() + for _, p := range procs { + select { + case err := <-p.failed: + t.Fatalf("%s failed: %v", p.id, err) + default: + } + } +} + +// tcpProxy forwards a client to the NATS server until kill, which ends every +// connection and refuses new ones: to the client, the server is gone. +type tcpProxy struct { + url string + ln net.Listener + mu sync.Mutex + conns []net.Conn + dead atomic.Bool +} + +func startTCPProxy(t *testing.T, target string) *tcpProxy { + t.Helper() + u := target[len("nats://"):] + var lc net.ListenConfig + ln, err := lc.Listen(context.Background(), "tcp", "127.0.0.1:0") + require.NoError(t, err) + p := &tcpProxy{url: "nats://" + ln.Addr().String(), ln: ln} + t.Cleanup(p.kill) + go func() { + for { + in, err := ln.Accept() + if err != nil { + return + } + var d net.Dialer + out, err := d.DialContext(context.Background(), "tcp", u) + if err != nil { + _ = in.Close() + continue + } + p.mu.Lock() + if p.dead.Load() { + p.mu.Unlock() + _ = in.Close() + _ = out.Close() + return + } + p.conns = append(p.conns, in, out) + p.mu.Unlock() + go func() { _, _ = io.Copy(out, in); _ = out.Close() }() + go func() { _, _ = io.Copy(in, out); _ = in.Close() }() + } + }() + return p +} + +func (p *tcpProxy) kill() { + p.mu.Lock() + defer p.mu.Unlock() + if p.dead.Swap(true) { + return + } + _ = p.ln.Close() + for _, c := range p.conns { + if err := c.Close(); err != nil && !errors.Is(err, net.ErrClosed) { + continue + } + } +} diff --git a/tests/integration/shard_order_test.go b/tests/integration/shard_order_test.go new file mode 100644 index 000000000..f3827bbf9 --- /dev/null +++ b/tests/integration/shard_order_test.go @@ -0,0 +1,408 @@ +//go:build integration + +package tests + +import ( + "bufio" + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "slices" + "strconv" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/Wave-RF/WaveHouse/internal/chconn" + "github.com/Wave-RF/WaveHouse/internal/ingest" + "github.com/Wave-RF/WaveHouse/internal/mq" + "github.com/Wave-RF/WaveHouse/internal/mq/natstest" + "github.com/Wave-RF/WaveHouse/internal/tenant" + "github.com/Wave-RF/WaveHouse/internal/testutil" +) + +// insertLog is a stub ClickHouse shared by every process: the rows each +// table received, in the order their inserts completed, and who sent them. +type insertLog struct { + mu sync.Mutex + rows map[string][]int + writers map[string][]string + // first and last are when each process's first and last insert of each + // table completed: proc → table → time. + first, last map[string]map[string]time.Time + total int +} + +// server is proc's ClickHouse. gate, when set, runs before an insert is +// recorded, as a slow server would take its time. +func (l *insertLog) server(t *testing.T, proc string, gate func(table string)) *httptest.Server { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + table := r.URL.Query().Get("param_target_table") + var seqs []int + sc := bufio.NewScanner(r.Body) + for sc.Scan() { + var row []int + if err := json.Unmarshal(sc.Bytes(), &row); err != nil || len(row) != 1 { + http.Error(w, "bad row", http.StatusBadRequest) + return + } + seqs = append(seqs, row[0]) + } + if gate != nil { + gate(table) + } + l.mu.Lock() + if l.rows == nil { + l.rows, l.writers = map[string][]int{}, map[string][]string{} + l.first, l.last = map[string]map[string]time.Time{}, map[string]map[string]time.Time{} + } + if l.first[proc] == nil { + l.first[proc], l.last[proc] = map[string]time.Time{}, map[string]time.Time{} + } + now := time.Now() + if _, ok := l.first[proc][table]; !ok { + l.first[proc][table] = now + } + l.last[proc][table] = now + l.rows[table] = append(l.rows[table], seqs...) + if w := l.writers[table]; len(w) == 0 || w[len(w)-1] != proc { + l.writers[table] = append(w, proc) + } + l.total += len(seqs) + l.mu.Unlock() + w.WriteHeader(http.StatusOK) + })) + t.Cleanup(srv.Close) + return srv +} + +func (l *insertLog) count() int { + l.mu.Lock() + defer l.mu.Unlock() + return l.total +} + +func (l *insertLog) table(name string) []int { + l.mu.Lock() + defer l.mu.Unlock() + return slices.Clone(l.rows[name]) +} + +// inOrder reports whether seqs is exactly 0..n-1: every row once, in order. +func inOrder(seqs []int, n int) bool { + if len(seqs) != n { + return false + } + for i, s := range seqs { + if s != i { + return false + } + } + return true +} + +func seqEnvelope(t *testing.T, table string, seq int) []byte { + t.Helper() + out, err := json.Marshal(ingest.EventMessage{ + TableName: table, Format: ingest.FormatJSONCompactEachRow, + Columns: []string{"seq"}, Row: json.RawMessage("[" + strconv.Itoa(seq) + "]"), + }) + require.NoError(t, err) + return out +} + +// workerProc is one ingest process: the real worker over shard claims, as +// the restricted wavehouse user, writing to its own stub ClickHouse. +type workerProc struct { + id string + broker *mq.ExternalNATS + stop func(context.Context) error + failed <-chan error +} + +func startWorkerProc(t *testing.T, url, id string, lease time.Duration, cfg ingest.ClaimConfig, chURL string) *workerProc { + t.Helper() + ctx := t.Context() + p := &workerProc{id: id, broker: shardBroker(t, url)} + leases, err := p.broker.Leases(ctx, natstest.CoordBucket, id, mq.WithLeaseTimings(lease, lease*7/10, 100*time.Millisecond)) + require.NoError(t, err) + q, err := ingest.ClaimShards(p.broker, leases, cfg) + require.NoError(t, err) + p.stop, p.failed, err = ingest.StartIngestWorker(context.WithoutCancel(ctx), q, &testutil.MockCache{}, + func(tenant.ID) chconn.Target { return chconn.Target{URL: chURL, Database: "db"} }, nil) + require.NoError(t, err) + t.Cleanup(func() { _ = p.stop(context.Background()) }) + return p +} + +func noWorkerFailure(t *testing.T, procs ...*workerProc) { + t.Helper() + for _, p := range procs { + select { + case err := <-p.failed: + t.Fatalf("%s failed: %v", p.id, err) + default: + } + } +} + +// unitOf publishes data to topic before anything consumes it, and returns +// the unit whose durable the row waits on. +func unitOf(t *testing.T, srv *natstest.Server, b mq.Broker, topic mq.Topic, data []byte) string { + t.Helper() + before := pendingByUnit(t, srv) + require.NoError(t, b.Publish(t.Context(), topic, data)) + for u, n := range pendingByUnit(t, srv) { + if n > before[u] { + return u + } + } + t.Fatalf("no unit took %v's row", topic) + return "" +} + +func pendingByUnit(t *testing.T, srv *natstest.Server) map[string]uint64 { + t.Helper() + out := map[string]uint64{} + for p := range claimPartitions { + for s := range claimShards { + stream, durable := "WH_INGEST_"+strconv.Itoa(p), "wh-ingest-"+strconv.Itoa(s) + c, err := srv.Operator.JetStream().Consumer(t.Context(), stream, durable) + require.NoError(t, err) + out[stream+"/"+durable] = c.CachedInfo().NumPending + } + } + return out +} + +// unitState is what the server says of a unit: its pin, and its rows +// delivered and awaiting an ack. +func unitState(t *testing.T, srv *natstest.Server, unit string) (pin string, delivered uint64, ackPending int) { + t.Helper() + stream, durable, _ := strings.Cut(unit, "/") + c, err := srv.Operator.JetStream().Consumer(t.Context(), stream, durable) + require.NoError(t, err) + info := c.CachedInfo() + for _, g := range info.PriorityGroups { + pin = g.PinnedClientID + } + return pin, info.Delivered.Consumer, info.NumAckPending +} + +// An owner whose ClickHouse takes longer than the durable's pinned_ttl (10s) +// to answer, with its unit's share of held rows full, keeps its pin all the +// while: it takes at most one row past its share per 5s renewal, the unit +// never reads as unowned, and no other process may reset it. Once the insert returns, the rest of +// the table's rows follow, in order. +func TestShardClaims_BlockedOwnerKeepsItsShard(t *testing.T) { + srv := natstest.Start(t) + pub := shardBroker(t, srv.URL()) + const rows, share = 20, 8 + unit := unitOf(t, srv, pub, mq.Topic{Tenant: "acme", Table: "stuck"}, seqEnvelope(t, "stuck", 0)) + for i := 1; i < rows; i++ { + require.NoError(t, pub.Publish(t.Context(), mq.Topic{Tenant: "acme", Table: "stuck"}, seqEnvelope(t, "stuck", i))) + } + + entered, unblock := make(chan struct{}), make(chan struct{}) + var first sync.Once + log := &insertLog{} + ch := log.server(t, "proc-a", func(string) { + first.Do(func() { close(entered); <-unblock }) + }) + a := startWorkerProc(t, srv.URL(), "proc-a", 3*time.Second, ingest.ClaimConfig{Every: 100 * time.Millisecond, MaxHeld: share}, ch.URL) + t.Cleanup(func() { closeOnce(unblock) }) // first, should the test end early + select { + case <-entered: + case <-time.After(30 * time.Second): + t.Fatal("the first insert never started") + } + pin, delivered, ackPending := unitState(t, srv, unit) + require.NotEmpty(t, pin) + require.GreaterOrEqual(t, ackPending, share, "the unit's share is full") + + other := shardBroker(t, srv.URL()) + blocked := time.Now() + for time.Since(blocked) < 13*time.Second { + time.Sleep(500 * time.Millisecond) + now, d, _ := unitState(t, srv, unit) + require.Equal(t, pin, now, "the pin moved %s into the block", time.Since(blocked).Round(time.Millisecond)) + require.LessOrEqual(t, d-delivered, uint64(time.Since(blocked)/(5*time.Second))+1, "at most one row past the share per renewal") + if time.Since(blocked) > 11*time.Second { + n, err := a.broker.Unowned(t.Context()) + require.NoError(t, err) + assert.Zero(t, n, "a blocked owner's unit is not unowned") + _, err = other.ResetOrphaned(t.Context(), unit) + assert.ErrorIs(t, err, mq.ErrUnitHeld, "nor may another process reset it") + } + } + _, d, ackPending := unitState(t, srv, unit) + assert.Equal(t, int(d), ackPending, "every row delivered is still the owner's, none reset") + t.Logf("%d rows past the share delivered over %s of renewals", d-delivered, time.Since(blocked).Round(time.Second)) + + blockedFor := time.Since(blocked) + closeOnce(unblock) + resumed := time.Now() + require.Eventually(t, func() bool { return len(log.table("stuck")) == rows }, 30*time.Second, 50*time.Millisecond) + t.Logf("blocked %s with its pin kept throughout; the other %d rows followed %s after the insert returned", + blockedFor.Round(100*time.Millisecond), rows-share, time.Since(resumed).Round(10*time.Millisecond)) + assert.True(t, inOrder(log.table("stuck"), rows), "%v", log.table("stuck")) + noWorkerFailure(t, a) +} + +// A table whose ClickHouse hangs fills only its own unit's share of held +// rows: a table on another unit of the same process keeps flowing. +func TestShardClaims_StuckShardDoesNotStallTheOthers(t *testing.T) { + srv := natstest.Start(t) + pub := shardBroker(t, srv.URL()) + stuck := mq.Topic{Tenant: "acme", Table: "stuck"} + stuckUnit := unitOf(t, srv, pub, stuck, seqEnvelope(t, "stuck", 0)) + for i := 1; i < 20; i++ { + require.NoError(t, pub.Publish(t.Context(), stuck, seqEnvelope(t, "stuck", i))) + } + var healthy mq.Topic + for i := 0; healthy.Table == ""; i++ { + topic := mq.Topic{Tenant: "acme", Table: "healthy" + strconv.Itoa(i)} + if unitOf(t, srv, pub, topic, seqEnvelope(t, topic.Table, 0)) != stuckUnit { + healthy = topic + } + } + + entered, unblock := make(chan struct{}), make(chan struct{}) + var first sync.Once + log := &insertLog{} + ch := log.server(t, "proc-a", func(table string) { + if table == "stuck" { + first.Do(func() { close(entered) }) + <-unblock + } + }) + a := startWorkerProc(t, srv.URL(), "proc-a", 3*time.Second, ingest.ClaimConfig{Every: 100 * time.Millisecond, MaxHeld: 8}, ch.URL) + t.Cleanup(func() { closeOnce(unblock) }) // first, should the test end early + select { + case <-entered: + case <-time.After(30 * time.Second): + t.Fatal("the stuck table's insert never started") + } + _, _, held := unitState(t, srv, stuckUnit) + require.GreaterOrEqual(t, held, 8, "the stuck unit's share is full") + + const more = 30 + for i := 1; i <= more; i++ { + require.NoError(t, pub.Publish(t.Context(), healthy, seqEnvelope(t, healthy.Table, i))) + } + began := time.Now() + require.Eventually(t, func() bool { return len(log.table(healthy.Table)) == more+1 }, 20*time.Second, 50*time.Millisecond, + "the healthy table stalled behind the stuck one: %v", log.table(healthy.Table)) + t.Logf("with %s's share full, %s's %d rows were written %s after they were published", + stuck.Table, healthy.Table, more, time.Since(began).Round(10*time.Millisecond)) + assert.True(t, inOrder(log.table(healthy.Table), more+1)) + assert.Empty(t, log.table("stuck"), "the stuck insert has not returned") + closeOnce(unblock) + noWorkerFailure(t, a) +} + +func closeOnce(ch chan struct{}) { + select { + case <-ch: + default: + close(ch) + } +} + +// Under continuous publishing, every table's rows reach ClickHouse exactly +// once and in order while a second process joins (half the units hand over) +// and the first then stops cleanly (the rest move): the process giving a +// unit up writes what it holds before the next owner receives anything. +// The first process's ClickHouse takes longer than a batch window, so a +// unit released before its rows were written would have the next owner's +// rows land first. And a clean stop leaves nothing unsettled behind, so the +// next owner binds its units at once rather than waiting to judge them +// orphaned. +func TestShardClaims_TableOrderAcrossHandoverAndStop(t *testing.T) { + srv := natstest.Start(t) + pub := shardBroker(t, srv.URL()) + log := &insertLog{} + chA := log.server(t, "proc-a", func(string) { time.Sleep(6 * time.Second) }) + chB := log.server(t, "proc-b", nil) + const lease, tables = 3 * time.Second, 16 + cfg := ingest.ClaimConfig{Every: 100 * time.Millisecond} + + a := startWorkerProc(t, srv.URL(), "proc-a", lease, cfg, chA.URL) + var published atomic.Int64 + quit := make(chan struct{}) + var publisher sync.WaitGroup + publisher.Go(func() { + for seq := 0; ; seq++ { + select { + case <-quit: + return + default: + } + for i := range tables { + table := "t" + strconv.Itoa(i) + if err := pub.Publish(context.Background(), mq.Topic{Tenant: "acme", Table: table}, seqEnvelope(t, table, seq)); err != nil { + t.Errorf("publish %s/%d: %v", table, seq, err) + return + } + } + published.Store(int64(seq + 1)) + time.Sleep(20 * time.Millisecond) + } + }) + + time.Sleep(3 * time.Second) + b := startWorkerProc(t, srv.URL(), "proc-b", lease, cfg, chB.URL) + time.Sleep(15 * time.Second) // membership moves, and each moved unit's rows are written first + stopping := time.Now() + stopCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + require.NoError(t, a.stop(stopCtx)) + stopped := time.Now() + t.Logf("proc-a stopped cleanly in %s", stopped.Sub(stopping).Round(10*time.Millisecond)) + time.Sleep(6 * time.Second) + close(quit) + publisher.Wait() + + rows := int(published.Load()) + require.Eventually(t, func() bool { return log.count() >= rows*tables }, 45*time.Second, 100*time.Millisecond, + "inserted %d of %d rows", log.count(), rows*tables) + time.Sleep(time.Second) // anything written twice would show now + moved := 0 + for i := range tables { + table := "t" + strconv.Itoa(i) + assert.True(t, inOrder(log.table(table), rows), "%s: %d rows, want 0..%d in order", table, len(log.table(table)), rows-1) + log.mu.Lock() + if len(log.writers[table]) > 1 { + moved++ + } + assert.LessOrEqual(t, len(log.writers[table]), 3, "%s changed writer more than a handover and a stop explain: %v", table, log.writers[table]) + log.mu.Unlock() + } + t.Logf("%d rows in each of %d tables; %d tables changed writer", rows, tables, moved) + assert.Positive(t, moved, "some table moved between the processes") + + // The tables proc-a still wrote while stopping moved at the stop: proc-b's + // first write of each follows within its batch window (5s), not after a + // wait for the unit to count as orphaned (pinned_ttl, 10s) on top. + log.mu.Lock() + defer log.mu.Unlock() + var gaps []time.Duration + for i := range tables { + table := "t" + strconv.Itoa(i) + if last, ok := log.last["proc-a"][table]; ok && !last.Before(stopping) { + gaps = append(gaps, log.first["proc-b"][table].Sub(stopped)) + } + } + require.NotEmpty(t, gaps, "some tables moved at the stop") + slices.Sort(gaps) + t.Logf("%d tables moved at the stop; proc-b's first write of each came %s to %s after it", len(gaps), gaps[0].Round(10*time.Millisecond), gaps[len(gaps)-1].Round(10*time.Millisecond)) + assert.Less(t, gaps[len(gaps)-1], 7*time.Second, "a unit the stop gave up waited to be judged orphaned") + noWorkerFailure(t, b) +}