diff --git a/.testcoverage.yml b/.testcoverage.yml index d786004fa..6c8b9ffc6 100644 --- a/.testcoverage.yml +++ b/.testcoverage.yml @@ -112,7 +112,7 @@ exclude: - ^internal/mq/nats_topology\.go$ - ^internal/mq/nats_manifests\.go$ - ^internal/mq/subject_nats\.go$ - - ^internal/mq/external\.go$ + - ^internal/mq/external(_consumer)?\.go$ - ^internal/mq/lease\.go$ - ^cmd/wavehouse/mq\.go$ # Their wiring, apart from wire.go for this: the NATS queue and lease @@ -133,6 +133,6 @@ exclude: # The external NATS broker's tests start a server per case, which the # unit suite's 15s per package cannot hold: they are integration-tagged # (make test-integration), and the merged total counts them. - - ^internal/mq/external\.go$ + - ^internal/mq/external(_consumer)?\.go$ # The NATS KV leases, the same way. - ^internal/mq/lease\.go$ diff --git a/AGENTS.md b/AGENTS.md index 8483759c5..03bafa627 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -38,9 +38,9 @@ Twenty internal packages under `internal/` (plus `internal/testutil/` for shared - **`coord/`** — leases for work that must run in one process at a time (`Observer.Held` reads whether one is held without campaigning): `Coordinator.TryAcquire(ctx, name)` → a `Term` (fencing `Token`, strictly increasing per name; `Done`/`Err`, `ErrLost` on loss; `Resign`), `ErrHeld` while another holder's — or this coordinator's own — term is live; `RunElected` runs a loop only while holding its lease, resigning when the loop returns and campaigning again every `RetryPeriod`. `Local` is the in-process implementation (first taker wins, never expires; `Peer` is a second handle over the same table for tests); every implementation runs `coordtest.Conformance`. Imports only the standard library, so a distributed backend lives beside its connection: `coord.backend: nats` is `internal/mq/lease.go` (`ExternalNATS.Leases`), a key per lease in the operator's KV bucket, the KV revision as the fencing token, and expiry judged on the candidate's own clock (the same revision seen unchanged for 15s), never by a server TTL. `internal/app`'s `wireCoord` opens the one `coord.backend` selects and the sweeper runs through `RunElected` under the `sweeper` lease - **`dedupe/`** — `Deduplicator` interface (two-phase `Reserve`/`Commit`/`Release` over `Key{Table, ID}`; every backend passes the `dedupetest` conformance suite) → `Embedded` (Pebble: every tenant's seen ids in one instance at `data_dir/pebble`, each key led by its tenant and table, pending claims in memory, committed ids stored with their expiry and deleted by an hourly background sweep along with the version-0 keys from before the table joined the key, open while any tenant's store is — the layout is the implementation's call, and the wiring hands it `data_dir` once; its `Stats` feed the system gauges) or `Dynamo` (one shared DynamoDB table, conditional `PutItem` claims; conformance-tested against dynamodb-local, selected by `dedupe.backend: dynamodb`; boot checks the table and never creates it outside dynamodb-local), wrapped by `Managed` whose open/closed state follows the hot-reloadable `dedupe.enabled` in the settings directory's `config.json`; `Stores` holds one `Managed` per tenant, built through a `Factory` (`func(tenant.ID) *Managed`, `Embedded.Tenant` or, gated on the table check (`Factory.Gated`), `Dynamo.Tenant` in production; `Managed` opens its store through a function, so every backend gets the same switch), and reconciled from the registry's `AfterAdopt` hook — open exactly when the tenant is served with its switch on, closed with its seen ids kept otherwise ([#583](https://github.com/Wave-RF/WaveHouse/issues/583) stories 7 and 3) - **`discovery/`** — `SchemaRegistry`, one per served tenant over a `Source` read once per refresh — the tenant's pool's connection and the database that pool was opened for, one snapshot, so a refused move keeps discovering the database the tenant's queries still use (`internal/app`'s `discoveries` builds, runs and stops them from `AfterAdopt` and `App.Close`: `RetryRefresh` until the first success, then `StartAutoRefresh` with a random first tick; `Lookup` answers `ErrNotLoaded` before the first success — the handlers' `503` with `Retry-After` — and `ErrUnknownTable` after; a failed loop attempt counts in `wavehouse_schema_refresh_failures_total{tenant}`), that introspects ClickHouse `system.columns` (name/type/nullability plus `default_expression` and 1-based `position`) and `system.tables` (each table's `create_table_query`, kept in-process and never serialized — an external-engine table renders its wiring there unconditionally — endpoint, bucket/host, database, username, S3 access key id; ClickHouse masks the password as `[HIDDEN]` from ~23.9, so the exposure is the topology, not the secret), records the server version, + `Validate()` for ingest payloads + `CanonicalizeTimestamps()` rewriting top-level `DateTime`/`DateTime64` column values to the canonical RFC 3339 UTC wire form pre-publish (Key Design Decision #19) -- **`ingest/`** — Ingest worker pipeline (`worker.go`: JetStream input → per-table batch INSERT with DLQ output; over a `Sharded` queue, `claims.go`'s `ClaimShards` narrows the worker to the units this process is assigned — membership leases, capped rendezvous, drain-then-release handover, reset at takeover from a dead owner, a 10,000-row cap on unsettled rows). The pipeline is **insert-only**. The wire format `EventMessage` (`types.go`) carries `{table_name, scope, received_timestamp, format, columns, row}` and nothing else — `row` is one positional `JSONCompactEachRow` line and `columns` names its slots, the table's **insertable** columns (a `MATERIALIZED`/`ALIAS` column cannot be named in an `INSERT`); the worker batches per (tenant, table, column list), the tenant read off each message's `mq.Topic`, and inserts each batch into its tenant's own ClickHouse (`chconn.Pools.Target`); the worker accepts whatever table name the envelope carries (table existence was already checked by the HTTP ingest handler, which `404`s an unknown table before publish; the worker doesn't re-validate), then bulk-INSERTs. In the embedded-NATS deployment (the default), the server runs with `DontListen: true` (`internal/mq/embedded.go`), so the only Publishers reachable on the `ingest.>` subjects are in-process Go code — today, only the HTTP `/v1/ingest?table={table}` handler. Non-insert mutations (`DELETE`/`UPDATE`/`TRUNCATE`/…) must go through `POST /v1/ops/query` under the admin role (the same `RequireAdmin` gate as the rest of `/v1/ops/*`, so non-admin callers never reach the proxy) or through an operator-authored pipe that writes, gated only by its `allowed_roles` (#386). A request with no token (or an invalid one) resolves to the `default_role`, which in a production config is not the admin role (setting them equal is a loudly-warned dev-only setting), so it can't reach this endpoint. Plus `Sweeper` (Active Sweeper for NATS message lifecycle) + `EventMessage`/`BufferConsumerName` types (`types.go`) +- **`ingest/`** — Ingest worker pipeline (`worker.go`: JetStream input → per-table batch INSERT with DLQ output; over a `Sharded` queue, `claims.go`'s `ClaimShards` narrows the worker to the units this process is assigned — membership leases, capped rendezvous, halt-drain-then-release handover and stop, reset at takeover from a dead owner, each unit's share of a 10,000-row budget of unsettled rows). The pipeline is **insert-only**. The wire format `EventMessage` (`types.go`) carries `{table_name, scope, received_timestamp, format, columns, row}` and nothing else — `row` is one positional `JSONCompactEachRow` line and `columns` names its slots, the table's **insertable** columns (a `MATERIALIZED`/`ALIAS` column cannot be named in an `INSERT`); the worker batches per (tenant, table, column list), the tenant read off each message's `mq.Topic`, and inserts each batch into its tenant's own ClickHouse (`chconn.Pools.Target`); the worker accepts whatever table name the envelope carries (table existence was already checked by the HTTP ingest handler, which `404`s an unknown table before publish; the worker doesn't re-validate), then bulk-INSERTs. In the embedded-NATS deployment (the default), the server runs with `DontListen: true` (`internal/mq/embedded.go`), so the only Publishers reachable on the `ingest.>` subjects are in-process Go code — today, only the HTTP `/v1/ingest?table={table}` handler. Non-insert mutations (`DELETE`/`UPDATE`/`TRUNCATE`/…) must go through `POST /v1/ops/query` under the admin role (the same `RequireAdmin` gate as the rest of `/v1/ops/*`, so non-admin callers never reach the proxy) or through an operator-authored pipe that writes, gated only by its `allowed_roles` (#386). A request with no token (or an invalid one) resolves to the `default_role`, which in a production config is not the admin role (setting them equal is a loudly-warned dev-only setting), so it can't reach this endpoint. Plus `Sweeper` (Active Sweeper for NATS message lifecycle) + `EventMessage`/`BufferConsumerName` types (`types.go`) - **`keyenc/`** — the one escaping composite keys are built from: `Escape` keeps `[A-Za-z0-9_-]` (exactly the tenant-id grammar, so a tenant id is its own escaped form) and writes every other byte as `%XX`, `Unescape` is `url.PathUnescape` (lenient: either hex case, and a byte left unescaped reads as itself, so a `%2D` an earlier build wrote still reads), `Join`/`AppendJoin` escape each field and put a separator between them (they panic on no fields, and on a separator the escaping could write or one outside ASCII) and `Split` reverses them. The package that builds a key takes raw names and escapes them itself, so no caller has to and no field reaches a key unescaped: NATS subjects (`Join`/`Split` after the verbatim tenant) and the cache's keys — the version index and the shared backend's Redis keys (`internal/cache`) — and the dedupe keys (`//`) use it; changing what it keeps orphans every stored key (an orphaned dedupe key lets a seen id through again), and on the shared backend, whose keys every process builds for itself, splits them between builds for the length of a rolling upgrade (a bump one build makes misses the entries the other filed, served until their TTL) -- **`mq/`** — the message-queue boundary: the **only** package that imports NATS/JetStream (Key Design Decision #20), and the only one that knows how the broker works. Everything else addresses events by `Topic{Tenant, Table, Scope}` (a validated tenant id and raw names — the tenant leads every subject, `ingest..
`, so one wildcard selects a tenant's traffic, and a topic without one is refused) and states intent through the interfaces — `Publisher` (`ErrQueueFull` is the backpressure signal, `ErrUnavailable` a broker that cannot be reached — both a `503`, with `Retry-After` `30` and `5`; `WithIdempotencyKey` makes a republish inside the queue's duplicate window a no-op), `Subscriber`, `ConsumerManager`/`Consumer`/`ConsumerConfig` (the ingest worker's durable consumer), `DeadLetterer` and `DeadLetterStats` (park a message, count what is parked), `Purger` (drop what is both acked and older than a cutoff — the sweeper), `Replayer` (SSE gap-fill) — composed into `Broker`, which adds each tenant's byte budget (`SetMaxBytes`/`MaxBytes`: the `mq.max_bytes_gb` reload, which opens a tenant's queue the first time) and `Stats` (the system gauges' source). A broker whose ingest queue is split into units one consumer at a time owns implements `Sharded` too (`IngestUnits`, `ResetOrphaned`, `Unowned`; `ConsumerConfig.Units` narrows a consumer to some of them, and its consumer is a `Releaser`), and `Message.OnSettled` runs a hook once, at the first ack or nak attempt, confirmed or not. Every interface speaks per tenant, never per stream: the embedded implementation gives each tenant a queue of its own (a stream pair, `INGEST_`/`DLQ_`), and nothing outside the package may assume that layout — an external implementation may keep one shared stream. Subjects, prefixes, wildcards, stream names, sequences, and ack floors are private to the implementations, whose subject tokens are escaped by the shared `internal/keyenc`: `EmbeddedNATS` (`embedded.go`, `subject.go`, `purge.go`, `deadletter.go`), which `internal/app` constructs and hands everything else as a `mq.Broker`, and `ExternalNATS` (`external.go`, `subject_nats.go`, `nats_topology.go`: an operator-owned cluster whose streams, durables and lease bucket it never creates, changes, purges or deletes; `lease.go` holds `coord.backend: nats`'s leases in that bucket), which `internal/app` constructs from the `mq.nats` block when `mq.backend` is `nats` ([#613](https://github.com/Wave-RF/WaveHouse/issues/613)). Every implementation passes the conformance suite in `internal/mq/mqtest` (`mqtest.Run`), which states the `Broker` contract as behavior; a new backend runs it from its own test, with `mqtest.Caps` only where its semantics legitimately differ +- **`mq/`** — the message-queue boundary: the **only** package that imports NATS/JetStream (Key Design Decision #20), and the only one that knows how the broker works. Everything else addresses events by `Topic{Tenant, Table, Scope}` (a validated tenant id and raw names — the tenant leads every subject, `ingest..
`, so one wildcard selects a tenant's traffic, and a topic without one is refused) and states intent through the interfaces — `Publisher` (`ErrQueueFull` is the backpressure signal, `ErrUnavailable` a broker that cannot be reached — both a `503`, with `Retry-After` `30` and `5`; `WithIdempotencyKey` makes a republish inside the queue's duplicate window a no-op), `Subscriber`, `ConsumerManager`/`Consumer`/`ConsumerConfig` (the ingest worker's durable consumer), `DeadLetterer` and `DeadLetterStats` (park a message, count what is parked), `Purger` (drop what is both acked and older than a cutoff — the sweeper), `Replayer` (SSE gap-fill) — composed into `Broker`, which adds each tenant's byte budget (`SetMaxBytes`/`MaxBytes`: the `mq.max_bytes_gb` reload, which opens a tenant's queue the first time) and `Stats` (the system gauges' source). A broker whose ingest queue is split into units one consumer at a time owns implements `Sharded` too (`IngestUnits`, `ResetOrphaned`, `Unowned`; `ConsumerConfig.Units` narrows a consumer to some of them, and its consumer is a `Releaser` and a `Halter`, capped per unit by `ConsumerConfig.MaxHeld`, with `ErrConsumerMismatch` when an operator durable no longer fits), and `Message.OnSettled` runs a hook once, at the first ack or nak attempt, confirmed or not. Every interface speaks per tenant, never per stream: the embedded implementation gives each tenant a queue of its own (a stream pair, `INGEST_`/`DLQ_`), and nothing outside the package may assume that layout — an external implementation may keep one shared stream. Subjects, prefixes, wildcards, stream names, sequences, and ack floors are private to the implementations, whose subject tokens are escaped by the shared `internal/keyenc`: `EmbeddedNATS` (`embedded.go`, `subject.go`, `purge.go`, `deadletter.go`), which `internal/app` constructs and hands everything else as a `mq.Broker`, and `ExternalNATS` (`external.go`, `subject_nats.go`, `nats_topology.go`: an operator-owned cluster whose streams, durables and lease bucket it never creates, changes, purges or deletes; `lease.go` holds `coord.backend: nats`'s leases in that bucket), which `internal/app` constructs from the `mq.nats` block when `mq.backend` is `nats` ([#613](https://github.com/Wave-RF/WaveHouse/issues/613)). Every implementation passes the conformance suite in `internal/mq/mqtest` (`mqtest.Run`), which states the `Broker` contract as behavior; a new backend runs it from its own test, with `mqtest.Caps` only where its semantics legitimately differ - **`observability/`** — OpenTelemetry pipeline: `InitProvider` wires trace/metric/log providers via OTLP gRPC (each signal independently gated). A top-level `Prometheus` config block drives an optional `/metrics` scrape endpoint that runs independently of OTLP push — standalone (Alloy/Mimir scrape, no collector), alongside OTLP, or off. `NewLogger` produces a slog handler that fans out to stdout AND OTLP (stdout always 100%, OTLP sample-rate-aware). `TraceHandler` injects trace_id/span_id from active spans. `tracer.go` provides W3C trace context propagation over message headers (`InjectHeaders`/`ExtractHeaders` on a plain header map; `internal/mq` injects on every publish and extracts on the `Subscribe` path, so this package never sees a NATS type). - **`pipes/`** — Named query pipes: `NamedQuery` type + `BindParams` + `Source` (read per request; `settings.Store` in production, `Static(q...)` in tests) - **`policy/`** — Hasura-style access control, **role-first**: `TablePolicy` is `map[string]RolePermissions`, and a role's grant splits by operation into `SelectPermissions` (columns, row `filter`, aggregations, the `max_*` limits) and `InsertPermissions` (columns, `check`) — so a field only one side honors does not exist on the other. `Evaluate()` resolves ONE operation and leaves the other side **nil** (`Select *ResolvedSelect` / `Insert *ResolvedInsert`), which every accessor fails closed on — nil is "not resolved", distinct from an empty side, which is "unrestricted" (what the admin return builds). Claim templating (`{{ jwt.claim.path }}`) resolves during that call. Policies come from `Source`, a `func() *Policy` read per call (`settings.Store.Policy` in production, `Static(p)` in tests) diff --git a/CHANGELOG.md b/CHANGELOG.md index e456ce02a..389a7e58c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,9 +11,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), ### Added - **`mq.backend: nats` runs WaveHouse on an operator-owned NATS JetStream, so several processes share one queue** (`internal/mq/{external,settle,subject_nats}.go` (new; + integration-tagged tests, among them `external_conformance_test.go`), `internal/mq/{mq,embedded}.go` (+ tests), `internal/config/{backends,config,mq_nats}.go` (`mq_nats.go` new; + tests), `internal/app/{app,wire,wire_nats,wire_ops}.go` (`wire_nats.go` and `wire_ops.go` new; + `mq_nats_test.go`, `retention_warn_test.go`), `internal/settings/store.go` (+ test), `tests/integration/{setup,mq_nats}_test.go`, `Makefile`, `.testcoverage.yml`, `go.mod`, `config.yaml`, `CONTRIBUTING.md`, `AGENTS.md`, `docs/src/content/docs/{deployment,api,architecture,ingest-pipeline,durability,development}.md`, `docs/src/content/docs/{configuration,settings-directory}.mdx`): part of [#613](https://github.com/Wave-RF/WaveHouse/issues/613). `mq.backend` now takes `nats`, configured by a new `mq.nats` block (`WH_MQ_NATS_*`): the server URLs; one of a creds file, an nkey seed file, or a user with a password file (secrets are file paths only); TLS and mutual TLS; a JetStream domain; the subject prefix; the partition and shard counts (`partitions`, and `shards`, default 32, 1 to 256), which must match the operator's topology; the shard durables' base name and the history stream's name; and the connect, publish and topology-wait timeouts. WaveHouse never creates, changes, purges or deletes a stream, a durable or a bucket there: boot connects, waits up to `topology_wait` (60s) for the operator's topology (see the next entry), refuses a NATS server older than 2.14, and refuses to start with every finding at once while the topology is still wrong; it is checked again every five minutes. A publish retried after a lost answer reuses its `Nats-Msg-Id`, so the partition stores it once, and the ack's stream is checked, so a row another stream stored is a topology fault. A full partition or a table at its per-subject cap is `ErrQueueFull`; a broker that does not answer, a lost connection or a deleted partition is `mq.ErrUnavailable`, a `503` with `Retry-After: 5`. A deleted configured shard durable, or a connection closed for good, ends the worker and the process, so the next boot names what is missing. SSE replay and each API process's live events read the history stream through short-lived consumers of their own. The history is best effort: a row republished while it is unavailable reaches ClickHouse but not SSE, and a full or missing history never holds up ingest. Dead-letter counts are one subject-filtered read of the shared dead-letter stream, keyed by table (the fold the embedded broker now uses too), and zero for a tenant that never parked a row. A tenant's `mq.max_bytes_gb` is not applied, and boot says so. **An ingest `200` rests on the stream's replicas:** it follows the partition's ack, which at 3 replicas comes once a Raft quorum has stored the event, so WaveHouse does not require `sync_always`; the topology check recommends 3 replicas on every stream (so a one-server development cluster still boots) and refuses `persist_mode: async` on a partition. The embedded broker still `fsync`s every event. **There is no sweeper under `nats`:** the streams' retention does its work, so it is not wired and takes no lease, and a process whose only role is `sweeper` is refused. The API processes instead warn once per tenant and window about a gap window longer than the history's `max_age`, at boot, after every reload, and when the periodic check reads a changed `max_age`. Every partition's `duplicate_window` must cover every attempt of one publish, and `dedupe.lease` twice over plus a second, or boot refuses, and a tenant with dedupe on whose finite retention is under that window is logged at `WARN`. Lowering N loses no rows: the ingest worker also drains the shard durables left on any stream holding ingest subjects outside the N partitions (the generated `preventDelete` keeps them), and boot counts the rows each holds; a stream with no shard durable is only reported. Process `roles` can be split on it: an `ingest` process needs `coord.backend: nats`, and `api` and `ingest` in separate processes need `cache.backend: redis` too. Three gauges report on the broker: `wavehouse_mq_connected`, `wavehouse_mq_topology_ok` and `wavehouse_mq_history_behind_seconds`. The `mqtest` conformance suite passes against it, connected as the shipped restricted `wavehouse` user. `internal/mq/natstest` stands NATS up from the shipped Helm values and manifests for tests outside `internal/mq`, and `make test-integration` runs the external backend's tests, since each starts a NATS server. The deployment guide gains an External NATS section: the topology and how to create it, durability, permissions, the limits that differ from the embedded queue, choosing and changing N and V, scaling ingest processes, the manual checks before production, and monitoring. -- **The JetStream topology an external NATS must provide, `wavehouse mq manifests` and `wavehouse mq permissions`, and a check for it** (`internal/mq/{nats_topology,nats_manifests}.go` (new; + tests), `internal/mq/natstest/` (new), `internal/mq/natsspike/` (new, integration-tagged), `cmd/wavehouse/{main,mq}.go` (`mq.go` new; + test), `deployments/nats/{jetstream.yaml,values.yaml}` (new), `docs/src/content/docs/deployment.md`): part of [#613](https://github.com/Wave-RF/WaveHouse/issues/613). The operator owns every stream, durable and bucket: N ingest partitions of V shards each, a history stream, one dead-letter stream (`.dlq.>`) and the lease bucket (`_coord`). A row's subject is `.ingest.

...

`, its partition and shard chosen by independent jump consistent hashes of the tenant and table, so changing N or V moves only about 1/N or 1/V of the tables, and a tenant's tables spread over the partitions. Each partition has work-queue retention (a row stays until the ingest worker acks it, even while no consumer covers it, and is deleted then, so one tenant's unwritten rows never hold back another's) and republishes every row to `.hist..
`, which the history stream captures: it holds `.hist.>`, has no sources, and keeps `15m` in the generated manifests, the seed's `stream.gap_window_minutes`. Each shard has its own durable, `-`, with the `pinned_client` priority policy in the group `wavehouse` and a pinned TTL of at least twice WaveHouse's 5-second pull expiry, so the server delivers a shard to one process at a time. `wavehouse mq manifests` (`--partitions`, `--shards`, `--prefix`, `--replicas`, `--coord-bucket`, `--dedupe-lease`) prints the whole topology as nack resources; `deployments/nats/jetstream.yaml` is its output for four partitions of eight shards, and `deployments/nats/values.yaml` is a NATS Helm chart snippet for a three-node cluster with a `wavehouse` user. `wavehouse mq permissions --shards V` prints that user's permissions: it may publish, read stream and consumer info, pull from, unpin and reset each shard durable, named one by one because a NATS wildcard is a whole token, and read and write the lease keys; it may not create, change, purge or delete a stream or a bucket, create a durable on a partition, or publish to the history. The verifier checks a live server against the same spec and reports every mismatch at once, `required` ones refusing boot and `recommended` ones logged. NATS 2.14 or later is required, the first release with the consumer reset that shard takeover uses. Tests pin the JetStream behavior this rests on against nats-server 2.14.6 (`internal/mq/natsspike`). Two things are left to a manual check on the operator's own cluster, listed in the deployment guide: that nack applies every generated field, and how a three-replica cluster behaves when a shard durable's or a partition's leader is killed, and under load with every shard's consumer. +- **The JetStream topology an external NATS must provide, `wavehouse mq manifests` and `wavehouse mq permissions`, and a check for it** (`internal/mq/{nats_topology,nats_manifests}.go` (new; + tests), `internal/mq/natstest/` (new), `internal/mq/natsspike/` (new, integration-tagged), `cmd/wavehouse/{main,mq}.go` (`mq.go` new; + test), `deployments/nats/{jetstream.yaml,values.yaml}` (new), `docs/src/content/docs/deployment.md`): part of [#613](https://github.com/Wave-RF/WaveHouse/issues/613). The operator owns every stream, durable and bucket: N ingest partitions of V shards each, a history stream, one dead-letter stream (`.dlq.>`) and the lease bucket (`_coord`). A row's subject is `.ingest.

...

`, its partition and shard chosen by independent jump consistent hashes of the tenant and table, so changing N or V moves only about 1/N or 1/V of the tables, and a tenant's tables spread over the partitions. Each partition has work-queue retention (a row stays until the ingest worker acks it, even while no consumer covers it, and is deleted then, so one tenant's unwritten rows never hold back another's) and republishes every row to `.hist..
`, which the history stream captures: it holds `.hist.>`, has no sources, and keeps `15m` in the generated manifests, the seed's `stream.gap_window_minutes`. Each shard has its own durable, `-`, with the `pinned_client` priority policy in the group `wavehouse` and a pinned TTL of at least twice the 5 seconds between the pulls that keep the pin of a shard at its cap of held rows, or stopping, so the server delivers a shard to one process at a time. `wavehouse mq manifests` (`--partitions`, `--shards`, `--prefix`, `--replicas`, `--coord-bucket`, `--dedupe-lease`) prints the whole topology as nack resources; `deployments/nats/jetstream.yaml` is its output for four partitions of eight shards, and `deployments/nats/values.yaml` is a NATS Helm chart snippet for a three-node cluster with a `wavehouse` user. `wavehouse mq permissions --shards V` prints that user's permissions: it may publish, read stream and consumer info, pull from, unpin and reset each shard durable, named one by one because a NATS wildcard is a whole token, and read and write the lease keys; it may not create, change, purge or delete a stream or a bucket, create a durable on a partition, or publish to the history. The verifier checks a live server against the same spec and reports every mismatch at once, `required` ones refusing boot and `recommended` ones logged. NATS 2.14 or later is required, the first release with the consumer reset that shard takeover uses. Tests pin the JetStream behavior this rests on against nats-server 2.14.6 (`internal/mq/natsspike`). Two things are left to a manual check on the operator's own cluster, listed in the deployment guide: that nack applies every generated field, and how a three-replica cluster behaves when a shard durable's or a partition's leader is killed, and under load with every shard's consumer. - **`coord.backend: nats` holds leases in a KV bucket on the external NATS, which the ingest processes share** (`internal/mq/lease.go` (new; + integration-tagged `lease_test.go`), `internal/mq/{nats_topology,nats_manifests}.go` (+ tests), `internal/mq/natstest/natstest.go`, `internal/config/{backends,config}.go` (+ `coord_nats_test.go`, tests), `internal/app/{app,wire}.go`, `cmd/wavehouse/mq.go` (+ test), `deployments/nats/{jetstream.yaml,values.yaml}`, `tests/integration/{coord_nats,mq_nats}_test.go`, `Makefile`, `.testcoverage.yml`, `config.yaml`, `docs/src/content/docs/{deployment,architecture}.md`, `docs/src/content/docs/configuration.mdx`, `AGENTS.md`): part of [#613](https://github.com/Wave-RF/WaveHouse/issues/613), the first distributed `coord.Coordinator`. `ExternalNATS.Leases` keeps each lease as a key (`lease.`) in a KV bucket the operator creates, reached over the `mq.nats` connection and credentials; the KV revision a term was taken at is its fencing token. A candidate takes another holder's lease only after seeing the same revision unchanged for 15 seconds on its own clock, so no two servers' clocks are compared and the bucket needs no per-key TTL; the holder renews every 2 seconds and steps down after 10 without a renewal, before anyone can take over, and a clean stop deletes the key so the next holder takes over at once. Boot refuses a process running the `ingest` role with `mq.backend: nats` and `coord.backend: local`, and `coord.backend: nats` without `mq.backend: nats`. The bucket, `_coord` (`wh_coord`; `coord.nats.bucket` / `WH_COORD_NATS_BUCKET` names another), is part of the topology: `wavehouse mq manifests` prints it as a nack `KeyValue` (`--coord-bucket` renames it), boot waits for it with the streams and refuses while it is missing, the periodic check reports it on `wavehouse_mq_topology_ok`, and the shipped `wavehouse` user may read and write `lease.` keys in it and nothing else there. -- **Ingest processes share the external NATS shards out between them** (`internal/ingest/{claims,assign}.go` (new; + tests), `internal/coord/{coord,local}.go`, `internal/coord/coordtest/coordtest.go`, `internal/mq/{lease,external,mq}.go` (+ tests), `internal/app/wire.go`, `tests/integration/{shard_claims,coord_nats}_test.go`, `docs/src/content/docs/{deployment,architecture,ingest-pipeline}.md`, `docs/src/content/docs/configuration.mdx`, `AGENTS.md`): part of the external-NATS workstream of [#613](https://github.com/Wave-RF/WaveHouse/issues/613). Under `mq.backend: nats` each ingest process holds a membership lease (`ingest.m`) in the lease bucket, and every process gives every shard to one live member by rendezvous hashing capped at an even share, so no process owns more than an even share, rounded up, and a process joining or leaving moves roughly one to two times its share while that share holds (several times it where the share steps down, never more than half the shards). At most 64 processes hold a membership slot, read 16 at a time; leftover shard durables are assigned apart from the configured ones. While a rollout changes N or V, processes on the old and new counts share the shards out differently, so some may have no owner until it ends. A shard given up is drained, then its pin released, so its next owner receives at once; a clean stop releases every shard. A shard whose owner died is reset to its ack floor before its new owner pulls it (waiting, unbound, until the dead owner's pin lapses), so the rows the dead process held come back at once instead of after `ack_wait`; a lone process restarting after a clean stop does the same on its first pass. Rows a process holds across its shards are capped at 10,000 (a row stops counting at its first ack or nak attempt, or after `ack_wait`), the worker's `maxAckPending`, where a shard durable's own `max_ack_pending` of 10,000 would otherwise let a process owning 32 shards hold 320,000; a per-shard `max_ack_pending` of 10,000/V was not chosen, since at V=32 it is below the worker's batch of 500 and a busy table's batch could never fill. New metrics: `wavehouse_ingest_shards_owned`, `wavehouse_ingest_shard_members`, `wavehouse_ingest_shards_unowned` (the lowest-ranked member only), `wavehouse_ingest_rows_held`, `wavehouse_ingest_rows_held_waits_total`, `wavehouse_ingest_shard_events_total` and `wavehouse_ingest_shard_handover_seconds`. `coord.Observer` reads whether a lease is held without campaigning; the broker's side is `mq.Sharded` (`IngestUnits`, `ResetOrphaned`, which answers `ErrUnitHeld` while a consumer still holds the unit, and `Unowned`) and `mq.Releaser`. +- **Ingest processes share the external NATS shards out between them** (`internal/ingest/{claims,assign}.go` (new; + tests), `internal/coord/{coord,local}.go`, `internal/coord/coordtest/coordtest.go`, `internal/mq/{lease,external,mq}.go` (+ tests), `internal/app/wire.go`, `tests/integration/{shard_claims,coord_nats}_test.go`, `docs/src/content/docs/{deployment,architecture,ingest-pipeline}.md`, `docs/src/content/docs/configuration.mdx`, `AGENTS.md`): part of the external-NATS workstream of [#613](https://github.com/Wave-RF/WaveHouse/issues/613). Under `mq.backend: nats` each ingest process holds a membership lease (`ingest.m`) in the lease bucket, and every process gives every shard to one live member by rendezvous hashing capped at an even share, so no process owns more than an even share, rounded up, and a process joining or leaving moves roughly one to two times its share while that share holds (several times it where the share steps down, never more than half the shards). At most 64 processes hold a membership slot, read 16 at a time; leftover shard durables are assigned apart from the configured ones. While a rollout changes N or V, processes on the old and new counts share the shards out differently, so some may have no owner until it ends. A shard given up stops fetching while keeping its pin, waits until the rows it delivered are written, then releases the pin, so its next owner receives at once and only newer rows; a clean stop does the same for every shard around the worker's final flush. A shard whose owner died is reset to its ack floor before its new owner pulls it (waiting, unbound, until the dead owner's pin lapses and the shard has been idle for the pinned TTL), so the rows the dead process held come back at once instead of after `ack_wait`; a lone process restarting after a clean stop does the same on its first pass. Each shard may hold unsettled rows up to an even share of the worker's 10,000 (`maxAckPending`) over the shards its process holds, and never under 1,000, two batches (a row stops counting at its first ack or nak attempt, or after `ack_wait`), where a shard durable's own `max_ack_pending` of 10,000 would let a process owning 32 shards hold 320,000; a per-shard `max_ack_pending` of 10,000/V was not chosen, since at V=32 it is below the worker's batch of 500 and a busy table's batch could never fill. New metrics: `wavehouse_ingest_shards_owned`, `wavehouse_ingest_shard_members`, `wavehouse_ingest_shards_unowned` (the lowest-ranked member only), `wavehouse_ingest_rows_held`, `wavehouse_ingest_shard_events_total` and `wavehouse_ingest_shard_handover_seconds`. `coord.Observer` reads whether a lease is held without campaigning; the broker's side is `mq.Sharded` (`IngestUnits`, `ResetOrphaned`, which answers `ErrUnitHeld` while a consumer holds the unit or was active on it within its pinned TTL, and `Unowned`), `mq.Releaser` and `mq.Halter`. - **A multi-process test of the split roles on shards** (`tests/integration/{roles,setup}_test.go`): part of the external-NATS workstream of [#613](https://github.com/Wave-RF/WaveHouse/issues/613). `TestRoles_SeparateProcesses` runs the real binary as separate processes, two with `roles=api` and two with `roles=ingest`, over an operator-provisioned NATS (the shipped four partitions of eight shards), a Redis cache and a dynamodb-local dedupe table. It checks that the two ingest processes split the 32 shards evenly with none unowned, that ingest lands in ClickHouse exactly once, that cache invalidation and SSE reach both API processes, and that a duplicate id is accepted once. A third ingest process then takes its share by handover, and one killed with `SIGKILL` has its shards taken over about one membership lease later, with every batch still written once. `TestRoles_BootRefusesWhatTheBackendsCannotServe` drives the boot rules through the binary, including an `ingest` process on nats with local leases and a sweeper-only process on nats. - **`dedupe.backend: dynamodb` selects the shared DynamoDB dedupe table** (`internal/config/{backends,config}.go` (+ tests), `internal/app/{app,wire,wire_dynamodb}.go` (+ `dedupe_dynamodb_test.go`), `internal/dedupe/{stores,dynamodb}.go` (+ tests), `tests/integration/dedupe_dynamodb_app_test.go` (new), `.testcoverage.yml`, `config.yaml`, `AGENTS.md`, `docs/src/content/docs/{configuration.mdx,settings-directory.mdx,deployment.md,architecture.md,api.md,sdk/reference.md}`): PR F5 of [#613](https://github.com/Wave-RF/WaveHouse/issues/613). Pods that set it share seen ids, so an id ingested through one is a duplicate through every other. New boot keys: `dedupe.lease` (`WH_DEDUPE_LEASE`, `30s`, how long a claimed id stays pending and the in-flight `503`'s `Retry-After`; at most `59s` with the embedded queue, so that the lease plus its own ceiling to the next second plus one more second fits its 2-minute duplicate window: a client obeying that `Retry-After` after an uncertain publish can republish as late as the lease plus that ceiling plus a second after the claim, since DynamoDB rounds a claim's expiry up to the second), `dedupe.reserve_concurrency` (`WH_DEDUPE_RESERVE_CONCURRENCY`, `64`, which also sizes the DynamoDB client's idle connections per host; ingest sends a window of up to 256 ids per call), and the `dedupe.dynamodb` block (`table` (required), `region`, `endpoint`, `timeout` `250ms`, `max_attempts` `3`, `retry_mode` `standard`/`adaptive`, `create_table`), each with its `WH_DEDUPE_DYNAMODB_*` variable. Their defaults are in `defaults()` like every boot key's, so an explicit `0` lease, concurrency, timeout or attempt count, or an empty `retry_mode`, refuses boot rather than becoming the default (the `dynamodb` block's only while `dynamodb` is selected). Credentials come from the AWS SDK's default chain, never from config. Boot checks the table (key schema `pk` String alone; TTL off on `ex` is a warning) in a process running the `api` role, the one that opens the dedupe stores, whether or not a tenant has dedupe on: a misconfigured table (missing, the wrong key schema, access denied) refuses boot over a flat settings directory whose tenant has dedupe on and is logged at `ERROR` otherwise; any other failure (a throttle, a timeout, the network), a nested directory, or no tenant deduping yet boots and fails every switched-on tenant's ingest closed until the check passes, retried in the background (1s backing off to 30s) and at once after every reload. A reload makes no table call and does not wait on a tenant whose dedupe setting is unchanged: it holds the lock that serializes reloads, so it applies each tenant's switch against the last check's result and only wakes the retry; it waits only for a tenant whose store it closes — dedupe switched off, or the tenant removed or rejected — and then only for that tenant's in-flight calls, before its store closes. No region from the config or the SDK chain refuses boot. `create_table` creates a missing table at boot (an endpoint not up yet is a transient failure, retried like the check) and is refused unless `endpoint` is set, so it only ever reaches dynamodb-local. - **A DynamoDB dedupe backend** (`internal/dedupe/dynamodb.go` (new, + tests), `tests/integration/{setup,dedupe_dynamodb}_test.go`, `go.mod`, `AGENTS.md`, `docs/src/content/docs/{architecture,deployment}.md`): PR F3 of [#613](https://github.com/Wave-RF/WaveHouse/issues/613). `dedupe.Dynamo` keeps every tenant's seen ids in one shared table, so pods that share the table also share seen ids, which the per-process Pebble store cannot do. The table's only key is a String `pk` holding the readable dedupe key (`acme/clicks/evt-123`), so an item reads as-is in the console. `Reserve` is a conditional `PutItem` that is atomic across pods; an SDK retry of a put DynamoDB applied but whose answer was lost (a `500`, a reset connection) finds its own item by token and keeps the claim, rather than answer `InFlight` and hold the id for the lease. `Commit` is `BatchWriteItem`, retrying for up to eight jittered rounds both the items DynamoDB leaves unprocessed and a batch it throttled whole, and `Release` is a `DeleteItem` conditional on the claim's token. A client that disconnects mid-`Reserve` does not strand its claim: puts already sent run to their answer on a context its cancellation does not reach, and are then released, so its retry is not answered `InFlight` for the lease. Only a put cut off by its own call timeout (which DynamoDB may apply after the release), or a release that fails, still holds its id until the lease ends. An expired item counts as absent without waiting for TTL. Each call has a 250 ms timeout covering the SDK's three attempts, whose retries back off with full jitter under a ceiling that keeps their waits within half the timeout, so a throttled call fails with the throttle as its cause rather than on the deadline. Throttling, timeouts and an unreachable table wrap `ErrUnavailable`, and five such failures in a row within a second short-circuit claims for a second. Credentials come from the AWS SDK's default chain. The HTTP client keeps one idle connection per host for each of the 64 calls a `Reserve`, `Commit` or `Release` runs at once (the SDK's default keeps 10), so a warm 64-key `Reserve` reuses every connection rather than open about 50. WaveHouse never creates the production table: `CreateTable` works only against dynamodb-local, and the Deployment page carries an example Terraform table and IAM policy. The backend passes the `dedupetest` conformance suite against a pinned `amazon/dynamodb-local` container, along with 32 clients racing one id, injected throttles and an unreachable endpoint. New metrics: `wavehouse_dedupe_dynamodb_requests_total`, `_request_duration_seconds`, `_unprocessed_items_total`, `_short_circuits_total`. `dedupe.backend: dynamodb` selects it (entry above). New dependencies: `aws-sdk-go-v2` (`service/dynamodb`, `config`) and what they require. @@ -95,6 +95,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), ### Fixed +- **A shard's owner keeps it while its ClickHouse is slow, one stuck shard no longer stalls the process's others, and a handover or clean stop keeps each table's rows in order** (`internal/mq/external_consumer.go` (new, from `external.go`), `internal/mq/{external,mq}.go`, `internal/ingest/{claims,worker}.go`, `.testcoverage.yml`, tests in `internal/mq/external_test.go`, `internal/ingest/{claims,worker}_test.go` and `tests/integration/shard_order_test.go` (new), `docs/src/content/docs/{deployment,ingest-pipeline,architecture}.md`, `AGENTS.md`): part of the external-NATS workstream of [#613](https://github.com/Wave-RF/WaveHouse/issues/613), fixing the shard ownership of the entries above before they ship. The pull consumer ran the worker's handler on its delivery goroutine and pulled again only when it returned, and the server renews a shard's pin only on a pull, so a process at its cap of held rows for longer than the pinned TTL (10 seconds) lost its pins: `wavehouse_ingest_shards_unowned` counted its shards, and a peer that judged it dead reset them, redelivering rows it still held — two writers per table. Each shard now has a puller that never runs the handler, fetches only what the shard's cap leaves room for, and at the cap renews the pin every 5 seconds with a one-row fetch, at most one `ack_wait` of such rows past the cap, 12 at the generated 1 minute (past that, and once halted, with a pull of `max_bytes` 1, which delivers nothing); measured, an owner blocked 13 seconds on a hung insert kept one pin throughout and took 2 rows past its share, and a unit drains as fast as before (160,000–192,000 rows a second, against 158,000–172,000). `ResetOrphaned` and `Unowned` also require that the shard delivered nothing and had no ack for its pinned TTL. The process-wide cap of 10,000 held rows let one stuck table take every slot and stall every shard the process owned; each shard now holds its own share, plus at most the 12 rows its renewals take (see the entry above), and with one shard's share full a healthy shard's rows kept flowing. A clean stop now halts every shard first (`mq.Halter`), keeping the pins, so the worker writes what it holds and what the shards still deliver, and releases them after its final flush; rows delivered during that flush were dropped before, and came back only once they had been waited out. A handover keeps the pin until the rows delivered are written, bounded by the worker's 60-second ack wait instead of 15 seconds, and a row the worker drops while stopping is NAKed before the release. Checked under continuous publishing, with the first process's ClickHouse slower than a batch window: every table's rows arrived exactly once and in order across a handover and a clean stop, and the next owner wrote its first rows about 2 seconds after the stop, against 9 seconds when the stop leaves rows behind. A shard fetches in 1-second pulls, so a shard stops fetching within a second of a halt, and an idle shard costs one pull a second. A shard durable deleted while no pull of it was waiting stalled silently until the five-minute topology check, since the next pull only got no responders: two such pulls in a row, or one renewal, now look the durable up, and one that is gone, or whose stream is, ends the worker (measured: 5 seconds after the delete, with the handler busy). A bind whose durable or stream is gone, or whose durable no longer fits (`mq.ErrConsumerMismatch`: `ack_wait` shorter than asked, or no `max_ack_pending`), ends the worker instead of retrying every tick with a warning, and a bind failure that lasts a minute logs an error. `wavehouse_ingest_rows_held_waits_total` is gone: nothing waits at the cap any more. A renewal at the cap takes a row rather than holding it back: a held-back redelivery goes to the back of the server's redelivery queue, and with max_bytes-1 renewals rows NAKed as 0, 1, 2, 3 came back as 2, 3, 0, 1 (measured); now in order, until a stuck shard is 12 rows past its cap. The server requeues the same way while another process's pull waits for the pin, so rows redelivered across a handover or a stop may come back out of order among themselves, though still ahead of newer rows. The unpin on release names no pin id, so it is preceded by a check that the pin is still this process's; the unit renews its pin until the unpin returns (for up to `ack_wait` and one 5-second renewal after its halt has handed on what it fetched, longer than a handover waits), so the pin cannot lapse between the two, and only a consumer leader change there could move it. + - **One tenant's failed queue join no longer ends ingest for every tenant** (`internal/mq/{mq,embedded}.go` (+ tests), `docs/src/content/docs/{architecture,durability,ingest-pipeline}.md`, `settings-directory.mdx`): closes [#675](https://github.com/Wave-RF/WaveHouse/issues/675). When a tenant's queue opened while the server ran and the ingest worker's consumer could not join it, the broker reported that as the consumer's delivery ending, so the worker failed and the process restarted — one tenant's failure taking ingest down for all of them — while the stream hub's consumer failing the same way was only logged, and the tenant's `GET /v1/stream` connections got no live rows until a restart. A queue a consumer cannot join is now left unrecorded as open, whichever consumer it is: `SetMaxBytes` returns the error and `MaxBytes` keeps not reporting the budget, so the next reload tries the consumers that had not joined again, and the tenant's publishes answer `503` and try them too, paced as a queue that cannot open already is (one shared attempt, then refusals for five seconds). No row is accepted that a consumer would not read, no other tenant is touched, and nothing restarts. A queue reopened after its ingest stream went missing has its consumers' delivery started again on the new stream, where the stopped delivery on the old one used to stand in for it. A consumer that had joined and whose delivery then ends on its own (a deleted durable, a closed connection) still fails the worker, and boot is unchanged: a flat directory whose queue cannot open refuses boot. - **Dedupe claims an id, publishes, then commits it — and keys it by tenant, table and id** (`internal/dedupe/{dedupe,key,embedded,managed}.go` (+ tests), `internal/dedupe/dedupetest/` (new), `internal/api/ingest.go` (+ tests), `internal/settings/validate_test.go`, `internal/keyenc/keyenc.go`, `internal/testutil/mocks.go`, `internal/app/app_test.go`, `AGENTS.md`, `docs/src/content/docs/{architecture,api,deployment,development}.md`, `settings-directory.mdx`, `sdk/reference.md`): `CheckAndMark` is replaced by a two-phase `Reserve` → `Commit` / `Release` contract with a lease on the pending claim, and every backend now runs one conformance suite. Four bugs go with it. Two concurrent requests carrying one id no longer both publish it: Pebble's check and claim happen under one lock, and the loser answers `503` with `Retry-After` while the winner is still publishing ([#390](https://github.com/Wave-RF/WaveHouse/issues/390)). A publish that fails gives its id back, so the retry a `503` asks for is published instead of skipped as a duplicate of a record that never reached the queue ([#384](https://github.com/Wave-RF/WaveHouse/issues/384)) — the residual case is a publish that fails after it already reached the broker (a timeout, a disconnect), where the released id lets the retry through but that retry publishes a genuine second copy; [#629](https://github.com/Wave-RF/WaveHouse/pull/629) closes that with an idempotency key. The same id in two tables is two ids ([#222](https://github.com/Wave-RF/WaveHouse/issues/222)'s keyspace half). An explicit `null` id is a missing id — rejected under `require_id`, published un-deduped otherwise — instead of the one id `""` that made every null record after the first a duplicate ([#370](https://github.com/Wave-RF/WaveHouse/issues/370)). **Upgrade:** the key layout changes, so an id seen before the upgrade is accepted once more after it; nothing is migrated, and the old keys are left in `/pebble`, unread, deleted by the retention sweep (see Added) ([#220](https://github.com/Wave-RF/WaveHouse/issues/220)) ([Deployment → Upgrading across the dedupe key change](https://github.com/Wave-RF/WaveHouse/blob/main/docs/src/content/docs/deployment.md#upgrading-across-the-dedupe-key-change)). The key is readable text, `/
/` (for example `acme/clicks/evt-123`), with the table and id escaped and joined by `internal/keyenc`, the escaping NATS subject tokens already use, so any table name gets a keyspace of its own, including one holding a NUL byte or a `/`. New metrics: `wavehouse_ingest_dedupe_commit_failed_total` (a published record whose id failed to commit; the claim lapses with its lease) and `wavehouse_dedupe_hashed_id_total` (an id over 1,024 bytes once escaped, stored as its SHA-256). - **Ingest runs in windows of 256 records over the dedupe contract, and a dedupe store that cannot answer is a `503`** (`internal/api/ingest.go` (+ tests), `internal/mq/{mq,embedded}.go` (+ tests), `internal/dedupe/key.go` (+ tests), `internal/testutil/mocks.go`, `AGENTS.md`, `docs/src/content/docs/{api,architecture,durability}.md`, `settings-directory.mdx`, `sdk/reference.md`): each window of a request is prepared, then reserved in one dedupe call, published in order, and committed in one call, so a batch costs one dedupe round trip per phase per window rather than per record — on Pebble, one commit `fsync` per window (a 1,000-record batch: four syncs instead of a thousand, 24 ms against 5.7 s of dedupe time measured with the queue stubbed). Every deduped record is published under an idempotency key (`mq.WithIdempotencyKey`, JetStream's message id, derived by `dedupe.IdempotencyKey`), and each tenant's ingest stream now keeps an explicit two-minute duplicate window, sized to `2 × the 30-second lease + 1s`: an uncertain publish's `503` sends the *full* lease as `Retry-After`, so an obedient client's retry can land up to ~2×lease after the original request, and the `+1s` covers a backend whose claim expiry itself rounds up by that much. That closes the last path of [#384](https://github.com/Wave-RF/WaveHouse/issues/384): a publish that fails with an unknown outcome (anything but a full queue) keeps its record's claim until the lease lapses instead of releasing it, and a retry after the lease but within two minutes of the first publish is dropped by the queue if the first copy was stored (a later one is stored again). A dedupe store that is not open or that reports `dedupe.ErrUnavailable` now answers `503 {"error":"dedupe store unavailable"}` with `Retry-After: 5`, which the SDK retries, rather than `500 dedupe failed`. A mid-body read error or dedupe failure now drops the open window unpublished, where records before it used to be published; `wavehouse_ingest_dedupe_commit_failed_total` and `wavehouse_ingest_dedupe_disabled_total` count records, as before, now added a window at a time. diff --git a/docs/src/content/docs/architecture.md b/docs/src/content/docs/architecture.md index 3d3456f14..d140ef2c2 100644 --- a/docs/src/content/docs/architecture.md +++ b/docs/src/content/docs/architecture.md @@ -167,18 +167,18 @@ The SSE fan-out, factored out of `api/` so the delivery hot path ([#294](https:/ - **backoff.go** — The retry backoff behind `retryLater`: a small circuit breaker per ClickHouse pool (the target's URL, user and database), and one per pool and table for a failure of one table (`chconn.TableScoped`). A failure opens it for 1 s, doubling to a 30 s cap, each window jittered down to half; while it is open, flushes and arriving rows are handed back without a request, and once it elapses one flush probes. Any answer that is not an outage closes it. - **types.go** — `EventMessage` struct (TableName, Scope — reserved, always empty today, ReceivedTimestamp, Format, Columns, Row; `Format` is `FormatJSONCompactEachRow` and `Row` is one positional line whose slots `Columns` names) and `BufferConsumerName` constant, shared across API handlers and the ingest pipeline. - **compact.go** — `EncodeCompactRow`, the positional row encoder every published row goes through, rendering one record over the table's **insertable** columns in declaration order. Serialization only: it validates nothing and judges no value. -- **claims.go**, **assign.go** — `ClaimShards` wraps a `Sharded` queue so an ingest process consumes only its share of the units, one process per unit at a time. Each process holds one membership lease, `ingest.m` for the lowest free `j` under the number of configured units (at most 64, read 16 at a time each tick), and every tick (2s) reads which slots are live (`coord.Observer.Held`); `assignUnits` gives every unit to a live slot by rendezvous hashing capped at ⌈units/slots⌉, the same result in every process for the same view; the extras are assigned apart from the configured units, so processes whose lists of extras differ (each refreshes it every five minutes) still agree on every configured unit. A unit no longer assigned stops fetching, waits (bounded by 15s) for the rows it delivered to settle (`Message.OnSettled`), then releases its pin (`mq.Releaser`). A unit taken over from a slot that is no longer live is reset (`ResetOrphaned`) before it is bound — waiting, unbound, while the dead owner's pin has not lapsed (`ErrUnitHeld`), up to 30s — so the dead owner's unacked rows come back at once; on a process's first tick only if it is the one live member (a restart after a clean stop; after a crash the dead run's lease still counts as live for a lease duration). Rows delivered across every unit held are capped at the worker's 10,000 (`ClaimConfig.MaxHeld`): at the cap a unit's delivery waits. A row frees its slot at its first ack or nak attempt (`Message.OnSettled` fires whether or not the broker confirms), or after `ack_wait` if the worker never settles it, since the broker redelivers it then as a new row. A configured unit whose delivery ends, or whose durable is gone when it is bound, fails the worker; any other bind failure is retried next tick, and an extra's end is logged. The lowest live slot counts the units with rows and no owner (`Sharded.Unowned`) for `wavehouse_ingest_shards_unowned`. +- **claims.go**, **assign.go** — `ClaimShards` wraps a `Sharded` queue so an ingest process consumes only its share of the units, one process per unit at a time. Each process holds one membership lease, `ingest.m` for the lowest free `j` under the number of configured units (at most 64, read 16 at a time each tick), and every tick (2s) reads which slots are live (`coord.Observer.Held`); `assignUnits` gives every unit to a live slot by rendezvous hashing capped at ⌈units/slots⌉, the same result in every process for the same view; the extras are assigned apart from the configured units, so processes whose lists of extras differ (each refreshes it every five minutes) still agree on every configured unit. A unit no longer assigned halts (`mq.Halter`: stops fetching, keeps its pin, returns once what it fetched reached the worker), waits (bounded by the worker's 60s ack wait, the shortest `ack_wait` a durable may have) for the rows it delivered to settle (`Message.OnSettled`), then releases its pin (`mq.Releaser`). `Halt` on the claiming consumer ends the ticks and halts every unit at once; the worker calls it before its final flush, and the claims' stop releases the units and resigns the membership lease after it. A unit taken over from a slot that is no longer live is reset (`ResetOrphaned`) before it is bound — waiting, unbound, while the broker answers `ErrUnitHeld` (the dead owner's pin has not lapsed, or the unit was active within its pinned TTL), up to 30s — so the dead owner's unacked rows come back at once; on a process's first tick only if it is the one live member (a restart after a clean stop; after a crash the dead run's lease still counts as live for a lease duration). Each unit's rows delivered and unsettled are capped at its share of the worker's 10,000 (`ClaimConfig.MaxHeld`): an even share over the units assigned (`unitShare`, recomputed each tick and read by the broker before every fetch through `ConsumerConfig.MaxHeld`), never under 1,000, two batches; at its share a unit fetches only what keeps its pin, and one stuck unit never takes another's. A row stops counting at its first ack or nak attempt (`Message.OnSettled` fires whether or not the broker confirms), or after `ack_wait` if the worker never settles it, since the broker redelivers it then as a new row. A configured unit whose delivery ends, or whose durable or stream is gone or whose durable no longer fits (`mq.ErrConsumerNotFound`, `mq.ErrConsumerMismatch`) when it is bound, fails the worker; any other bind failure is retried next tick, logged as an error once it has lasted a minute, and an extra's end is logged. The lowest live slot counts the units with rows and no owner (`Sharded.Unowned`) for `wavehouse_ingest_shards_unowned`. - **sweeper.go** — `Sweeper` implements the Active Sweeper pattern. It runs every minute and asks the MQ (`mq.Purger.PurgeAcked`) to drop the ingest events that are **both** ACKed by the buffer consumer (written to ClickHouse) **and** older than the gap window (re-read every sweep: each tenant's own `stream.gap_window_minutes`, a rejected tenant's as its folder last had it (unbounded for one rejected since boot) — `internal/app`'s `gapWindows` — and none for a removed tenant). Finding the purge point is `internal/mq`'s (`purge.go`). ### `mq/` — Message Queue The **only** package that imports NATS/JetStream — a `depguard` rule in `.golangci.yml` fails `make lint` on any `github.com/nats-io` import in every package golangci-lint builds; the `integration`-tagged files under `tests/` sit outside its default build context, so the boundary there rests on convention (AGENTS.md Key Design Decision #20). Every other package talks to the broker through the types below, so a subject, stream, or broker change lands here once. -- **mq.go** — The owned surface, stated as intent rather than broker mechanics. `Topic{Tenant, Table, Scope}` is the only address the rest of the process handles (a validated tenant id and raw names; comparable, so the SSE hub keys its index by the value). `Message` carries `Data`, its topic (`Topic()` decodes the delivered key on demand — the tenant included, which is how the hub bridge and the worker learn whose event it is; `TopicKey()` is the delivered form, for log lines), and the ack family (`DoubleAck(ctx)`, `Ack()`, `Nak()`, `NakWithDelay(d)`, which falls back to `Nak` for a message built without `WithNakDelay`); `Headers` is the message header map (`Add`/`Set`/`Get`, exact-key) that `PublishOpt`s such as `WithHeader` shape; `WithIdempotencyKey` marks a publish so that a second one carrying the same key inside the queue's duplicate window is dropped and reported as success. Interfaces, each speaking per tenant and never per stream: `Publisher` (`ErrQueueFull` when the tenant's ingest queue is at its byte budget or not open yet — the API's 503 + `Retry-After: 30`; `ErrUnavailable` when the broker cannot be reached or does not answer in time — the 503 + `Retry-After: 5`, which the external NATS backend returns; the embedded broker's publish failures are the `500`), `Subscriber` (every ingest event of every tenant, under a named durable consumer, its fetch-ahead split across the tenants — the hub bridge), `ConsumerManager` → `Consumer` (a durable explicit-ack consumer from a `ConsumerConfig`, whose `MaxAckPending` holds per tenant, and which may name `Units` of a `Sharded` broker — one whose ingest queue is split into units one consumer at a time owns, with `IngestUnits`, `ResetOrphaned` (`ErrUnitHeld` while a consumer holds the unit) and `Unowned`; such a consumer is a `Releaser`, and `Message.OnSettled` (settle.go, apart from mq.go for the e2e coverage gate) runs a hook once, at the first ack or nak attempt, confirmed or not; `Consume` delivers each tenant's messages on a goroutine of that tenant's, in order, so a blocking handler is backpressure on its own tenant alone, spreads the prefetch across the tenants, and returns a `stop` plus a `failed` channel that reports delivery ending on its own — `ErrDeliveryEnded`, e.g. a deleted consumer or a closed connection — since no message would ever say so) for the ingest worker, `DeadLetterer.DeadLetter` (park a message under its own topic, in its tenant's dead-letter queue; the caller acks), `DeadLetterStats.DeadLetterCounts` (one tenant's; `ErrNoDeadLetterQueue` when it has none), `Purger.PurgeAcked` (drop what is both acked by a consumer and stored before its tenant's cutoff, everything acked for a tenant given none; one error per failed tenant, joined — `ErrConsumerNotFound` for a queue the consumer has not been created on yet, the one failure the sweeper logs as a warning rather than an error) for the sweeper, and `Replayer.ReplaySince` for SSE gap-fill. `Broker` composes them with each tenant's byte budget (`SetMaxBytes`/`MaxBytes`), `Stats`, and `Close`; it is what `internal/app` holds. +- **mq.go** — The owned surface, stated as intent rather than broker mechanics. `Topic{Tenant, Table, Scope}` is the only address the rest of the process handles (a validated tenant id and raw names; comparable, so the SSE hub keys its index by the value). `Message` carries `Data`, its topic (`Topic()` decodes the delivered key on demand — the tenant included, which is how the hub bridge and the worker learn whose event it is; `TopicKey()` is the delivered form, for log lines), and the ack family (`DoubleAck(ctx)`, `Ack()`, `Nak()`, `NakWithDelay(d)`, which falls back to `Nak` for a message built without `WithNakDelay`); `Headers` is the message header map (`Add`/`Set`/`Get`, exact-key) that `PublishOpt`s such as `WithHeader` shape; `WithIdempotencyKey` marks a publish so that a second one carrying the same key inside the queue's duplicate window is dropped and reported as success. Interfaces, each speaking per tenant and never per stream: `Publisher` (`ErrQueueFull` when the tenant's ingest queue is at its byte budget or not open yet — the API's 503 + `Retry-After: 30`; `ErrUnavailable` when the broker cannot be reached or does not answer in time — the 503 + `Retry-After: 5`, which the external NATS backend returns; the embedded broker's publish failures are the `500`), `Subscriber` (every ingest event of every tenant, under a named durable consumer, its fetch-ahead split across the tenants — the hub bridge), `ConsumerManager` → `Consumer` (a durable explicit-ack consumer from a `ConsumerConfig`, whose `MaxAckPending` holds per tenant, and which may name `Units` of a `Sharded` broker — one whose ingest queue is split into units one consumer at a time owns, with `IngestUnits`, `ResetOrphaned` (`ErrUnitHeld` while a consumer holds the unit or was active on it within its pinned TTL) and `Unowned`; such a consumer is a `Releaser`, may be a `Halter` (stops fetching ahead of its stop, keeping its hold on its units, so the worker writes what it holds first), is capped per unit by `ConsumerConfig.MaxHeld`, and `Message.OnSettled` (settle.go, apart from mq.go for the e2e coverage gate) runs a hook once, at the first ack or nak attempt, confirmed or not; `Consume` delivers each tenant's messages on a goroutine of that tenant's, in order, so a blocking handler is backpressure on its own tenant alone, spreads the prefetch across the tenants, and returns a `stop` plus a `failed` channel that reports delivery ending on its own — `ErrDeliveryEnded`, e.g. a deleted consumer or a closed connection — since no message would ever say so) for the ingest worker, `DeadLetterer.DeadLetter` (park a message under its own topic, in its tenant's dead-letter queue; the caller acks), `DeadLetterStats.DeadLetterCounts` (one tenant's; `ErrNoDeadLetterQueue` when it has none), `Purger.PurgeAcked` (drop what is both acked by a consumer and stored before its tenant's cutoff, everything acked for a tenant given none; one error per failed tenant, joined — `ErrConsumerNotFound` for a queue the consumer has not been created on yet, the one failure the sweeper logs as a warning rather than an error) for the sweeper, and `Replayer.ReplaySince` for SSE gap-fill. `Broker` composes them with each tenant's byte budget (`SetMaxBytes`/`MaxBytes`), `Stats`, and `Close`; it is what `internal/app` holds. - **subject.go** — The embedded broker's naming, private to the package: the stream names (`INGEST_` and `DLQ_` — prefixes that differ in their first letter, so no tenant id makes one kind's name the other's — and the one pair an earlier build kept for every tenant together, `WAVEHOUSE`/`WAVEHOUSE_DLQ`, which boot deletes), the `ingest.`/`dlq.` prefixes and `>` wildcards, the subject tokens (`internal/keyenc`: ASCII letters, digits, `_` and `-` pass, everything else is percent-encoded, so a name can never split or wildcard a subject), and `Topic` ↔ subject conversion. A subject is `.
[.]`: the tenant verbatim — its grammar (`tenant.Parse`) makes it one token, and it is checked on the way to the wire, so a topic without one has no subject — then the table and scope as encoded tokens; tenant first so one wildcard selects a tenant's traffic (`ingest.acme.>`). A topic has the same tail on both streams, so parking on the DLQ is a prefix swap on the delivered subject — nothing is decoded or re-encoded — and the tail's first token picks the tenant's stream. - **deadletter.go** — `deadLetterTables`, the per-table count `DeadLetterCounts` reports: a dead-letter stream's per-subject counts, each subject parsed back to its topic and counted under its table — every scope of a table under the table itself, so a dotted table name never shares a count with a table + scope pair — and a table filter keeps that table with all of its scopes. - **purge.go** — The Active Sweeper's arithmetic over JetStream sequences: purge target = `MIN(consumer ack floor + 1, first sequence stored at or after the cutoff)`, the latter found by binary search over message timestamps (~15 lookups). Every uncertainty resolves toward purging less: a sequence that holds no message is kept as a candidate bound rather than discarding the half below it, and a lookup that fails outright aborts that tenant's purge. It runs on each tenant's stream at that tenant's cutoff, and a failure on one tenant's stream is reported without stopping the sweep of the others. Healthy state keeps exactly the gap window; ClickHouse down freezes purging; a catastrophic outage fills the stream to `MaxBytes` and `DiscardNew` pushes back. -- **external.go** — `ExternalNATS`, the `Broker` over an operator-owned NATS cluster (`mq.backend: nats`): N work-queue ingest partitions shared by every tenant, each split into V shards by a subject token (`natsRoute`: jump hashes of the tenant and table, one for the partition and an independent one for the shard), with one pinned-client durable per shard, each partition republishing every row it stores to the history stream under `.hist.` for SSE replay and the hub, and one dead-letter stream. Ingest publishes do not name the stream they expect, since the republished copy would carry that expectation to the history, which would refuse it; the ack's stream is checked instead. It never creates, changes, purges or deletes a stream or a durable; it creates only auto-expiring consumers on the history stream, one per `Subscribe` and one per replay. `NewNATS` connects and waits for the topology to pass the verifier; publishes carry a `Nats-Msg-Id` reused across retries; a broker that does not answer is `ErrUnavailable`; `PurgeAcked` removes nothing; as a `Sharded` broker it names its units (`/`, the configured N×V then any extras a lower N or V left holding rows, which are drained like the rest); a consumer pulls each unit in the priority group `wavehouse` with a 5s pull expiry, so the server delivers a unit to one pinned puller at a time, and its stop drains what was fetched; `Release` unpins a unit only while the server still holds this consumer's own pin (from the `Nats-Pin-Id` of the last row it received); `ResetOrphaned` resets a unit's durable to its ack floor when no client is pinned and rows await an ack, which a caller does before it pulls the unit, never after, since a reset also redelivers the caller's own unacked rows. A consumer of every unit resets orphaned units before pulling and releases each after its drain. It exports the `wavehouse_mq_connected`, `wavehouse_mq_topology_ok` and `wavehouse_mq_history_behind_seconds` gauges. +- **external.go** — `ExternalNATS`, the `Broker` over an operator-owned NATS cluster (`mq.backend: nats`): N work-queue ingest partitions shared by every tenant, each split into V shards by a subject token (`natsRoute`: jump hashes of the tenant and table, one for the partition and an independent one for the shard), with one pinned-client durable per shard, each partition republishing every row it stores to the history stream under `.hist.` for SSE replay and the hub, and one dead-letter stream. Ingest publishes do not name the stream they expect, since the republished copy would carry that expectation to the history, which would refuse it; the ack's stream is checked instead. It never creates, changes, purges or deletes a stream or a durable; it creates only auto-expiring consumers on the history stream, one per `Subscribe` and one per replay. `NewNATS` connects and waits for the topology to pass the verifier; publishes carry a `Nats-Msg-Id` reused across retries; a broker that does not answer is `ErrUnavailable`; `PurgeAcked` removes nothing; as a `Sharded` broker it names its units (`/`, the configured N×V then any extras a lower N or V left holding rows, which are drained like the rest); a consumer (external_consumer.go) fetches each unit in the priority group `wavehouse`, 1s pulls sized to what the unit's cap leaves room for, on a goroutine that never runs the handler (another hands the rows on, in order), so the server delivers a unit to one pinned puller at a time and a busy handler never lets the pin lapse; at the cap it renews the pin every 5s with a one-row fetch, up to one `ack_wait` of renewals past the cap (a pull that held a due redelivery back would have the server requeue it behind the others), and otherwise, and once halted, with a pull of `max_bytes` 1, which renews the pin and delivers nothing (the server holds the row back). `Halt` stops fetching and returns once the fetch in flight has ended and its rows reached the handler; a halted unit keeps renewing until `Release`, or until `ack_wait` and one renewal have passed since it handed on what it fetched. `Release` unpins a unit only while the server still holds this consumer's own pin (from the `Nats-Pin-Id` of the last row it received); the check and the unpin are two requests, and the pin cannot lapse between them because the unit renews it until the unpin returns, for up to `ack_wait` (and one renewal) after its halt has handed on what it fetched, which outlasts the handover's own wait. A pull that gets no responders twice in a row, or a renewal that gets one, looks the durable up, and a durable or stream that is gone ends the unit's delivery (`failed`). `ResetOrphaned` resets a unit's durable to its ack floor when no client is pinned, the durable delivered nothing and had no ack within its pinned TTL, and rows await an ack; a caller does it before it pulls the unit, never after, since a reset also redelivers the caller's own unacked rows. `Unowned` applies the same activity test. A consumer of every unit resets orphaned units before pulling and releases each after its halt. It exports the `wavehouse_mq_connected`, `wavehouse_mq_topology_ok` and `wavehouse_mq_history_behind_seconds` gauges. - **lease.go** — `ExternalNATS.Leases(ctx, bucket, holder)`, the `coord.Coordinator` (and `coord.Observer`: `Held` counts another holder's lease as held until this coordinator has seen its revision unchanged for the lease duration, the clock `TryAcquire` keeps) for `coord.backend: nats`, over the operator's KV bucket (a missing one is `ErrTopology`; WaveHouse never creates it). A lease is the key `lease.`, its value JSON `{holder, duration_ms, session}`; the KV revision a term was taken at is its fencing `Token`. `TryAcquire` creates an absent (or resigned: a delete marker) key; takes over another holder's only after seeing the same revision unchanged for the lease duration on its own monotonic clock (no clocks are compared, and the bucket keeps no per-key TTL, which a renewal could not extend), by a compare-and-set at that revision; and resumes its own write (same `session`) at once. The term renews every `coord.RetryPeriod` (2s) at the revision it last wrote; a write refused for its revision ends it with `ErrLost` unless the key holds its own value at a later revision (a renewal whose answer was lost), and a timer ends it the moment the renew deadline (10s, before the 15s lease duration) passes without a stored renewal, timed from when that renewal was sent, since a candidate's clock can start as soon as it is stored. `Resign` and `Close` cancel a renewal in flight and delete the key at the last revision (or at a later one holding this term's own value, a renewal the cancel cut short), so a successor need not wait. `TryAcquire` holds no lock across a request. `WithLeaseTimings` shortens the timings for tests. - **nats_topology.go**, **nats_manifests.go**, **subject_nats.go** — what the operator must create (`NATSTopology`: with a dedupe lease, every partition's duplicate window must cover the lease, the lease rounded up to a second, and one more second; with the lease bucket, `CoordBucket`, checked only when a process holds leases there: it must exist, keep a value per key, allow direct gets, and expire nothing), the verifier that checks a live server against it and reports every finding (required or recommended), the nack resources `wavehouse mq manifests` prints from the same spec (`deployments/nats/jetstream.yaml` is its output for N=4, V=8) and the `wavehouse` user's permissions `wavehouse mq permissions` prints (one entry per shard durable: a NATS wildcard is a whole token), and the external broker's subjects (`.ingest.

...

`, `.hist..
`, `.dlq..
`). - **natstest/** — Test code that stands up NATS as an operator deploys it, from the shipped `deployments/nats` values and manifests: the config for a server (in process, or in the integration suite's container) and the operator's hand on it (applying the manifests, the lease bucket included; deleting a durable or the bucket; reading which process holds a lease). It lets `internal/app` and `tests/integration` run against a real server without importing NATS themselves. diff --git a/docs/src/content/docs/deployment.md b/docs/src/content/docs/deployment.md index 89753feaa..8925bf076 100644 --- a/docs/src/content/docs/deployment.md +++ b/docs/src/content/docs/deployment.md @@ -340,7 +340,7 @@ With `mq.backend: nats`, WaveHouse's message queue is a NATS JetStream cluster y ### What WaveHouse needs - **N ingest partition streams.** Partition `p` holds `.ingest.

.>`, and each row's subject is `.ingest.

...

`, where `s` is one of its V shards. It has work-queue retention: a row stays until the ingest worker has written it and acked it, even while no consumer covers it, and is deleted then, so one tenant whose ClickHouse is down keeps only its own rows on disk. Each partition also republishes every row it stores to `.hist..
`, the same subject without the partition and shard, where the history stream captures it. A work queue takes no consumer beside the ingest durables whose subjects overlap theirs, so a debugging consumer on a partition is refused; read the history stream, or get a row directly, instead. A table's events always go to the same partition and shard, chosen by a consistent (jump) hash of the tenant and table: the tenant's other tables spread over the partitions, and raising N to N′ moves (N′−N)/N′ of the tables, each into one of the new partitions (about 1/N′ for one more), and lowering it moves only the removed partitions' tables; V the same. Each partition has no age limit (an age limit would drop rows not yet written), and `discard: new` with a byte limit: a full partition refuses new events with `503` and `Retry-After: 30`, for every tenant in it. -- **A durable per shard on every partition,** `wh-ingest-` for shard `s` (unpadded, so a name never changes when V grows), filtering `.ingest.

..>`, with `priority_policy: pinned_client` in the priority group `wavehouse` and a pinned TTL (`priority_timeout`) of at least 10 seconds, twice WaveHouse's 5-second pull expiry. The ingest processes share the shards out (see [Scaling ingest processes](#scaling-ingest-processes)), and the server delivers a shard to one puller at a time, the one it pinned; the pin lapses once its holder stops pulling for the TTL, and a holder that stops cleanly releases it at once. +- **A durable per shard on every partition,** `wh-ingest-` for shard `s` (unpadded, so a name never changes when V grows), filtering `.ingest.

..>`, with `priority_policy: pinned_client` in the priority group `wavehouse` and a pinned TTL (`priority_timeout`) of at least 10 seconds, twice the 5 seconds between the pulls that keep the pin of a shard at its share of held rows, or stopping. The ingest processes share the shards out (see [Scaling ingest processes](#scaling-ingest-processes)), and the server delivers a shard to one puller at a time, the one it pinned; the pin lapses once its holder stops pulling for the TTL, and a holder that stops cleanly releases it at once. - **The history stream,** which holds `.hist.>` and has no sources: the partitions' republish is what writes it. SSE replay (`Last-Event-ID`) and every API process's live events read from it. It is best effort: a row republished while the history is unavailable (deleted, or electing a leader) reaches ClickHouse but never SSE, and a full or missing history never holds up ingest. Its `max_age` is how far back a replay can reach, so set it to at least the longest [`stream.gap_window_minutes`](/settings-directory#streaming) among the tenants you serve. The generated manifests use `15m`, the seed's default window; the API processes check at boot and after each reload, and warn once per tenant and window (again only if that tenant's window changes) for each tenant whose window is longer. - **One dead-letter stream** holding `.dlq.>`, shared by every tenant. - **The lease bucket,** a KV bucket named `_coord` (`wh_coord`; [`coord.nats.bucket`](/configuration#nats-leases-coordnats) names another), where [`coord.backend: nats`](/configuration#backends) holds the ingest processes' leases. Every process running the `ingest` role needs it, because `mq.backend: nats` refuses `coord.backend: local` there. Keep one value per key (`history: 1`), allow direct gets (`allow_direct`, which nack and `nats kv add` always set, because the `wavehouse` user reads leases only that way), and set no `ttl`: a lease expires on its candidates' clocks, and a key the server expires would end a live holder's lease. Boot checks it only in a process with `coord.backend: nats`, and refuses while it is missing. @@ -370,9 +370,9 @@ The generated manifests satisfy every required finding (pass `--dedupe-lease` wh - A partition's `duplicate_window` must cover every attempt of one publish: three times `mq.nats.publish_timeout`, plus half a second. A publish that got no answer is retried with the same message id, so the partition stores it once. - It must also cover [`dedupe.lease`](/configuration#dedupe) twice over, plus a second: the lease, the lease rounded up to whole seconds, and one more second (61s for the default 30s). With dedupe on, a publish whose outcome was unknown keeps its id claimed until the lease lapses, and a client obeying `Retry-After` republishes it as late as that under the same idempotency key; the partition drops the copy only while it still remembers the first. The shipped `2m` covers any lease up to `59s`; for a longer one, `wavehouse mq manifests --dedupe-lease ` widens it. - A tenant with dedupe on whose finite [`dedupe.retention`](/settings-directory#deduplication), for the tenant or one of its tables, is shorter than the partitions' `duplicate_window` is logged at `WARN`, at boot and after every reload. An id re-sent after its retention but inside the window would be claimed again and then dropped by the partition, while the client is told it was accepted. The settings directory refuses a retention under `2m`, the embedded queue's window, but cannot see yours: keep retention at least as long as the window, or `"0"`. -- Each shard durable needs `max_expires` unset or at least 5 seconds, or the server refuses WaveHouse's pulls. +- Each shard durable needs `max_expires` unset or at least 5 seconds. WaveHouse's pulls wait at most a second today; the floor leaves room for longer ones, since the server refuses a pull that asks for more than `max_expires`. - Each shard durable needs `max_deliver: -1`. With a limit, a row that failed that many times would stay on its partition and never be delivered again. -- Each shard durable must filter exactly its shard's subjects and use `pinned_client` in the group `wavehouse`, with a pinned TTL of at least 10 seconds. The server renews a pin only when its holder sends a new pull, so a shorter TTL lets an idle live holder lose its pin between pulls. A TTL of 15 seconds or more is recommended against: a dead holder's shard is received by another process only once its pin lapses. +- Each shard durable must filter exactly its shard's subjects and use `pinned_client` in the group `wavehouse`, with a pinned TTL of at least 10 seconds. The server renews a pin only when its holder sends a new pull, so a shorter TTL lets a live holder that is at its share of held rows, or stopping, lose its pin between its 5-second renewals. A TTL of 15 seconds or more is recommended against: a dead holder's shard is received by another process only once its pin lapses. WaveHouse checks the topology again every five minutes and never repairs it. If you delete a partition, its publishes answer `503` with `Retry-After: 5`. If you delete one of the N×V configured shard durables (`wh-ingest-0` to `wh-ingest-` on each of the N partitions), or the connection is closed for good (for example, its credentials are revoked), the ingest worker ends and the process exits, so that the orchestrator restarts it and the next boot names what is missing. An ingest worker that stayed up without its queue would leave the API accepting events that nothing writes. @@ -411,7 +411,7 @@ WaveHouse's replies arrive under `_INBOX_.>`, which is why the subscribe ### Limits that differ from the embedded queue - **Per-tenant budgets are not enforced.** A tenant's [`mq.max_bytes_gb`](/settings-directory#message-queue) is not applied; a partition's byte limit is shared by the tenants in it. `maxMsgsPerSubject` with `discardPerSubject: true`, which the generated manifests set, refuses one tenant's table once it holds that many unwritten rows, before it fills the partition. -- **A shard's delivery can stall on one tenant.** If one tenant's ClickHouse is down, its unwritten rows can take up its shard durable's `max_ack_pending`, and then delivery pauses for every table in that shard, about 1/(N×V) of the tables. More shards shrink that share. +- **A shard's delivery can stall on one tenant.** If one tenant's ClickHouse hangs, its unwritten rows can fill its shard's share of the rows the ingest process may hold (see [Scaling ingest processes](#scaling-ingest-processes)), and then delivery pauses for every table in that shard, about 1/(N×V) of the tables. The other shards of the same process keep their own shares and keep flowing. More shards shrink that share. (A ClickHouse that refuses inserts outright does not stall a shard: its rows are handed back to the partition for a delayed retry.) - **Dead-lettered rows share one stream.** Its `discard: old` evicts the oldest rows when it is full; with `maxMsgsPerSubject` set, it evicts per table, so one tenant's flood evicts only its own rows. `GET /v1/ops/dlq/stats?tenant=` answers `200` with zeros for a tenant that has never parked a row, where the embedded queue answers `404`. - **SSE replay and live events are best effort.** A row republished while the history stream is unavailable, for example full with `discard: new`, deleted, or electing a leader, is written to ClickHouse but never reaches SSE. `wavehouse_mq_history_behind_seconds` shows a history that stops taking rows, and a deleted one fails the topology check (`wavehouse_mq_topology_ok`). @@ -428,14 +428,14 @@ Every generated partition records its index and N in its metadata (`wavehouse.de Scale the ingest processes as you like; no manifest changes. Each one holds a membership lease in the lease bucket (`lease.ingest.m`, for the lowest free `j`; there are at most 64 such slots, so at most 64 ingest processes get work), and every process computes the same owner for every shard from the live members: rendezvous hashing, capped so that no process owns more than an even share, rounded up. A process that joins or leaves moves roughly one to two times its share of the shards while the even share, rounded up, stays the same (a few times it at worst); past about seven processes, where that share steps down (over 32 shards in all, partitions × shards: at 8, 11, 16 and 32 processes), it moves several times its share. It never moves more than half of them. The cap bounds the most a process owns, not the least: above about half as many processes as shards, some hold none, and a process beyond partitions × shards, or beyond 64, always holds none. Shard durables left over after N or V is lowered are shared out apart from the configured ones, so deleting a drained one moves no configured shard. -- **A shard changing owner** stops fetching at the old owner, which waits for the rows it delivered to be acked, or handed back to the partition (at most 15 seconds), then releases the shard's pin; the new owner receives at once. If the wait runs out, the new owner may receive before the old one has finished, so that one shard's rows are briefly written by two processes, out of order. -- **A clean stop** releases every shard at once, after the worker has written what it held, and resigns the membership lease; the others take over within about a tick (2 seconds). -- **A process that dies** keeps its membership lease until the others have seen it unrenewed for 15 seconds, and its shards' pins until the pinned TTL (10 seconds in the generated manifests) after its last pull. Then each new owner resets the shard to its ack floor before pulling it, so the rows the dead process received and never acked come back at once rather than after `ack_wait` (at least 1 minute). A lone ingest process restarting after a clean stop does the same on its first pass, for every shard nobody holds. One restarting after a crash still sees its predecessor's lease as live for 15 seconds: the shards the new process is assigned beside it come back after `ack_wait`, the rest at once when that lease lapses. Rows it had written but not acked are written again: delivery is at least once, so use a deduplicating table engine, or [`dedupe`](/configuration#dedupe). -- **A live process that stops making progress** (a hung ClickHouse insert) keeps its shards: liveness is not progress. The rows it holds across its shards are capped at 10,000; at the cap its deliveries wait, and the rest stay on the partitions. A row stops counting at its first ack or nak, confirmed or not, or once `ack_wait` has passed, when the partition redelivers it anyway. +- **A shard changing owner** stops fetching at the old owner, which keeps the shard's pin while it waits for the rows it delivered to be acked, or handed back to the partition, then releases the pin; the new owner receives at once, and only rows newer than the ones the old owner wrote. Fetching stops within a second, the length of one pull; the rows already fetched must then still reach the worker, which takes as long as the worker is behind. The wait for them to be acked is bounded by one minute from when they have reached the worker, the worker's ack wait and the shortest `ack_wait` a shard durable may have, and the old owner keeps the pin that long and a few seconds more; if it runs out, the new owner may receive before the old one has finished, so that one shard's rows are briefly written by two processes, out of order. +- **A clean stop** stops fetching every shard first, keeping their pins, so the worker writes what it holds and whatever those shards still deliver; then it releases every shard at once and resigns the membership lease, and the others take over within about a tick (2 seconds). A row the worker could not take while stopping is handed back to the partition before the release, so it comes back to the next owner ahead of anything newer. Rows the partition redelivers (handed back, or past `ack_wait`) come back ahead of newer rows, but not always in order among themselves across a handover or a stop: while the next owner's pull waits for the pin, the server requeues a due redelivery behind the others. +- **A process that dies** keeps its membership lease until the others have seen it unrenewed for 15 seconds, and its shards' pins until the pinned TTL (10 seconds in the generated manifests) after its last pull. Then each new owner resets the shard to its ack floor before pulling it, so the rows the dead process received and never acked come back at once rather than after `ack_wait` (at least 1 minute). The reset needs more than the membership view: the shard must also hold no pin and have delivered nothing and had no ack for the pinned TTL, so a process the others wrongly think dead, but which still works the shard, is never reset under it. A lone ingest process restarting after a clean stop does the same on its first pass, for every shard nobody holds. One restarting after a crash still sees its predecessor's lease as live for 15 seconds: the shards the new process is assigned beside it come back after `ack_wait`, the rest at once when that lease lapses. Rows it had written but not acked are written again: delivery is at least once, so use a deduplicating table engine, or [`dedupe`](/configuration#dedupe). +- **A live process that stops making progress** (a hung ClickHouse insert) keeps its shards, and their pins, however long its inserts take: liveness is not progress. The worker never holds up a shard's pulls, so its pins never lapse, and no other process takes its shards or resets them. Each shard may hold rows delivered and not yet acked up to its share of 10,000: an even share over the shards the process holds, but never under 1,000 (two batches), so one busy table still fills whole batches. At its share a shard fetches one row every 5 seconds, which keeps its pin (a row the partition redelivers comes back in order that way), up to one `ack_wait` of such rows past its share (12 at the generated 1-minute `ack_wait`); past that it keeps its pin with pulls that deliver nothing, and a stuck shard's redelivered rows may then come back out of order among themselves; the rest stay on the partition, and one stuck shard never takes another shard's share. A process so holds about 10,000 rows at most, or 1,000 per shard when it holds more than ten, plus those 12 per stuck shard. A row stops counting at its first ack or nak, confirmed or not, or once `ack_wait` has passed, when the partition redelivers it anyway. ### Choosing and changing V -A shard's rows reach ClickHouse through one ingest process at a time, so V×N caps how many ingest processes get work, and one table's insert rate is bounded by what one process writes. A process splits its prefetch (500 rows) between the shards it pulls, so one shard's pull asks for fewer rows the more shards the process holds. The default V of 32 leaves room to scale out; a larger V costs a consumer per shard on every partition (a Raft group each at 3 replicas), and a permission entry per shard. V must match `mq.nats.shards` in every process, and each partition records it in its metadata (`wavehouse.dev/shards`). +A shard's rows reach ClickHouse through one ingest process at a time, so V×N caps how many ingest processes get work, and one table's insert rate is bounded by what one process writes. A process splits its prefetch (500 rows) between the shards it pulls, so one shard's pull asks for fewer rows the more shards the process holds. The default V of 32 leaves room to scale out; a larger V costs a consumer per shard on every partition (a Raft group each at 3 replicas), a permission entry per shard, and one pull a second per shard that has nothing to deliver. V must match `mq.nats.shards` in every process, and each partition records it in its metadata (`wavehouse.dev/shards`). - **To raise V,** regenerate with `--shards ` and apply the whole output: the new shards' durables are added, and the existing ones keep their names and their rows. Regenerate the permissions with the new V too, then roll WaveHouse out with it promptly: the regenerated metadata says the new V, so processes still on the old V report `wavehouse_mq_topology_ok` `0` at their next check and cannot restart. Raising V to V′ moves (V′−V)/V′ of the tables, each into one of the new shards; a moved table's older rows drain from its old shard while its new rows go to the new one, so for that table alone two processes may insert at once for a while, and its events are not in order across the move. A process still on the old V publishes to the old shards. Until every ingest process runs the new V, the old and new processes share the shards out differently, so some shards (the new ones among them) may have no owner: their rows wait, and `wavehouse_ingest_shards_unowned` reads above `0` until the rollout ends. - **To lower V,** apply the regenerated manifests (the generated durables set `preventDelete`, so the removed shards' durables stay with their rows) and roll WaveHouse out with the smaller V, promptly, for the same reason. Keep the permissions for the larger V until they drain. The ingest processes share out each durable past the new V and drain it, and boot warns about each with the rows it holds; once one holds none and no process runs the old V, delete its nack `Consumer` resource if it is still applied (nack's control loop would recreate the durable otherwise), then the durable itself with the operator's credentials (`nats consumer rm `): `preventDelete` keeps the durable when only its resource goes. That ends delivery from that durable only, not the worker. Then regenerate the permissions for the smaller V. @@ -463,10 +463,9 @@ The ingest processes export these for their shards: | --- | --- | | `wavehouse_ingest_shards_owned` | Shards this process holds and consumes. The sum over processes is partitions × shards, plus any extras still draining, once membership has settled. | | `wavehouse_ingest_shard_members` | Ingest processes this process last counted. | -| `wavehouse_ingest_shards_unowned` | Shards with rows waiting that no process holds, read every 15 seconds by the lowest-ranked member only (the others report nothing, so take the `max` across processes). Alert when it stays above `0` for a minute or more: rows nobody is writing. | -| `wavehouse_ingest_rows_held` | Rows delivered to this process that it has not yet acked or nacked (confirmed or not), and that are younger than `ack_wait`, across its shards; at most 10,000. | -| `wavehouse_ingest_rows_held_waits_total` | Deliveries that waited because the process was at that cap. | -| `wavehouse_ingest_shard_events_total{event}` | `taken`, `released`, `reset` (a dead owner's rows redelivered at takeover), `handover_timeout` (a shard released before its rows were acked or handed back), `lost` (the membership lease ended under the process), `bind_failed` (a shard could not be bound; tried again next tick). | +| `wavehouse_ingest_shards_unowned` | Shards with rows waiting that no process holds, and that have delivered nothing and had no ack for the pinned TTL, read every 15 seconds by the lowest-ranked member only (the others report nothing, so take the `max` across processes). Alert when it stays above `0` for a minute or more: rows nobody is writing. A shard whose process is stuck on a hung insert keeps its pin, so it does not count. | +| `wavehouse_ingest_rows_held` | Rows delivered to this process that it has not yet acked or nacked (confirmed or not), and that are younger than `ack_wait`, across its shards; each shard holds about its share at most (see [Scaling ingest processes](#scaling-ingest-processes)). | +| `wavehouse_ingest_shard_events_total{event}` | `taken`, `released`, `reset` (a dead owner's rows redelivered at takeover), `handover_timeout` (a shard released before its rows were acked or handed back), `lost` (the membership lease ended under the process), `bind_failed` (a shard could not be bound; tried again next tick, and logged as an error once it has failed for a minute). A shard durable that is gone, with its stream or alone, or that no longer fits the worker (its `ack_wait` shorter than a minute, or no `max_ack_pending`), ends the worker instead, as a durable deleted while it is pulled does. | | `wavehouse_ingest_shard_handover_seconds` | From giving a shard up to releasing it. | `wavehouse_nats_connections` and `wavehouse_nats_in_msgs_total` describe this process's client connection under `nats` (`1` or `0`, and the messages it has received), where under `embedded` they describe the embedded server. diff --git a/docs/src/content/docs/ingest-pipeline.md b/docs/src/content/docs/ingest-pipeline.md index b4688439a..65f717285 100644 --- a/docs/src/content/docs/ingest-pipeline.md +++ b/docs/src/content/docs/ingest-pipeline.md @@ -17,7 +17,7 @@ It is deliberately detailed: this is a hot, concurrency-heavy path, and the goro | `backoff.go` | The retry backoff per ClickHouse pool: one outage backs off every table on it together, probing once per window; a failure of one table (read-only, too many parts or mutations, a missing grant) backs off that table alone — see [When ClickHouse cannot take an insert](#when-clickhouse-cannot-take-an-insert) | | `compact.go` | `EncodeCompactRow` — renders one record as a `JSONCompactEachRow` line over the table's **insertable** columns, in declaration order. Serialization only: it validates nothing and judges no value | | `sweeper.go` | The **Active Sweeper** — every minute, asks the MQ to purge the events that are both written to ClickHouse and past the SSE gap window (the purge arithmetic below lives in `internal/mq/purge.go`) | -| `claims.go` | `ClaimShards` — over the external broker's shards, narrows the worker to this process's share: the membership lease, the per-tick assignment, handover by drain then release, the reset of a dead owner's shards at takeover, the `MaxHeld` cap on rows held, and the shard gauges. The behavior is in [Scaling ingest processes](/deployment#scaling-ingest-processes). | +| `claims.go` | `ClaimShards` — over the external broker's shards, narrows the worker to this process's share: the membership lease, the per-tick assignment, handover by drain then release, the reset of a dead owner's shards at takeover, each shard's share of the `MaxHeld` budget of rows held, the halt that lets the worker write what it holds before a clean stop releases its shards, and the shard gauges. The behavior is in [Scaling ingest processes](/deployment#scaling-ingest-processes). | | `assign.go` | `assignUnits` — rendezvous hashing capped at ⌈units/slots⌉, the same owners in every process for the same view | | `types.go` | `EventMessage` wire format and the `BufferConsumerName` constant | @@ -238,7 +238,7 @@ Several layers throttle the pipeline, inner to outer: 2. **`msgChan`** (cap `maxBatch*2`) — when full, the consume callback blocks and delivery pauses. 3. **`pullMaxMessages`** — nats.go's client-side prefetch buffer in front of `msgChan`, shared by the tenants' streams (at least one message each). 4. **`maxAckPending`** — the server suspends a tenant's delivery once this many of its messages are delivered-but-unacked; no other tenant's delivery waits on it. The outermost in-memory bound, and a per-tenant one: while ClickHouse stalls, the worker can hold up to `maxAckPending` rows for every tenant served. -5. **`MaxBytes` + `DiscardNew`** on each tenant's stream (its `mq.max_bytes_gb` in the [settings directory](/settings-directory#message-queue), resized in place on reload) — when it fills (e.g. ClickHouse is down so nothing acks/purges), that tenant's new publishes are rejected and the API returns 503. Steps 4 and 5 are the embedded broker's; under `mq.backend: nats` a shard durable's `max_ack_pending` pauses every table in that shard, a process holds at most 10,000 rows across its shards, and a full partition refuses every tenant in it (see [Limits that differ from the embedded queue](/deployment#limits-that-differ-from-the-embedded-queue)). +5. **`MaxBytes` + `DiscardNew`** on each tenant's stream (its `mq.max_bytes_gb` in the [settings directory](/settings-directory#message-queue), resized in place on reload) — when it fills (e.g. ClickHouse is down so nothing acks/purges), that tenant's new publishes are rejected and the API returns 503. Steps 4 and 5 are the embedded broker's; under `mq.backend: nats` a shard slows to what keeps its pin at its share of the 10,000 rows a process may hold unsettled (an even share over its shards, at least 1,000), which pauses every table in that shard, and a full partition refuses every tenant in it (see [Limits that differ from the embedded queue](/deployment#limits-that-differ-from-the-embedded-queue)). | Knob | Default | Meaning / invariant | | --- | --- | --- | @@ -288,9 +288,9 @@ flowchart TD W --> D ``` -- **Work distribution.** Each partition is split into V shards, and a table's rows all go to one shard. Each shard has its own durable with the `pinned_client` priority policy, so the server delivers a shard to one pulling process at a time: a table's rows are batched and written by one process at a time. The ingest processes share the shards out: each holds a membership lease (`ingest.m`), and every process computes the same owner for every shard from the live members, by rendezvous hashing capped at an even share (`ClaimShards`). A shard given up is drained, then its pin released, so the next owner receives at once; a shard whose owner died is reset to its ack floor before its new owner pulls it, so the rows the dead owner held come back at once rather than after `ack_wait`. Rows delivered and unsettled across a process's shards are capped at 10,000. +- **Work distribution.** Each partition is split into V shards, and a table's rows all go to one shard. Each shard has its own durable with the `pinned_client` priority policy, so the server delivers a shard to one pulling process at a time: a table's rows are batched and written by one process at a time. The ingest processes share the shards out: each holds a membership lease (`ingest.m`), and every process computes the same owner for every shard from the live members, by rendezvous hashing capped at an even share (`ClaimShards`). A shard given up stops fetching but keeps its pin until the rows it delivered are written, then releases it, so the next owner receives at once and only newer rows; a stopping process first halts every shard the same way (`mq.Halter`), writes what it holds, and only then releases them. A shard whose owner died is reset to its ack floor before its new owner pulls it, so the rows the dead owner held come back at once rather than after `ack_wait`; the broker refuses the reset while the shard is pinned or was active within its pinned TTL. The worker never blocks a shard's pulls: at the shard's share of the 10,000 rows a process may hold unsettled (an even share over its shards, at least 1,000), the shard fetches only what keeps its pin (a row every 5 seconds, at most one `ack_wait` of them past its share, then pulls that take none), so a stuck table neither loses its shard nor stalls the process's other shards. - **Idempotent inserts matter more.** At-least-once delivery plus redelivery after a crash means another process can re-insert a batch the dead one had written but not acked. Use `ReplacingMergeTree` (or a dedup key). A single process already re-inserts after a crash, or after an insert whose outcome it could not see ([When ClickHouse cannot take an insert](#when-clickhouse-cannot-take-an-insert)); several make it routine. -- **NATS resilience.** The external broker reconnects on its own, with backoff; while it is disconnected a publish answers `503` with `Retry-After: 5`, and consumption resumes after the reconnect. A consumer whose delivery ends for good (one of its configured shard durables deleted, or the connection closed) ends the worker and the process, as the embedded one does. +- **NATS resilience.** The external broker reconnects on its own, with backoff; while it is disconnected a publish answers `503` with `Retry-After: 5`, and consumption resumes after the reconnect. A consumer whose delivery ends for good (one of its configured shard durables deleted, or its stream, even while no pull of it is waiting, which the next pull finds; a durable that no longer fits the worker's config; or the connection closed) ends the worker and the process, as the embedded one does. - **The sweeper.** Work-queue retention deletes each row once it is acked, one row at a time, so one tenant's unwritten rows never hold back another's reclaim, which a shared ack floor would, and it keeps a row no consumer covers rather than dropping it. SSE replay reads the history stream, which the partitions republish to and which expires by `max_age`; it is best effort, so an outage of the history costs SSE the rows republished meanwhile and never holds up ingest. So there is nothing for the sweeper to purge. ## Deferred / not yet implemented diff --git a/internal/ingest/claims.go b/internal/ingest/claims.go index 5e5ed4f91..1cadb378d 100644 --- a/internal/ingest/claims.go +++ b/internal/ingest/claims.go @@ -38,16 +38,20 @@ import ( // of them. Views can differ for up to a lease: each process starts a // silent member's expiry clock when it first sees it. // -// Handover. A unit no longer assigned here stops fetching, waits until the -// rows it delivered are settled (bounded by Handover), then releases the -// broker's pin, so the next owner receives at once. A clean stop releases -// every unit after the worker has flushed. +// Handover. A unit no longer assigned here stops fetching while keeping the +// broker's pin, waits until the rows it delivered are settled (bounded by +// Handover), then releases the pin, so the next owner receives at once and +// only rows newer than the ones written here. A clean stop halts every unit +// first (Halt), so the worker can flush what it holds, and releases them +// after. // // Takeover. A unit whose owner in the previous tick's view is no longer a // member is reset to its ack floor before it is consumed (ResetOrphaned), so // the dead owner's unacked rows come back at once instead of after ack_wait. -// While the dead owner's pin has not lapsed yet the unit waits, unbound, and -// is tried again next tick, up to orphanWait. On a process's first tick, +// The broker refuses the reset while the unit is pinned or was recently +// active, so the membership view alone never resets a live owner's rows; +// the unit then waits, unbound, and is tried again next tick, up to +// orphanWait. On a process's first tick, // which has no previous view, only when it is the one live member: then // whatever holds rows unpinned is gone. That covers the only process // restarting after a clean stop, which resigned its lease. After a crash @@ -57,18 +61,26 @@ import ( // the takeover above once its lease has lapsed. Never after pulling: a // reset also redelivers the caller's own unacked rows. // -// Memory. Rows delivered and not yet settled, across every unit this process -// holds, are capped at MaxHeld: at the cap a unit's delivery waits, and the -// broker keeps the rest. +// Memory. Each unit may hold delivered and unsettled rows up to its share of +// MaxHeld: an even share over the units this process is assigned, and never +// less than minUnitHeld (or MaxHeld, if smaller), so one hot table still +// fills whole batches. At its share a unit fetches only what keeps its pin +// (the broker's: a row every few seconds, at most one ack_wait's worth past +// the share), and the broker keeps the rest; a stuck unit +// never takes another unit's share. The process holds about max(MaxHeld, +// units × minUnitHeld) at most. const ( memberLeasePrefix = "ingest.m" - defaultHandover = 15 * time.Second defaultUnownedEvery = 15 * time.Second - // settleQuiet is how long a unit given up must deliver nothing before its - // handover counts it settled: the rows it fetched ahead reach the worker - // in that time. - settleQuiet = 250 * time.Millisecond + // minUnitHeld is two of the worker's batches: one being written while + // the next fills. + minUnitHeld = 2 * defaultMaxBatch + // settlePoll is how often a handover checks its rows have settled. + settlePoll = 50 * time.Millisecond + // bindEscalate is how long a configured unit may fail to bind, for a + // reason that may pass, before each failure logs an error. + bindEscalate = time.Minute // tickTimeout bounds one tick's requests. tickTimeout = 10 * time.Second // releaseWait bounds a lease resign or a unit's release. @@ -92,10 +104,12 @@ type ClaimConfig struct { // default coord.RetryPeriod. Every time.Duration // Handover bounds how long a unit given up waits for its delivered rows - // to settle before it is released; default 15s. + // to settle before it is released; default the worker's ack_wait, after + // which the broker redelivers them anyway. Handover time.Duration - // MaxHeld caps the rows delivered and not yet settled across every unit - // this process holds; default the worker's maxAckPending. + // MaxHeld is the process's budget of rows delivered and not yet settled, + // shared out per unit (see Memory above); default the worker's + // maxAckPending. MaxHeld int // UnownedEvery is how often the lowest live member reads how many units // have rows and no owner; default 15s. @@ -128,7 +142,7 @@ func ClaimShards(q Queue, c coord.Coordinator, cfg ClaimConfig) (Queue, error) { cfg.Every = coord.RetryPeriod } if cfg.Handover <= 0 { - cfg.Handover = defaultHandover + cfg.Handover = ackWait } if cfg.MaxHeld <= 0 { cfg.MaxHeld = maxAckPending @@ -179,17 +193,19 @@ type claimingConsumer struct { // Consume starts claiming. stop ends it, releasing every unit held and // resigning the membership lease; failed reports, exactly once, a configured -// unit's delivery ending on its own or a unit that could not be bound. +// unit's delivery ending on its own or a unit that could not be bound +// because its durable or stream is gone or no longer fits. func (c *claimingConsumer) Consume(handler func(*mq.Message), prefetch int) (func(), <-chan error, error) { configured, _ := c.q.sq.IngestUnits() loopCtx, cancel := context.WithCancel(context.WithoutCancel(c.ctx)) l := &claimLoop{ q: c.q, ctx: loopCtx, cancel: cancel, cfg: c.cfg, handler: handler, prefetch: prefetch, slots: min(len(configured), maxMembers), configured: map[string]bool{}, - owned: map[string]*claim{}, releasing: map[string]struct{}{}, orphaned: map[string]time.Time{}, - held: newHeldRows(c.q.cfg.MaxHeld), - failed: make(chan error, 1), quit: make(chan struct{}), done: make(chan struct{}), + owned: map[string]*claim{}, releasing: map[string]*claim{}, orphaned: map[string]time.Time{}, + unbound: map[string]time.Time{}, + failed: make(chan error, 1), halting: make(chan struct{}), quit: make(chan struct{}), done: make(chan struct{}), } + l.share.Store(int64(c.q.cfg.MaxHeld)) for _, u := range configured { l.configured[u] = true } @@ -202,9 +218,19 @@ func (c *claimingConsumer) Consume(handler func(*mq.Message), prefetch int) (fun return l.stop, l.failed, nil } -// claimLoop is one process's claims. member, owned, prev and lastUnowned -// belong to the run goroutine (and to stop once it has exited); releasing -// is shared with the handovers. +// Halt implements mq.Halter: no unit is taken or given up any more, and every +// unit stops fetching while keeping its pin; it returns once what they +// fetched has reached the handler. stop, after the worker has flushed, +// releases them. +func (c *claimingConsumer) Halt() { + if l := c.loop.Load(); l != nil { + l.halt() + } +} + +// claimLoop is one process's claims. member, owned, prev, unbound and +// lastUnowned belong to the run goroutine (and to halt and stop once it has +// exited); releasing is shared with the handovers. type claimLoop struct { q *claimingQueue ctx context.Context @@ -222,14 +248,19 @@ type claimLoop struct { prev map[string]int // the previous tick's owners, nil before the first // orphaned holds the units taken over from a dead owner that still wait // for a reset, and since when. - orphaned map[string]time.Time + orphaned map[string]time.Time + // unbound holds the configured units whose bind keeps failing, and since + // when. + unbound map[string]time.Time lastUnowned time.Time mu sync.Mutex - releasing map[string]struct{} + releasing map[string]*claim handovers sync.WaitGroup - held *heldRows + // share is each unit's cap on rows delivered and unsettled, as of the + // last tick; held counts them across the units. + share, held atomic.Int64 ownedGauge, membersGauge, unownedGauge atomic.Int64 rankZero atomic.Bool @@ -237,8 +268,9 @@ type claimLoop struct { failed chan error reported, stopped atomic.Bool - quit, done chan struct{} - stopOnce sync.Once + // halting ends the ticks (halt); quit also cuts handovers short (stop). + halting, quit, done chan struct{} + haltOnce, stopOnce sync.Once } // claim is one unit this process holds. @@ -247,19 +279,23 @@ type claim struct { cons mq.Consumer stopOnce sync.Once stopFn func() - // inflight counts rows delivered and not yet settled; last is when the - // latest was delivered (unix nanoseconds). + // inflight counts rows delivered and not yet settled. inflight atomic.Int64 - last atomic.Int64 - quit chan struct{} + // halted closes once the unit has stopped fetching. + halted chan struct{} } +// stop halts the unit's fetching, keeping its pin, and returns once what it +// fetched has reached the handler, where the consumer can tell (mq.Halter). func (cl *claim) stop() { cl.stopOnce.Do(func() { - close(cl.quit) + if h, ok := cl.cons.(mq.Halter); ok { + h.Halt() + } if cl.stopFn != nil { cl.stopFn() } + close(cl.halted) }) } @@ -270,7 +306,7 @@ func (l *claimLoop) run() { for { l.tick() select { - case <-l.quit: + case <-l.halting: return case <-t.C: } @@ -309,6 +345,9 @@ func (l *claimLoop) tick() { } } } + if len(targets) > 0 { + l.share.Store(int64(unitShare(l.q.cfg.MaxHeld, len(targets)))) + } l.giveUp(targets) l.take(ctx, targets, len(targets), live) l.prev = assignment @@ -400,7 +439,7 @@ func (l *claimLoop) giveUp(targets map[string]bool) { cl := l.owned[u] delete(l.owned, u) l.mu.Lock() - l.releasing[u] = struct{}{} + l.releasing[u] = cl l.mu.Unlock() l.handovers.Go(func() { l.handOver(cl) }) } @@ -424,7 +463,7 @@ func (l *claimLoop) take(ctx context.Context, targets map[string]bool, count int if since, ok := l.orphaned[u]; ok { switch reset, err := l.q.sq.ResetOrphaned(ctx, u); { case errors.Is(err, mq.ErrUnitHeld) && time.Since(since) < orphanWait: - continue // the dead owner's pin has not lapsed yet + continue // its pin has not lapsed yet, or it was active too recently case errors.Is(err, mq.ErrUnitHeld): case err != nil: l.warn("ingest: could not take over a shard's unsettled rows; they come back after ack_wait", err) @@ -439,19 +478,32 @@ func (l *claimLoop) take(ctx context.Context, targets map[string]bool, count int slog.Info("ingest: could not bind a shard durable outside the configured ones", "unit", u, "error", err) continue } - if errors.Is(err, mq.ErrConsumerNotFound) { - // Its durable is gone: the same end as a delivery the - // broker ends because the durable was deleted under it. + if errors.Is(err, mq.ErrConsumerNotFound) || errors.Is(err, mq.ErrConsumerMismatch) { + // Its durable or stream is gone, or the durable no longer + // fits: the same end as a delivery the broker ends because + // the durable was deleted under it. l.fail(fmt.Errorf("shard %s: %w: %w", u, mq.ErrDeliveryEnded, err)) return } // Anything else (a request the server did not answer while a // consumer leader moves) is tried again next tick; the unit - // shows as unowned meanwhile. + // shows as unowned meanwhile, and a failure that lasts logs an + // error. claimEvents.Add(context.Background(), 1, eventAttr("bind_failed")) - l.warn("ingest: could not bind a shard; trying again next tick", fmt.Errorf("shard %s: %w", u, err)) + since, ok := l.unbound[u] + if !ok { + since = time.Now() + l.unbound[u] = since + } + err = fmt.Errorf("shard %s: %w", u, err) + if unbound := time.Since(since); unbound >= bindEscalate && l.ctx.Err() == nil { + slog.Error("ingest: a shard has not been bound for a while; its rows wait", "unbound_for", unbound.Round(time.Second), "error", err) + } else { + l.warn("ingest: could not bind a shard; trying again next tick", err) + } continue } + delete(l.unbound, u) } } @@ -462,32 +514,35 @@ func (l *claimLoop) isReleasing(u string) bool { return ok } -// open binds unit u and starts delivering it, counting each row until it -// settles and holding it against the process-wide cap. +// unitShare is each of count units' cap on rows delivered and unsettled. +func unitShare(maxHeld, count int) int { + return max(min(maxHeld, minUnitHeld), maxHeld/max(1, count)) +} + +// open binds unit u and starts delivering it, under the unit's share of +// MaxHeld, counting each row until it settles. func (l *claimLoop) open(u string, count int) error { cfg := l.cfg cfg.Units = []string{u} + cfg.MaxHeld = func() int { return int(l.share.Load()) } cons, err := l.q.sq.CreateConsumer(l.ctx, cfg) if err != nil { return err } - cl := &claim{unit: u, cons: cons, quit: make(chan struct{})} + cl := &claim{unit: u, cons: cons, halted: make(chan struct{})} share := l.prefetch if share > 0 { share = max(1, share/max(1, count)) } stop, failed, err := cons.Consume(func(m *mq.Message) { - if !l.held.acquire(cl.quit, l.quit) { - return // stopping: the row is unsettled and comes back - } cl.inflight.Add(1) - cl.last.Store(time.Now().UnixNano()) + l.held.Add(1) // The row leaves this process's hands at its first settle attempt, // or, if the worker never makes one (it leaves some rows for the // broker to redeliver), once the broker would redeliver it anyway. letGo := sync.OnceFunc(func() { cl.inflight.Add(-1) - l.held.release() + l.held.Add(-1) }) expire := time.AfterFunc(l.q.cfg.SlotExpiry, letGo) m.OnSettled(func() { expire.Stop(); letGo() }) @@ -515,22 +570,23 @@ func (l *claimLoop) watch(cl *claim, failed <-chan error) { return } l.fail(fmt.Errorf("shard %s: %w", cl.unit, err)) - case <-cl.quit: + case <-cl.halted: case <-l.quit: } }() } -// handOver stops fetching cl's unit, waits for what it delivered to settle, -// bounded by Handover (or cut short by stop), and releases it. +// handOver stops fetching cl's unit, keeping its pin, waits for what it +// delivered to settle, bounded by Handover (or cut short by stop), and +// releases it. func (l *claimLoop) handOver(cl *claim) { began := time.Now() cl.stop() deadline := time.After(l.q.cfg.Handover) - poll := time.NewTicker(settleQuiet / 5) + poll := time.NewTicker(settlePoll) defer poll.Stop() wait: - for cl.inflight.Load() > 0 || time.Since(time.Unix(0, cl.last.Load())) < settleQuiet { + for cl.inflight.Load() > 0 { select { case <-poll.C: case <-deadline: @@ -595,18 +651,35 @@ func (l *claimLoop) fail(err error) { l.failed <- err } -// stop ends the claims: every unit stops, is released, and the membership -// lease is resigned, at once — the worker has flushed what it held before -// calling it. +// halt ends the ticks and halts every unit held or being handed over, at +// once, returning when none will deliver again. +func (l *claimLoop) halt() { + l.haltOnce.Do(func() { + close(l.halting) + <-l.done + var halts sync.WaitGroup + for _, cl := range l.owned { + halts.Go(cl.stop) + } + l.mu.Lock() + for _, cl := range l.releasing { + halts.Go(cl.stop) + } + l.mu.Unlock() + halts.Wait() + }) +} + +// stop ends the claims: every unit is halted, then released, and the +// membership lease resigned, at once. The worker halts the claims and +// flushes what it held before calling it, so each unit's next owner receives +// only rows newer than the ones written here. func (l *claimLoop) stop() { l.stopOnce.Do(func() { l.stopped.Store(true) l.cancel() + l.halt() close(l.quit) - <-l.done - for _, cl := range l.owned { - cl.stop() - } l.handovers.Wait() // At once, so a broker that no longer answers costs one release's // wait, not one per unit. @@ -631,37 +704,6 @@ func (l *claimLoop) stop() { }) } -// heldRows caps the rows this process holds delivered and unsettled. -type heldRows struct { - slots chan struct{} -} - -func newHeldRows(n int) *heldRows { return &heldRows{slots: make(chan struct{}, n)} } - -// acquire takes a slot, waiting while every one is taken, false if either -// quit channel closes first. -func (h *heldRows) acquire(a, b <-chan struct{}) bool { - select { - case h.slots <- struct{}{}: - return true - default: - } - heldWaits.Add(context.Background(), 1) - select { - case h.slots <- struct{}{}: - return true - case <-a: - return false - case <-b: - return false - } -} - -// release frees the slot a row took; each row frees exactly one. -func (h *heldRows) release() { <-h.slots } - -func (h *heldRows) count() int { return len(h.slots) } - var ( claimEvents, _ = otel.Meter("wavehouse-ingest").Int64Counter( "wavehouse_ingest_shard_events_total", @@ -672,10 +714,6 @@ var ( metric.WithDescription("Time from giving a shard up to releasing it: the drain of what it had delivered"), metric.WithUnit("s"), ) - heldWaits, _ = otel.Meter("wavehouse-ingest").Int64Counter( - "wavehouse_ingest_rows_held_waits_total", - metric.WithDescription("Deliveries that waited because this process already held its cap of unsettled rows"), - ) ) // claimMeter is the meter the claim gauges register on (tests substitute @@ -714,7 +752,7 @@ func registerClaimGauges(l *claimLoop) (metric.Registration, error) { return meter.RegisterCallback(func(_ context.Context, o metric.Observer) error { o.ObserveInt64(owned, l.ownedGauge.Load()) o.ObserveInt64(members, l.membersGauge.Load()) - o.ObserveInt64(held, int64(l.held.count())) + o.ObserveInt64(held, l.held.Load()) if l.rankZero.Load() { o.ObserveInt64(unowned, l.unownedGauge.Load()) } diff --git a/internal/ingest/claims_test.go b/internal/ingest/claims_test.go index 92e26ff3a..09071ccc8 100644 --- a/internal/ingest/claims_test.go +++ b/internal/ingest/claims_test.go @@ -188,7 +188,7 @@ func (p *fakeProc) CreateConsumer(_ context.Context, cfg mq.ConsumerConfig) (mq. return nil, errors.New("no such durable") } p.f.record(p.name, "create", u) - return &fakeUnitConsumer{p: p, unit: u, failed: make(chan error, 1)}, nil + return &fakeUnitConsumer{p: p, unit: u, failed: make(chan error, 1), maxHeld: cfg.MaxHeld}, nil } type fakeUnitConsumer struct { @@ -197,6 +197,7 @@ type fakeUnitConsumer struct { handler func(*mq.Message) failed chan error running atomic.Bool + maxHeld func() int } func (c *fakeUnitConsumer) Consume(handler func(*mq.Message), _ int) (func(), <-chan error, error) { @@ -352,7 +353,7 @@ func TestClaims_HandoverWaitsForDeliveredRows(t *testing.T) { pending.Add(1) require.True(t, f.deliver(u, nil)) } - require.Eventually(t, func() bool { return a.loop.held.count() == 4 }, 5*time.Second, 10*time.Millisecond) + require.Eventually(t, func() bool { return a.loop.held.Load() == 4 }, 5*time.Second, 10*time.Millisecond) startClaims(t, f, c.Peer(), "b", ClaimConfig{}, nil) require.Eventually(t, func() bool { return slices.ContainsFunc(f.eventLog(), func(e string) bool { return strings.HasPrefix(e, "a stop ") }) @@ -364,7 +365,7 @@ func TestClaims_HandoverWaitsForDeliveredRows(t *testing.T) { close(release) pending.Wait() require.Eventually(t, func() bool { return f.split("a", "b") }, 5*time.Second, 10*time.Millisecond) - assert.Zero(t, a.loop.held.count(), "every settled row gave its slot back") + assert.Zero(t, a.loop.held.Load(), "every settled row gave its slot back") } // A process that loses its membership lease gives every unit up. @@ -407,6 +408,7 @@ func crash(t *testing.T, p *claimProc) { l.stopOnce.Do(func() { // stop is then a no-op l.stopped.Store(true) l.cancel() + close(l.halting) close(l.quit) <-l.done require.NoError(t, l.member.Resign(context.Background())) @@ -514,75 +516,34 @@ func TestClaims_EndedDelivery(t *testing.T) { }) } -// Rows delivered and unsettled across every unit stop at MaxHeld: a unit at -// the cap waits, and resumes as rows settle. -func TestClaims_HeldRowsCap(t *testing.T) { +// Each unit's cap on rows delivered and unsettled is an even share of +// MaxHeld over the units the process is assigned, never below two batches, +// so one stuck unit cannot take the others' room; it follows the assignment. +func TestClaims_UnitShareOfMaxHeld(t *testing.T) { t.Parallel() - f := newFakeShards(8) - var received atomic.Int64 - var mu sync.Mutex - var held []*mq.Message - p := startClaims(t, f, coord.NewLocal(), "a", ClaimConfig{MaxHeld: 5}, func(m *mq.Message) { - received.Add(1) - mu.Lock() - held = append(held, m) - mu.Unlock() - }) - require.Eventually(t, func() bool { return f.split("a") }, 5*time.Second, 10*time.Millisecond) - for i := range 20 { - require.True(t, f.deliver(f.units[i%8], nil)) - } - require.Eventually(t, func() bool { return received.Load() == 5 }, 5*time.Second, 10*time.Millisecond) - time.Sleep(100 * time.Millisecond) - assert.Equal(t, int64(5), received.Load(), "never above the cap") - assert.Equal(t, 5, p.loop.held.count()) + assert.Equal(t, 5000, unitShare(10_000, 2)) + assert.Equal(t, minUnitHeld, unitShare(10_000, 32), "never below two batches") + assert.Equal(t, 64, unitShare(64, 32), "nor above MaxHeld") - mu.Lock() - for _, m := range held[:3] { - require.NoError(t, m.Ack()) - } - mu.Unlock() - require.Eventually(t, func() bool { return received.Load() == 8 }, 5*time.Second, 10*time.Millisecond, "settling three admits three") -} - -// A batcher that holds rows until a timer flushes them never deadlocks -// against the cap: each flush settles rows, which admits more. -func TestClaims_HeldRowsCapDrainsThroughFlushes(t *testing.T) { - t.Parallel() - f := newFakeShards(4) - var mu sync.Mutex - var batch []*mq.Message - var acked atomic.Int64 - startClaims(t, f, coord.NewLocal(), "a", ClaimConfig{MaxHeld: 10}, func(m *mq.Message) { - mu.Lock() - batch = append(batch, m) - mu.Unlock() - }) + f, c := newFakeShards(8), coord.NewLocal() + a := startClaims(t, f, c, "a", ClaimConfig{MaxHeld: 8000}, nil) require.Eventually(t, func() bool { return f.split("a") }, 5*time.Second, 10*time.Millisecond) - stop := make(chan struct{}) - defer close(stop) - go func() { // the batch timer: flush whatever is held every 20ms - tk := time.NewTicker(20 * time.Millisecond) - defer tk.Stop() - for { - select { - case <-stop: - return - case <-tk.C: - mu.Lock() - flush := batch - batch = nil - mu.Unlock() - for _, m := range flush { - _ = m.Ack() - } + caps := func(proc string) map[int]int { + f.mu.Lock() + defer f.mu.Unlock() + out := map[int]int{} + for _, by := range f.consumers { + if cons, ok := by[proc]; ok && cons.running.Load() { + out[cons.maxHeld()]++ } } - }() - for i := range 200 { - require.True(t, f.deliver(f.units[i%4], func() { acked.Add(1) })) + return out } - require.Eventually(t, func() bool { return acked.Load() == 200 }, 10*time.Second, 10*time.Millisecond, "acked %d", acked.Load()) + assert.Equal(t, map[int]int{1000: 8}, caps("a"), "8000 over 8 units") + startClaims(t, f, c.Peer(), "b", ClaimConfig{MaxHeld: 8000}, nil) + require.Eventually(t, func() bool { return f.split("a", "b") }, 5*time.Second, 10*time.Millisecond) + require.Eventually(t, func() bool { return a.loop.share.Load() == 2000 }, 5*time.Second, 10*time.Millisecond) + assert.Equal(t, map[int]int{2000: 4}, caps("a"), "the units kept read the new share") } // Only the lowest live member counts the units without an owner. @@ -665,14 +626,14 @@ func TestClaims_GaugesReport(t *testing.T) { //nolint:paralleltest // substitute } // A row the worker lets go of without the broker confirming it (an ack that -// failed) frees its slot all the same: the broker redelivers it as a new row, -// which takes a slot of its own. +// failed) is no longer counted as held: the broker redelivers it as a new +// row. func TestClaims_HeldRowsFreedOnAFailedSettle(t *testing.T) { t.Parallel() f := newFakeShards(2) fail := errors.New("no answer") var received atomic.Int64 - p := startClaims(t, f, coord.NewLocal(), "a", ClaimConfig{MaxHeld: 3}, func(m *mq.Message) { + p := startClaims(t, f, coord.NewLocal(), "a", ClaimConfig{}, func(m *mq.Message) { received.Add(1) _ = m.Ack() }) @@ -680,26 +641,26 @@ func TestClaims_HeldRowsFreedOnAFailedSettle(t *testing.T) { for i := range 10 { require.True(t, f.deliverFailing(f.units[i%2], fail)) } - require.Eventually(t, func() bool { return received.Load() == 10 }, 5*time.Second, 10*time.Millisecond, "no slot leaks: got %d", received.Load()) - require.Eventually(t, func() bool { return p.loop.held.count() == 0 }, 5*time.Second, 10*time.Millisecond) + require.Eventually(t, func() bool { return received.Load() == 10 }, 5*time.Second, 10*time.Millisecond) + require.Eventually(t, func() bool { return p.loop.held.Load() == 0 }, 5*time.Second, 10*time.Millisecond) } // A row the worker never settles (it leaves some for the broker to -// redeliver) frees its slot once the broker would redeliver it anyway. +// redeliver) stops counting as held once the broker would redeliver it +// anyway, so a handover does not wait on it past that. func TestClaims_HeldRowsExpire(t *testing.T) { t.Parallel() f := newFakeShards(1) var received atomic.Int64 - p := startClaims(t, f, coord.NewLocal(), "a", ClaimConfig{MaxHeld: 2, SlotExpiry: 200 * time.Millisecond}, func(*mq.Message) { + p := startClaims(t, f, coord.NewLocal(), "a", ClaimConfig{SlotExpiry: 200 * time.Millisecond}, func(*mq.Message) { received.Add(1) // never settled }) require.Eventually(t, func() bool { return f.split("a") }, 5*time.Second, 10*time.Millisecond) for range 4 { require.True(t, f.deliver(f.units[0], nil)) } - require.Eventually(t, func() bool { return received.Load() == 2 }, 5*time.Second, 10*time.Millisecond) - require.Eventually(t, func() bool { return received.Load() == 4 }, 5*time.Second, 10*time.Millisecond, "the expired slots admit the rest") - require.Eventually(t, func() bool { return p.loop.held.count() == 0 }, 5*time.Second, 10*time.Millisecond) + require.Eventually(t, func() bool { return received.Load() == 4 && p.loop.held.Load() == 4 }, 5*time.Second, 10*time.Millisecond) + require.Eventually(t, func() bool { return p.loop.held.Load() == 0 }, 5*time.Second, 10*time.Millisecond) } // A process whose membership lease ends under it takes a fresh one on its @@ -744,15 +705,14 @@ func TestClaims_FirstTickResetsOnlyWhenAlone(t *testing.T) { } // stop cuts a handover short: it does not wait out Handover for rows the -// worker will never settle, and a delivery waiting at the cap returns -// without reaching the handler. +// worker will never settle. func TestClaims_StopCutsHandoversShort(t *testing.T) { t.Parallel() f, c := newFakeShards(4), coord.NewLocal() var received atomic.Int64 - a := startClaims(t, f, c, "a", ClaimConfig{Handover: time.Minute, MaxHeld: 4}, func(*mq.Message) { received.Add(1) }) + a := startClaims(t, f, c, "a", ClaimConfig{Handover: time.Minute}, func(*mq.Message) { received.Add(1) }) require.Eventually(t, func() bool { return f.split("a") }, 5*time.Second, 10*time.Millisecond) - for i := range 6 { // four fill the cap, two wait at it + for i := range 4 { require.True(t, f.deliver(f.units[i%4], nil)) } require.Eventually(t, func() bool { return received.Load() == 4 }, 5*time.Second, 10*time.Millisecond) @@ -768,8 +728,6 @@ func TestClaims_StopCutsHandoversShort(t *testing.T) { case <-time.After(10 * time.Second): t.Fatal("stop waited out the handover") } - time.Sleep(100 * time.Millisecond) - assert.Equal(t, int64(4), received.Load(), "the waiting deliveries never reached the handler") } // Membership slots are capped: a queue of many units still reads at most @@ -782,27 +740,33 @@ func TestClaims_MembershipSlotsCapped(t *testing.T) { assert.Equal(t, maxMembers, p.loop.slots) } -// A configured unit whose durable is gone when it is bound ends delivery, -// the same end as the broker ending a durable deleted under it. +// A configured unit whose durable (or its stream) is gone, or no longer fits +// the worker's config, when it is bound ends delivery, the same end as the +// broker ending a durable deleted under it: no later tick can bind it. func TestClaims_MissingDurableEndsDelivery(t *testing.T) { t.Parallel() - f := newFakeShards(2) - bad := "S/u-01" - p := &fakeProc{f: f, name: "a", failWith: fmt.Errorf("consumer %s: %w", bad, mq.ErrConsumerNotFound)} - p.failCreate.Store(&bad) - q, err := ClaimShards(p, coord.NewLocal(), fastClaims(ClaimConfig{})) - require.NoError(t, err) - cons, err := q.CreateConsumer(t.Context(), mq.ConsumerConfig{}) - require.NoError(t, err) - stop, failed, err := cons.Consume(func(*mq.Message) {}, 10) - require.NoError(t, err) - t.Cleanup(stop) - select { - case err := <-failed: - assert.ErrorIs(t, err, mq.ErrDeliveryEnded) - assert.ErrorIs(t, err, mq.ErrConsumerNotFound) - case <-time.After(5 * time.Second): - t.Fatal("a missing durable did not end delivery") + for _, cause := range []error{mq.ErrConsumerNotFound, mq.ErrConsumerMismatch} { + t.Run(cause.Error(), func(t *testing.T) { + t.Parallel() + f := newFakeShards(2) + bad := "S/u-01" + p := &fakeProc{f: f, name: "a", failWith: fmt.Errorf("consumer %s: %w", bad, cause)} + p.failCreate.Store(&bad) + q, err := ClaimShards(p, coord.NewLocal(), fastClaims(ClaimConfig{})) + require.NoError(t, err) + cons, err := q.CreateConsumer(t.Context(), mq.ConsumerConfig{}) + require.NoError(t, err) + stop, failed, err := cons.Consume(func(*mq.Message) {}, 10) + require.NoError(t, err) + t.Cleanup(stop) + select { + case err := <-failed: + assert.ErrorIs(t, err, mq.ErrDeliveryEnded) + assert.ErrorIs(t, err, cause) + case <-time.After(5 * time.Second): + t.Fatal("an unbindable durable did not end delivery") + } + }) } } diff --git a/internal/ingest/worker.go b/internal/ingest/worker.go index 5ec26ac6a..523eb1c85 100644 --- a/internal/ingest/worker.go +++ b/internal/ingest/worker.go @@ -246,23 +246,29 @@ func waitOrDeadline(ctx context.Context, wg *sync.WaitGroup) error { // so a low-volume table can never strand another table's rows behind a shared // timer. It is the ONLY goroutine that watches ctx; tableLoops stop via // channel-close, which gives a deterministic drain with no abandoned messages. +// +// Stopping keeps each table's rows in order across the processes that share a +// sharded queue: the consumer is halted first (mq.Halter), keeping its hold on +// its units, and what it still delivers is routed like any other row; then +// every table flushes; only then does the consumer's stop give the units up. +// A delivery that arrives once this loop reads no more is NAKed at once, +// before that stop, so it comes back ahead of anything newer rather than +// after ack_wait. func (w *IngestWorker) dispatchLoop(ctx context.Context, cons mq.Consumer) { defer w.wg.Done() msgChan := make(chan *mq.Message, w.maxBatch*2) + abandon := make(chan struct{}) // Pull consumer with a push-like callback (the client prefetches pullMaxMessages, // shared by the tenants' queues). It runs on one delivery goroutine per tenant, // so the handoff is a channel send, safe from all of them at once. Hand off to // msgChan only, so a consume goroutine never blocks on flush work. - // The handoff also watches ctx: stop (deferred below) does not wait for a - // delivery already in the handler, so once this loop has stopped draining - // msgChan a full channel would otherwise pin a delivery goroutine - // forever. A message dropped here is unacked and simply redelivered. stop, deliveryEnded, err := cons.Consume(func(msg *mq.Message) { select { case msgChan <- msg: - case <-ctx.Done(): + case <-abandon: + nakAbandoned(msg) } }, pullMaxMessages) if err != nil { @@ -270,7 +276,24 @@ func (w *IngestWorker) dispatchLoop(ctx context.Context, cons mq.Consumer) { w.failed <- fmt.Errorf("ingest worker: start consumer: %w", err) return } - defer stop() + nakLeft := func() { + for { + select { + case m := <-msgChan: + nakAbandoned(m) + default: + return + } + } + } + defer func() { + close(abandon) + nakLeft() + stop() + // A consumer that cannot halt may still have been handing one + // over; it has no units to give up, so after its stop is as good. + nakLeft() + }() // flushCtx carries values (trace) but is never cancelled: a started flush must // finish so data already in ClickHouse gets acked rather than redelivered. It @@ -302,52 +325,89 @@ func (w *IngestWorker) dispatchLoop(ctx context.Context, cons mq.Consumer) { w.ackWg.Wait() } + // route hands a message to its table's loop. A tableLoop only blocks in + // its final drain, which starts after the last route, so the send waits + // at most for the loop to take the rows ahead of it. + route := func(m *mq.Message) { + pm, ok := w.parseMsg(flushCtx, m) + if !ok { + return // unreadable envelope: parked on the DLQ (or acked-and-dropped) in parseMsg + } + key := batchKey{tenant: pm.tenant, table: pm.tableName} + ch, exists := tableChans[key] + if !exists { + ch = make(chan parsedMsg, w.maxBatch) + tableChans[key] = ch + + // TODO(#263): tableLoops are spawned per distinct tenant table and + // never reaped — they live for the process lifetime. Safe while + // tenants and table names are bounded (a settings folder per + // tenant, schema-validated tables, in-process publishers only, + // DontListen:true). Add idle-reaping + route/teardown coordination + // before remote/untrusted publishers can create unbounded cardinality. + table, in := pm.tableName, ch + tableWg.Go(func() { w.tableLoop(flushCtx, table, in) }) + } + ch <- pm + } + for { select { case <-ctx.Done(): + haltRouting(cons, msgChan, route) shutdown() return case err := <-deliveryEnded: // The MQ gave up on the consumer: no message will arrive again, so // waiting on msgChan would stall ingestion silently. Flush and ack // what is already in hand — those rows are delivered and the - // publish path is not what broke — then fail loud. Messages still - // in msgChan are unacked and redelivered to the next consumer. + // publish path is not what broke — then fail loud. slog.ErrorContext(ctx, "ingest consumer delivery ended; ingestion has stopped", "error", err) + haltRouting(cons, msgChan, route) shutdown() w.failed <- fmt.Errorf("ingest worker: %w", err) return case m := <-msgChan: - pm, ok := w.parseMsg(flushCtx, m) - if !ok { - continue // unreadable envelope: parked on the DLQ (or acked-and-dropped) in parseMsg - } - key := batchKey{tenant: pm.tenant, table: pm.tableName} - ch, exists := tableChans[key] - if !exists { - ch = make(chan parsedMsg, w.maxBatch) - tableChans[key] = ch - - // TODO(#263): tableLoops are spawned per distinct tenant table and - // never reaped — they live for the process lifetime. Safe while - // tenants and table names are bounded (a settings folder per - // tenant, schema-validated tables, in-process publishers only, - // DontListen:true). Add idle-reaping + route/teardown coordination - // before remote/untrusted publishers can create unbounded cardinality. - table, in := pm.tableName, ch - tableWg.Go(func() { w.tableLoop(flushCtx, table, in) }) - } - // Route to the table's loop, but stay responsive to shutdown if its - // channel is full (a busy tableLoop must not wedge teardown). A pm - // dropped here is unacked and simply redelivered. + route(m) + } + } +} + +// haltRouting halts cons, if it can halt ahead of its stop, routing what it +// still delivers until it has, then routes what msgChan still holds. +func haltRouting(cons mq.Consumer, msgChan <-chan *mq.Message, route func(*mq.Message)) { + if h, ok := cons.(mq.Halter); ok { + halted := make(chan struct{}) + go func() { + h.Halt() + close(halted) + }() + halting: + for { select { - case ch <- pm: - case <-ctx.Done(): - shutdown() - return + case m := <-msgChan: + route(m) + case <-halted: + break halting } } } + for { + select { + case m := <-msgChan: + route(m) + default: + return + } + } +} + +// nakAbandoned hands back a delivery the worker will not write: redelivered +// at once, to whichever process owns its unit next. +func nakAbandoned(m *mq.Message) { + if err := m.Nak(); err != nil { + slog.Debug("ingest: could not hand back an undelivered row; it comes back after the ack wait", "error", err) + } } // tableBatcher accumulates one table's rows and flushes them to ClickHouse. diff --git a/internal/ingest/worker_test.go b/internal/ingest/worker_test.go index 42251f0da..26a6f7479 100644 --- a/internal/ingest/worker_test.go +++ b/internal/ingest/worker_test.go @@ -2471,3 +2471,123 @@ func TestTableBatcher_Add_HandsRowsBackWhileTheProbeIsOut(t *testing.T) { assert.True(t, m.Naked.Load()) assert.GreaterOrEqual(t, time.Duration(m.NakDelay.Load()), retryBase/2) } + +// scriptedConsumer hands the worker's handler to the test and records, in +// order, what became of each row and when the consumer was stopped. +type scriptedConsumer struct { + mu sync.Mutex + handler func(*mq.Message) + events []string +} + +func (c *scriptedConsumer) Consume(h func(*mq.Message), _ int) (func(), <-chan error, error) { + c.mu.Lock() + c.handler = h + c.mu.Unlock() + return func() { c.record("stop") }, make(chan error), nil +} + +func (c *scriptedConsumer) record(e string) { + c.mu.Lock() + defer c.mu.Unlock() + c.events = append(c.events, e) +} + +func (c *scriptedConsumer) log() []string { + c.mu.Lock() + defer c.mu.Unlock() + return slices.Clone(c.events) +} + +func (c *scriptedConsumer) deliver(t *testing.T, id string) { + t.Helper() + msg := mq.NewMessage(context.Background(), mq.Topic{Tenant: tenant.Default, Table: "events"}, + makeEnvelope(t, "events", "", map[string]any{"id": id}), time.Now(), + func(context.Context) error { c.record("ack " + id); return nil }, + func() error { c.record("ack " + id); return nil }, + func() error { c.record("nak " + id); return nil }) + c.mu.Lock() + h := c.handler + c.mu.Unlock() + h(msg) +} + +// haltingConsumer is a scriptedConsumer that halts ahead of its stop, and +// delivers one more row while halting, as a fetch in flight would. +type haltingConsumer struct { + *scriptedConsumer + t *testing.T +} + +func (c haltingConsumer) Halt() { + c.record("halt") + c.deliver(c.t, "late") +} + +// scriptedWorker runs a worker over cons whose ClickHouse records each +// insert on cons and then waits for gate, if set. +func scriptedWorker(t *testing.T, cons mq.Consumer, rec func(string), gate func()) (cancel func(), done <-chan struct{}) { + t.Helper() + ch := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(r.Body) + rec(fmt.Sprintf("insert %d", strings.Count(string(body), "\n"))) + if gate != nil { + gate() + } + w.WriteHeader(http.StatusOK) + })) + t.Cleanup(ch.Close) + w := &IngestWorker{ + clients: chconn.NewHTTPClients(ingestHTTPClient), + cache: &testutil.MockCache{}, + target: func(tenant.ID) chconn.Target { return chconn.Target{URL: ch.URL, Database: "db"} }, + // Flushed only at the stop. + maxBatch: defaultMaxBatch, + maxWait: time.Hour, + } + ctx, stop := context.WithCancel(context.Background()) + finished := make(chan struct{}) + w.wg.Add(1) + go func() { w.dispatchLoop(ctx, cons); close(finished) }() + t.Cleanup(func() { stop(); <-finished }) + return stop, finished +} + +// Stopping halts the consumer before the final flush, writes what it still +// delivered while halting, and stops it — giving its units up — only after. +func TestIngestWorker_StopHaltsBeforeTheFlushAndStopsAfter(t *testing.T) { + t.Parallel() + cons := haltingConsumer{scriptedConsumer: &scriptedConsumer{}, t: t} + cancel, done := scriptedWorker(t, cons, cons.record, nil) + require.Eventually(t, func() bool { cons.mu.Lock(); defer cons.mu.Unlock(); return cons.handler != nil }, 5*time.Second, time.Millisecond) + cons.deliver(t, "a") + cons.deliver(t, "b") + cancel() + <-done + log := cons.log() + require.Len(t, log, 6, "%v", log) + require.Equal(t, "halt", log[0], "%v", log) + assert.Equal(t, "insert 3", log[1], "the row delivered while halting is written with the rest: %v", log) + assert.ElementsMatch(t, []string{"ack a", "ack b", "ack late"}, log[2:5]) + assert.Equal(t, []string{"stop"}, log[5:], "stopped only after the flush: %v", log) +} + +// A consumer that cannot halt may still deliver while the worker flushes: +// those rows are NAKed, in the order they arrived, before its stop. +func TestIngestWorker_StopNaksWhatArrivesAfterItStopsReading(t *testing.T) { + t.Parallel() + cons := &scriptedConsumer{} + flushing, release := make(chan struct{}), make(chan struct{}) + var once sync.Once + cancel, done := scriptedWorker(t, cons, cons.record, func() { once.Do(func() { close(flushing) }); <-release }) + require.Eventually(t, func() bool { cons.mu.Lock(); defer cons.mu.Unlock(); return cons.handler != nil }, 5*time.Second, time.Millisecond) + cons.deliver(t, "a") + cancel() + <-flushing + for _, id := range []string{"x", "y", "z"} { + cons.deliver(t, id) + } + close(release) + <-done + assert.Equal(t, []string{"insert 1", "ack a", "nak x", "nak y", "nak z", "stop"}, cons.log()) +} diff --git a/internal/mq/external.go b/internal/mq/external.go index d58f4af42..5012b3377 100644 --- a/internal/mq/external.go +++ b/internal/mq/external.go @@ -50,8 +50,9 @@ type NATSConfig struct { TopologyWait time.Duration // recheckEvery and historyEvery override the periodic checks' intervals - // (tests). - recheckEvery, historyEvery time.Duration + // (tests), and activeWithin how recently a unit's durable must have + // delivered or had an ack to count as someone's (default its pinned_ttl). + recheckEvery, historyEvery, activeWithin time.Duration } // NATSTLS is the client side of TLS to the NATS servers. @@ -106,6 +107,11 @@ type ExternalNATS struct { topo NATSTopology nc *nats.Conn js jetstream.JetStream + // apiPrefix is the JetStream API's subject prefix, the domain's if set. + apiPrefix string + // activeWithin overrides each durable's pinned_ttl as the window + // recentlyActive reads (tests). + activeWithin time.Duration // partitions is partition p's stream name, dlq the dead-letter stream's, // both found by subject at boot; units is every partition's shard @@ -169,10 +175,12 @@ func NewNATS(ctx context.Context, cfg NATSConfig) (*ExternalNATS, error) { return nil, errors.New("nats: no server URLs") } e := &ExternalNATS{ - topo: topo, - stoppers: map[int]func(){}, - loopDone: make(chan struct{}), - connClosed: make(chan struct{}), + topo: topo, + apiPrefix: jetstream.DefaultAPIPrefix, + activeWithin: cfg.activeWithin, + stoppers: map[int]func(){}, + loopDone: make(chan struct{}), + connClosed: make(chan struct{}), } opts, err := e.connectOptions(cfg) if err != nil { @@ -184,6 +192,7 @@ func NewNATS(ctx context.Context, cfg NATSConfig) (*ExternalNATS, error) { } e.connected.Store(e.nc.IsConnected()) if cfg.JSDomain != "" { + e.apiPrefix = "$JS." + cfg.JSDomain + ".API." e.js, err = jetstream.NewWithDomain(e.nc, cfg.JSDomain) } else { e.js, err = jetstream.New(e.nc) @@ -756,11 +765,13 @@ func (e *ExternalNATS) unit(name string) (natsUnit, bool, bool) { return natsUnit{}, false, false } -// ResetOrphaned implements Sharded. A pinned client holds the unit -// (ErrUnitHeld). With nobody pinned and rows awaiting an ack, whoever received -// them is gone: the reset -// redelivers them from the ack floor at once, where the server would wait -// out ack_wait. Rows already acked are not delivered again. +// ResetOrphaned implements Sharded. A pinned client holds the unit, and one +// that delivered or had an ack within the durable's pinned_ttl may still: +// its pin can be gone for a moment (a consumer leader change releases it) +// while it lives. Either is ErrUnitHeld. Otherwise, with rows awaiting an +// ack, whoever received them is gone: the reset redelivers them from the ack +// floor at once, where the server would wait out ack_wait. Rows already +// acked are not delivered again. func (e *ExternalNATS) ResetOrphaned(ctx context.Context, name string) (bool, error) { u, _, ok := e.unit(name) if !ok { @@ -777,6 +788,9 @@ func (e *ExternalNATS) ResetOrphaned(ctx context.Context, name string) (bool, er if info.NumAckPending == 0 { return false, nil } + if e.recentlyActive(info) { + return false, fmt.Errorf("unit %s: %w: it delivered or had an ack within %s", name, ErrUnitHeld, e.activeWindow(info)) + } if _, err := e.js.ResetConsumer(ctx, u.stream, u.durable); err != nil { return false, fmt.Errorf("reset %s: %w", name, e.apiError(err)) } @@ -784,8 +798,9 @@ func (e *ExternalNATS) ResetOrphaned(ctx context.Context, name string) (bool, er } // Unowned implements Sharded: the units whose durable has rows pending or -// awaiting an ack while no client holds its pin. A live owner keeps its pin -// by pulling, so such a unit has nobody writing it. +// awaiting an ack while no client holds its pin and none was active on it +// within its pinned_ttl. A live owner keeps its pin by pulling, even with +// nothing more to fetch, so such a unit has nobody writing it. func (e *ExternalNATS) Unowned(ctx context.Context) (int, error) { units := slices.Clone(e.units) if xs := e.extras.Load(); xs != nil { @@ -801,270 +816,44 @@ func (e *ExternalNATS) Unowned(ctx context.Context) (int, error) { return 0, fmt.Errorf("consumer %s: %w", u.id(), e.apiError(err)) } info := c.CachedInfo() - if (info.NumPending > 0 || info.NumAckPending > 0) && pinnedClient(info) == "" { + if (info.NumPending > 0 || info.NumAckPending > 0) && pinnedClient(info) == "" && !e.recentlyActive(info) { n++ } } return n, nil } -// pinnedClient is the pin id the server holds for WaveHouse's priority group -// on a consumer, "" when no client is pinned. -func pinnedClient(info *jetstream.ConsumerInfo) string { - for _, g := range info.PriorityGroups { - if g.Group == natsPriorityGroup { - return g.PinnedClientID - } +// activeWindow is how recently a durable must have delivered or had an ack +// to count as someone's: its pinned_ttl, the time the server gives a silent +// pinned client before it unpins it. +func (e *ExternalNATS) activeWindow(info *jetstream.ConsumerInfo) time.Duration { + if e.activeWithin > 0 { + return e.activeWithin } - return "" + return max(info.Config.PinnedTTL, minPinnedTTL) } -// CreateConsumer finds the operator's shard durables — it never creates one -// — and checks each against cfg: its ack_wait must cover cfg.AckWait and its -// max_ack_pending must be set. cfg.Units picks the units (nil: every one, -// the extras still draining included). A durable name that does not map to -// the operator's is ErrConsumerNotFound; an extra whose durable is gone by -// now is skipped. -func (e *ExternalNATS) CreateConsumer(ctx context.Context, cfg ConsumerConfig) (Consumer, error) { - if _, ok := e.durable(cfg.Durable); !ok { - return nil, fmt.Errorf("consumer %q: %w: the ingest durables are %s-", cfg.Durable, ErrConsumerNotFound, e.topo.IngestConsumer) - } - names := cfg.Units - if names == nil { - configured, extra := e.IngestUnits() - names = slices.Concat(configured, extra) - } - c := &externalConsumer{e: e, ctx: ctx, whole: cfg.Units == nil, failed: make(chan error, 1)} - for _, name := range names { - u, extra, ok := e.unit(name) - if !ok { - return nil, fmt.Errorf("consumer unit %q: %w", name, ErrUnitsUnsupported) +// recentlyActive reports whether a durable delivered a row, or its ack +// floor moved, within its activeWindow, by the server's own clock. +func (e *ExternalNATS) recentlyActive(info *jetstream.ConsumerInfo) bool { + window := e.activeWindow(info) + for _, last := range []*time.Time{info.Delivered.Last, info.AckFloor.Last} { + if last != nil && info.TimeStamp.Sub(*last) < window { + return true } - // A handle of its own: the client keeps the pin id per handle. - h, err := e.js.Consumer(ctx, u.stream, u.durable) - switch { - case extra && (errors.Is(err, jetstream.ErrConsumerNotFound) || errors.Is(err, jetstream.ErrStreamNotFound) || - errors.Is(err, jetstream.ErrNotPullConsumer)): - continue // drained and deleted since it was listed - case errors.Is(err, jetstream.ErrConsumerNotFound): - return nil, fmt.Errorf("consumer %s: %w", name, ErrConsumerNotFound) - case err != nil: - return nil, fmt.Errorf("consumer %s: %w", name, e.apiError(err)) - } - have := h.CachedInfo().Config - if have.AckWait < cfg.AckWait { - return nil, fmt.Errorf("consumer %s: ack_wait %s is shorter than the %s asked for", name, have.AckWait, cfg.AckWait) - } - if have.MaxAckPending <= 0 { - return nil, fmt.Errorf("consumer %s: max_ack_pending must be set", name) - } - if extra { - slog.Info("mq: draining a shard durable outside the configured ones", "component", "nats", "unit", name, "pending", h.CachedInfo().NumPending) - } - c.parts = append(c.parts, &consumerPart{unit: u, extra: extra, h: h}) } - return c, nil -} - -// externalConsumer is the operator's shard durables it was created for. -type externalConsumer struct { - e *ExternalNATS - ctx context.Context - // whole is a consumer of every unit, the one a process with no claims - // runs: it takes orphaned units over itself, and releases them on stop. - whole bool - parts []*consumerPart - failed chan error - // reported and stopped keep failed to one error, none after stop. - reported, stopped atomic.Bool -} - -type consumerPart struct { - unit natsUnit - // extra is outside the configured units: its delivery ending is the - // operator deleting it once drained, not a failure. - extra bool - h jetstream.Consumer - // pin is the pin id of the last row this process received from the - // unit, "" before the first: whether, and as whom, it held the unit. - pin atomic.Pointer[string] -} - -// Consume pulls every unit in the priority group, each on its own delivery -// goroutine, so the server delivers a unit to one pinned puller at a time. -// Prefetch is split between the configured units (at least one each), and an -// extra gets a quarter share. A unit's delivery that the client ends on its -// own — the durable deleted, the connection closed for good — is reported on -// failed; an extra's is only logged. stop drains: what was fetched still -// reaches handler, rather than going back to the server after ack_wait. -func (c *externalConsumer) Consume(handler func(msg *Message), prefetch int) (func(), <-chan error, error) { - var ( - mu sync.Mutex - running []jetstream.ConsumeContext - ) - stopAll := func() { - c.stopped.Store(true) - mu.Lock() - defer mu.Unlock() - for _, cc := range running { - cc.Drain() - } - } - own := 0 - for _, part := range c.parts { - if !part.extra { - own++ - } - } - if c.whole { - c.takeOver() - } - for _, part := range c.parts { - // The client calls this for passing conditions too, and stops the - // subscription itself on a terminal one: closing without our stop is - // what terminal means (see fanIn.run). - var lastErr atomic.Pointer[error] - opts := []jetstream.PullConsumeOpt{ - jetstream.PullPriorityGroup(natsPriorityGroup), - jetstream.PullExpiry(natsPullExpiry), - jetstream.ConsumeErrHandler(func(_ jetstream.ConsumeContext, err error) { - // Another puller holds the pin, or the consumer's leader - // moved: the client pulls again, and nothing is lost. - if errors.Is(err, jetstream.ErrPinIDMismatch) || errors.Is(err, jetstream.ErrConsumerLeadershipChanged) { - slog.Debug("mq: shard delivery paused", "component", "nats", "unit", part.unit.id(), "reason", err) - return - } - lastErr.Store(&err) - level := slog.LevelWarn - if part.extra { - level = slog.LevelInfo // expected once the operator deletes it - } - slog.Log(context.Background(), level, "mq: consumer reported an error", "component", "nats", "unit", part.unit.id(), "error", err) - }), - } - if prefetch > 0 { - share := max(1, prefetch/max(1, own)) - if part.extra { - share = max(1, share/4) - } - opts = append(opts, jetstream.PullMaxMessages(share)) - } - cc, err := part.h.Consume(func(m jetstream.Msg) { - if pin := m.Headers().Get(pinIDHeader); pin != "" { - part.pin.Store(&pin) - } - handler(c.e.wrapMsg(c.ctx, m, true)) - }, opts...) - if err != nil { - stopAll() - return nil, nil, fmt.Errorf("consume %s: %w", part.unit.id(), err) - } - mu.Lock() - running = append(running, cc) - mu.Unlock() - go func() { - <-cc.Closed() - if c.stopped.Load() { - if c.whole { - c.releaseAfterDrain(part) - } - return - } - reason := ErrDeliveryEnded - if r := lastErr.Load(); r != nil { - reason = fmt.Errorf("%w: %w", ErrDeliveryEnded, *r) - } - if part.extra { - slog.Info("mq: stopped draining a shard durable outside the configured ones", "component", "nats", "unit", part.unit.id(), "reason", reason) - return - } - c.fail(fmt.Errorf("%s: %w", part.unit.id(), reason)) - }() - } - untrack := c.e.track(stopAll) - return func() { - untrack() - stopAll() - }, c.failed, nil + return false } -// pinIDHeader carries, on a row the server delivered to the pinned puller, -// that puller's pin id. -const pinIDHeader = "Nats-Pin-Id" - -// takeOver resets every unit no one holds that has rows awaiting an ack, -// before this consumer pulls it: whoever received them is gone. -func (c *externalConsumer) takeOver() { - ctx, cancel := context.WithTimeout(c.ctx, recheckTimeout) - defer cancel() - for _, part := range c.parts { - if reset, err := c.e.ResetOrphaned(ctx, part.unit.id()); errors.Is(err, ErrUnitHeld) { - continue // another consumer holds it; the pin decides - } else if err != nil { - slog.Warn("mq: could not take over a shard's unsettled rows; they come back after ack_wait", "component", "nats", "unit", part.unit.id(), "error", err) - } else if reset { - slog.Info("mq: took over a shard's unsettled rows", "component", "nats", "unit", part.unit.id()) +// pinnedClient is the pin id the server holds for WaveHouse's priority group +// on a consumer, "" when no client is pinned. +func pinnedClient(info *jetstream.ConsumerInfo) string { + for _, g := range info.PriorityGroups { + if g.Group == natsPriorityGroup { + return g.PinnedClientID } } -} - -// releaseAfterDrain releases part once its drain has delivered what it -// fetched. A whole consumer's handler has no settlement to wait for here, -// and an unsettled row comes back to the next owner after ack_wait. -func (c *externalConsumer) releaseAfterDrain(part *consumerPart) { - if c.e.nc.IsClosed() { - return - } - ctx, cancel := context.WithTimeout(context.Background(), releaseTimeout) - defer cancel() - if err := c.e.release(ctx, part); err != nil { - slog.Debug("mq: could not release a shard; its pin lapses on its own", "component", "nats", "unit", part.unit.id(), "error", err) - } -} - -// releaseTimeout bounds one unit's release. -const releaseTimeout = 2 * time.Second - -// Release implements Releaser: each unit this consumer was last pinned on, -// and whose pin the server still gives it, is unpinned. -func (c *externalConsumer) Release(ctx context.Context) error { - var errs []error - for _, part := range c.parts { - errs = append(errs, c.e.release(ctx, part)) - } - return errors.Join(errs...) -} - -// release unpins part if this process still holds its pin. Checking first -// keeps a process that never received from the unit, or lost the pin since, -// from unpinning the real owner. -func (e *ExternalNATS) release(ctx context.Context, part *consumerPart) error { - pin := part.pin.Load() - if pin == nil { - return nil - } - info, err := part.h.Info(ctx) - if err != nil { - return fmt.Errorf("consumer %s: %w", part.unit.id(), e.apiError(err)) - } - if pinnedClient(info) != *pin { - return nil - } - s, err := e.js.Stream(ctx, part.unit.stream) - if err != nil { - return fmt.Errorf("stream %s: %w", part.unit.stream, e.apiError(err)) - } - if err := s.UnpinConsumer(ctx, part.unit.durable, natsPriorityGroup); err != nil { - return fmt.Errorf("unpin %s: %w", part.unit.id(), e.apiError(err)) - } - return nil -} - -func (c *externalConsumer) fail(err error) { - if c.stopped.Load() || !c.reported.CompareAndSwap(false, true) { - return - } - c.failed <- err + return "" } // DeadLetterCounts counts tenant id's parked messages on the shared diff --git a/internal/mq/external_consumer.go b/internal/mq/external_consumer.go new file mode 100644 index 000000000..277940d93 --- /dev/null +++ b/internal/mq/external_consumer.go @@ -0,0 +1,649 @@ +package mq + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "log/slog" + "slices" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" +) + +const ( + // pinIDHeader carries, on a row the server delivered to the pinned + // puller, that puller's pin id. + pinIDHeader = "Nats-Pin-Id" + // releaseTimeout bounds one unit's release. + releaseTimeout = 2 * time.Second + // renewEvery is how often a unit at its cap, or halted, sends the server + // a pull that keeps its pin. The server resets + // the pin's timer on each pull from the pinned client, and the verifier + // holds pinned_ttl to at least twice natsPullExpiry, so the pin cannot + // lapse between two. + renewEvery = natsPullExpiry + // renewWait is how long a renewing pull waits: short, so that most of + // the time no pull of a paused unit is waiting, and a durable deleted + // meanwhile answers the next one with no responders. + renewWait = time.Second + // fetchExpiry is how long one fetch waits for rows. A halt waits for the + // fetch in flight to end rather than cut it short, since a row the server + // already sent to a pull its client dropped would be redelivered only + // after ack_wait, behind newer rows: so this bounds how long a halt, and + // with it a handover, takes. An idle unit costs one pull a second. + fetchExpiry = time.Second + // fetchRetryWait spaces the fetches of a unit whose last one failed. + fetchRetryWait = 250 * time.Millisecond + // noAnswersBeforeCheck is how many pulls in a row may get no responders + // (the durable is gone, or its leader is moving) before the durable is + // looked up. + noAnswersBeforeCheck = 2 + // goneCheckTimeout bounds that lookup. + goneCheckTimeout = 5 * time.Second +) + +// CreateConsumer finds the operator's shard durables — it never creates one +// — and checks each against cfg: its ack_wait must cover cfg.AckWait and its +// max_ack_pending must be set, else ErrConsumerMismatch. cfg.Units picks the +// units (nil: every one, the extras still draining included). A durable name +// that does not map to the operator's, or a configured unit whose durable or +// stream is gone, is ErrConsumerNotFound; an extra whose durable is gone by +// now is skipped. +func (e *ExternalNATS) CreateConsumer(ctx context.Context, cfg ConsumerConfig) (Consumer, error) { + if _, ok := e.durable(cfg.Durable); !ok { + return nil, fmt.Errorf("consumer %q: %w: the ingest durables are %s-", cfg.Durable, ErrConsumerNotFound, e.topo.IngestConsumer) + } + names := cfg.Units + if names == nil { + configured, extra := e.IngestUnits() + names = slices.Concat(configured, extra) + } + c := &externalConsumer{e: e, ctx: ctx, whole: cfg.Units == nil, maxHeld: cfg.MaxHeld, failed: make(chan error, 1)} + for _, name := range names { + u, extra, ok := e.unit(name) + if !ok { + return nil, fmt.Errorf("consumer unit %q: %w", name, ErrUnitsUnsupported) + } + // A handle of its own: the client keeps the pin id per handle. + h, err := e.js.Consumer(ctx, u.stream, u.durable) + gone := errors.Is(err, jetstream.ErrConsumerNotFound) || errors.Is(err, jetstream.ErrStreamNotFound) + switch { + case extra && (gone || errors.Is(err, jetstream.ErrNotPullConsumer)): + continue // drained and deleted since it was listed + case gone: + return nil, fmt.Errorf("consumer %s: %w: %w", name, ErrConsumerNotFound, err) + case errors.Is(err, jetstream.ErrNotPullConsumer): + return nil, fmt.Errorf("consumer %s: %w: %w", name, ErrConsumerMismatch, err) + case err != nil: + return nil, fmt.Errorf("consumer %s: %w", name, e.apiError(err)) + } + have := h.CachedInfo().Config + if have.AckWait < cfg.AckWait { + return nil, fmt.Errorf("consumer %s: %w: ack_wait %s is shorter than the %s asked for", name, ErrConsumerMismatch, have.AckWait, cfg.AckWait) + } + if have.MaxAckPending <= 0 { + return nil, fmt.Errorf("consumer %s: %w: max_ack_pending must be set", name, ErrConsumerMismatch) + } + if extra { + slog.Info("mq: draining a shard durable outside the configured ones", "component", "nats", "unit", name, "pending", h.CachedInfo().NumPending) + } + c.parts = append(c.parts, &consumerPart{unit: u, extra: extra, h: h, ackWait: have.AckWait}) + } + return c, nil +} + +// externalConsumer is the operator's shard durables it was created for. +// +// Each unit has two goroutines. A puller fetches from the server and never +// runs the handler, so it keeps pulling, and with that keeps the unit's pin, +// however long the handler takes. A deliverer hands what was fetched to the +// handler, in order. The puller fetches only as much as the unit's cap +// (MaxHeld, and prefetch for what waits for the handler) leaves room for; at +// the cap it renews the pin every renewEvery with a one-row fetch, up to one +// ack_wait's worth of renewals past the cap, and past that, and once halted, +// with pulls that deliver nothing (max_bytes 1: the server holds any row +// back). +type externalConsumer struct { + e *ExternalNATS + ctx context.Context + // whole is a consumer of every unit, the one a process with no claims + // runs: it takes orphaned units over itself, and releases them on stop. + whole bool + maxHeld func() int + parts []*consumerPart + failed chan error + // reported and stopped keep failed to one error, none after stop. + reported, stopped atomic.Bool +} + +type consumerPart struct { + unit natsUnit + // extra is outside the configured units: its delivery ending is the + // operator deleting it once drained, not a failure. + extra bool + h jetstream.Consumer + // ackWait is the durable's: a row not settled by then is the server's to + // redeliver, so it no longer counts against the cap. + ackWait time.Duration + // pin is the pin id of the last row this process received from the + // unit, nil before the first or once the server said it lost the pin. + pin atomic.Pointer[string] + + share int + maxHeld func() int + + mu sync.Mutex + // queue is fetched and not yet handed to the handler; held is fetched + // and not yet settled, queue included. + queue []jetstream.Msg + held int + + // wake tells the puller a row settled; ready tells the deliverer rows + // are queued. + wake, ready chan struct{} + // halting stops fetching; end stops renewing the pin. fetched closes + // once nothing more will be queued, drained once the deliverer has + // handed everything on. + halting, end, fetched, drained chan struct{} + haltOnce, endOnce sync.Once + // lastPull is when the puller last sent the server a pull (its + // goroutine's only). + lastPull time.Time +} + +func (p *consumerPart) start(share int, maxHeld func() int) { + p.share, p.maxHeld = share, maxHeld + p.wake, p.ready = make(chan struct{}, 1), make(chan struct{}, 1) + p.halting, p.end = make(chan struct{}), make(chan struct{}) + p.fetched, p.drained = make(chan struct{}), make(chan struct{}) +} + +func signal(ch chan struct{}) { + select { + case ch <- struct{}{}: + default: + } +} + +func (p *consumerPart) halt() { p.haltOnce.Do(func() { close(p.halting) }) } +func (p *consumerPart) endRenewals() { p.endOnce.Do(func() { close(p.end) }) } + +func (p *consumerPart) halted() bool { + select { + case <-p.halting: + return true + default: + return false + } +} + +// room is how many rows the unit may fetch now. +func (p *consumerPart) room() int { + p.mu.Lock() + defer p.mu.Unlock() + n := p.share - len(p.queue) + if p.maxHeld != nil { + n = min(n, p.maxHeld()-p.held) + } + return n +} + +// mayOvershoot reports whether a unit at its cap may renew its pin by taking +// one more row: while the rows so taken, each counted until it settles or its +// ack_wait passes, stay under one ack_wait of renewals past the cap, both +// MaxHeld and the prefetch share of rows waiting for the handler. Past +// that it renews without taking a row, which can reorder due redeliveries +// (see renewRequest), rather than hold more. +func (p *consumerPart) mayOvershoot() bool { + p.mu.Lock() + defer p.mu.Unlock() + if p.maxHeld == nil { + return false + } + k := max(1, int(p.ackWait/renewEvery)) + return p.held-p.maxHeld() < k && len(p.queue)-p.share < k +} + +func (p *consumerPart) enqueue(m jetstream.Msg) { + if pin := m.Headers().Get(pinIDHeader); pin != "" { + p.pin.Store(&pin) + } + p.mu.Lock() + p.queue = append(p.queue, m) + p.held++ + p.mu.Unlock() + signal(p.ready) +} + +func (p *consumerPart) settle() { + p.mu.Lock() + p.held-- + p.mu.Unlock() + signal(p.wake) +} + +// next is the next row for the handler, false once fetching has ended and +// every row was handed on. +func (p *consumerPart) next() (jetstream.Msg, bool) { + for { + p.mu.Lock() + if len(p.queue) > 0 { + m := p.queue[0] + p.queue[0] = nil + p.queue = p.queue[1:] + p.mu.Unlock() + return m, true + } + p.mu.Unlock() + select { + case <-p.ready: + case <-p.fetched: + p.mu.Lock() + empty := len(p.queue) == 0 + p.mu.Unlock() + if empty { + return nil, false + } + } + } +} + +// Consume pulls every unit in the priority group, so the server delivers a +// unit to one pinned puller at a time. Prefetch is split between the +// configured units (at least one each; 0 is the client's default per unit), +// and an extra gets a quarter share. A unit's delivery that ends on its own +// — the durable or its stream deleted, the connection closed for good — is +// reported on failed; an extra's is only logged. stop halts every unit +// without waiting (Halt waits); a consumer of every unit then releases each +// once it has handed on what it fetched, while one of chosen units keeps +// renewing their pins until Release, or until a durable's ack_wait and one +// renewal have passed since its halt handed on what it fetched. +func (c *externalConsumer) Consume(handler func(msg *Message), prefetch int) (func(), <-chan error, error) { + own := 0 + for _, part := range c.parts { + if !part.extra { + own++ + } + } + if c.whole { + c.takeOver() + } + for _, part := range c.parts { + share := jetstream.DefaultMaxMessages + if prefetch > 0 { + share = max(1, prefetch/max(1, own)) + } + if part.extra { + share = max(1, share/4) + } + part.start(share, c.maxHeld) + go c.run(part) + go c.deliver(part, handler) + } + untrack := c.e.track(c.shutdown) + return func() { + untrack() + c.stopped.Store(true) + for _, part := range c.parts { + part.halt() + if c.whole { + go func() { + <-part.drained + c.releaseAfterDrain(part) + }() + } + } + }, c.failed, nil +} + +// Halt implements Halter: every unit stops fetching, and Halt returns once +// what was fetched has reached the handler, which may take one fetch's +// expiry. The units keep their pins. +func (c *externalConsumer) Halt() { + c.stopped.Store(true) + for _, part := range c.parts { + part.halt() + } + for _, part := range c.parts { + <-part.drained + } +} + +// shutdown is Close's stop: it halts and ends the renewals without waiting. +func (c *externalConsumer) shutdown() { + c.stopped.Store(true) + for _, part := range c.parts { + part.halt() + part.endRenewals() + } +} + +func (c *externalConsumer) run(part *consumerPart) { + ended := c.pull(part) + close(part.fetched) + if !ended { + c.hold(part) + } +} + +// pull fetches part until it is halted, reporting whether its delivery ended +// for good. +func (c *externalConsumer) pull(part *consumerPart) bool { + noAnswers := 0 + for !part.halted() { + n := part.room() + if n <= 0 { + if !c.renewalDue(part) { + continue + } + if !part.mayOvershoot() { + if c.renew(part, *part.pin.Load()) { + return true + } + continue + } + // Renewed by taking one row past the cap: a pull that held a + // due redelivery back would have the server requeue it behind + // the others, reordering them. + n = 1 + } + part.lastPull = time.Now() + batch, err := part.h.Fetch(n, jetstream.FetchMaxWait(fetchExpiry), jetstream.FetchPriorityGroup(natsPriorityGroup)) + if err == nil { + for m := range batch.Messages() { + part.enqueue(m) + } + err = batch.Error() + } + switch { + case err == nil || errors.Is(err, nats.ErrTimeout): + noAnswers = 0 + case errors.Is(err, jetstream.ErrPinIDMismatch) || errors.Is(err, jetstream.ErrConsumerLeadershipChanged): + // Another puller holds the pin, or the consumer's leader moved: + // the next pull tries again, and nothing is lost. + noAnswers = 0 + slog.Debug("mq: shard delivery paused", "component", "nats", "unit", part.unit.id(), "reason", err) + case errors.Is(err, nats.ErrNoResponders): + if noAnswers++; noAnswers >= noAnswersBeforeCheck && c.gone(part) { + return true + } + c.pause(part, fetchRetryWait) + default: + if c.endedBy(part, err) { + return true + } + level := slog.LevelWarn + if part.extra { + level = slog.LevelInfo + } + slog.Log(context.Background(), level, "mq: consumer reported an error", "component", "nats", "unit", part.unit.id(), "error", err) + c.pause(part, fetchRetryWait) + } + } + return false +} + +// renewalDue, for a unit at its cap, reports whether its pin is due a +// renewal, and otherwise waits for a row to settle, a halt, or the renewal. +func (c *externalConsumer) renewalDue(part *consumerPart) bool { + if part.pin.Load() == nil { + c.pause(part, 0) // never received: no pin to keep + return false + } + if due := time.Until(part.lastPull.Add(renewEvery)); due > 0 { + c.pause(part, due) + return false + } + return true +} + +// pause waits up to d (0: no bound) for a row to settle, a halt, or the +// end of the renewals. +func (c *externalConsumer) pause(part *consumerPart, d time.Duration) { + var timeout <-chan time.Time + if d > 0 { + t := time.NewTimer(d) + defer t.Stop() + timeout = t.C + } + select { + case <-part.wake: + case <-part.halting: + case <-part.end: + case <-timeout: + } +} + +// hold keeps a halted unit's pin, so that no other process receives from it +// while this one writes what it holds, until Release, Close, the pin is lost, +// or the durable's ack_wait has passed since what it fetched reached the +// handler, and one renewal more, so that a caller's own wait of ack_wait from +// then (a handover's) ends before the renewals do. By then the server +// redelivers what this process still holds anyway. +func (c *externalConsumer) hold(part *consumerPart) { + var until time.Time + for { + if until.IsZero() { + select { + case <-part.drained: + until = time.Now().Add(part.ackWait + renewEvery) + default: + } + } + pin := part.pin.Load() + if pin == nil || (!until.IsZero() && !time.Now().Before(until)) { + return + } + t := time.NewTimer(max(0, time.Until(part.lastPull.Add(renewEvery)))) + select { + case <-part.end: + t.Stop() + return + case <-t.C: + } + if c.renew(part, *pin) { + return + } + } +} + +// renewRequest is a pull the server answers with no row: max_bytes 1 is +// below any row's size, so it holds the row back (409) and keeps it where it +// is. It still resets the pin's timer, which the server does on any pull that +// carries the current pin id. +type renewRequest struct { + Expires time.Duration `json:"expires"` + Batch int `json:"batch"` + MaxBytes int `json:"max_bytes"` + Group string `json:"group"` + ID string `json:"id"` +} + +// renew sends part's pin-keeping pull, reporting whether the unit's delivery +// ended for good (its durable deleted). +func (c *externalConsumer) renew(part *consumerPart, pin string) bool { + part.lastPull = time.Now() + body, err := json.Marshal(renewRequest{Expires: renewWait, Batch: 1, MaxBytes: 1, Group: natsPriorityGroup, ID: pin}) + if err != nil { + return false + } + ctx, cancel := context.WithTimeout(context.Background(), renewWait+time.Second) + defer cancel() + subj := c.e.apiPrefix + "CONSUMER.MSG.NEXT." + part.unit.stream + "." + part.unit.durable + resp, err := c.e.nc.RequestWithContext(ctx, subj, body) + switch { + case errors.Is(err, nats.ErrNoResponders): + return c.gone(part) + case err != nil: + if c.endedBy(part, err) { + return true + } + slog.Debug("mq: a shard's pin renewal got no answer", "component", "nats", "unit", part.unit.id(), "error", err) + return false + } + status, descr := resp.Header.Get("Status"), strings.ToLower(resp.Header.Get("Description")) + switch { + case status == "409" && strings.Contains(descr, "consumer deleted"): + c.end(part, jetstream.ErrConsumerDeleted) + return true + case status == "423": + // The pin lapsed or went to another puller: this unit no longer + // holds it, and the next fetch pins afresh if it can. + if cur := part.pin.Load(); cur != nil && *cur == pin { + part.pin.Store(nil) + } + slog.Debug("mq: a shard's pin was lost while it fetched nothing", "component", "nats", "unit", part.unit.id()) + } + // 408 (nothing waiting), 409 over max_bytes (a row held back) or a + // leadership change: the pin's timer was reset, or the next pull re-pins. + return false +} + +// gone looks a durable that answers no pull up: deleted, or its stream +// deleted, ends the unit's delivery. Anything else (its leader moving, the +// JetStream API not answering either) is tried again. +func (c *externalConsumer) gone(part *consumerPart) bool { + ctx, cancel := context.WithTimeout(context.Background(), goneCheckTimeout) + defer cancel() + _, err := part.h.Info(ctx) + if errors.Is(err, jetstream.ErrConsumerNotFound) || errors.Is(err, jetstream.ErrStreamNotFound) { + c.end(part, fmt.Errorf("%w: %w", ErrConsumerNotFound, err)) + return true + } + if err != nil && !c.stopped.Load() { + slog.Warn("mq: a shard's pulls get no answer; trying again", "component", "nats", "unit", part.unit.id(), "error", err) + } + return false +} + +// endedBy reports whether err ends part's delivery for good, and reports it +// if so: the durable deleted under a pull, or the connection closed for good. +func (c *externalConsumer) endedBy(part *consumerPart, err error) bool { + switch { + case errors.Is(err, jetstream.ErrConsumerDeleted) || errors.Is(err, jetstream.ErrBadRequest): + c.end(part, err) + return true + case errors.Is(err, nats.ErrConnectionClosed) || errors.Is(err, nats.ErrConnectionDraining) || c.e.nc.IsClosed(): + if !c.e.closing.Load() { + c.end(part, nats.ErrConnectionClosed) + } + return true + } + return false +} + +// end reports part's delivery ending on its own: a configured unit's on +// failed, an extra's only in the log. +func (c *externalConsumer) end(part *consumerPart, reason error) { + err := fmt.Errorf("%w: %w", ErrDeliveryEnded, reason) + if part.extra { + slog.Info("mq: stopped draining a shard durable outside the configured ones", "component", "nats", "unit", part.unit.id(), "reason", err) + return + } + c.fail(fmt.Errorf("%s: %w", part.unit.id(), err)) +} + +// deliver hands part's fetched rows to handler, in order, each counted +// against the unit's cap until it settles or its ack_wait passes. +func (c *externalConsumer) deliver(part *consumerPart, handler func(*Message)) { + defer close(part.drained) + for { + m, ok := part.next() + if !ok { + return + } + msg := c.e.wrapMsg(c.ctx, m, true) + letGo := sync.OnceFunc(part.settle) + expire := time.AfterFunc(part.ackWait, letGo) + msg.OnSettled(func() { expire.Stop(); letGo() }) + handler(msg) + } +} + +// takeOver resets every unit no one holds that has rows awaiting an ack, +// before this consumer pulls it: whoever received them is gone. +func (c *externalConsumer) takeOver() { + ctx, cancel := context.WithTimeout(c.ctx, recheckTimeout) + defer cancel() + for _, part := range c.parts { + if reset, err := c.e.ResetOrphaned(ctx, part.unit.id()); errors.Is(err, ErrUnitHeld) { + continue // another consumer holds it; the pin decides + } else if err != nil { + slog.Warn("mq: could not take over a shard's unsettled rows; they come back after ack_wait", "component", "nats", "unit", part.unit.id(), "error", err) + } else if reset { + slog.Info("mq: took over a shard's unsettled rows", "component", "nats", "unit", part.unit.id()) + } + } +} + +// releaseAfterDrain releases part once its halt has delivered what it +// fetched. A whole consumer's handler has no settlement to wait for here, +// and an unsettled row comes back to the next owner after ack_wait. +func (c *externalConsumer) releaseAfterDrain(part *consumerPart) { + defer part.endRenewals() + if c.e.nc.IsClosed() { + return + } + ctx, cancel := context.WithTimeout(context.Background(), releaseTimeout) + defer cancel() + if err := c.e.release(ctx, part); err != nil { + slog.Debug("mq: could not release a shard; its pin lapses on its own", "component", "nats", "unit", part.unit.id(), "error", err) + } +} + +// Release implements Releaser: each unit this consumer was last pinned on, +// and whose pin the server still gives it, is unpinned; then the unit stops +// renewing its pin. +func (c *externalConsumer) Release(ctx context.Context) error { + var errs []error + for _, part := range c.parts { + errs = append(errs, c.e.release(ctx, part)) + part.endRenewals() + } + return errors.Join(errs...) +} + +// release unpins part if this process still holds its pin. Checking first +// keeps a process that never received from the unit, or lost the pin since, +// from unpinning the real owner. +// +// The check and the unpin are two requests, and the unpin names no pin id: +// if the pin moved to another client between them, that client is unpinned +// and pins again with its next pull. A halted unit renews its pin until its +// release returns, if that comes within ack_wait of the halt's drain (a +// handover's bound), so the pin cannot lapse in between; only a consumer +// leader change there (which releases the pin itself) opens that window. +func (e *ExternalNATS) release(ctx context.Context, part *consumerPart) error { + pin := part.pin.Load() + if pin == nil { + return nil + } + info, err := part.h.Info(ctx) + if err != nil { + return fmt.Errorf("consumer %s: %w", part.unit.id(), e.apiError(err)) + } + if pinnedClient(info) != *pin { + return nil + } + s, err := e.js.Stream(ctx, part.unit.stream) + if err != nil { + return fmt.Errorf("stream %s: %w", part.unit.stream, e.apiError(err)) + } + if err := s.UnpinConsumer(ctx, part.unit.durable, natsPriorityGroup); err != nil { + return fmt.Errorf("unpin %s: %w", part.unit.id(), e.apiError(err)) + } + return nil +} + +func (c *externalConsumer) fail(err error) { + if c.stopped.Load() || !c.reported.CompareAndSwap(false, true) { + return + } + c.failed <- err +} diff --git a/internal/mq/external_test.go b/internal/mq/external_test.go index b61380af4..cd9b71032 100644 --- a/internal/mq/external_test.go +++ b/internal/mq/external_test.go @@ -426,6 +426,7 @@ func TestExternalNATS_CreateConsumerChecksTheDurable(t *testing.T) { _, err := e.CreateConsumer(t.Context(), ConsumerConfig{Durable: workerDurable, AckWait: time.Hour}) require.ErrorContains(t, err, "ack_wait") + require.ErrorIs(t, err, ErrConsumerMismatch) _, err = e.CreateConsumer(t.Context(), ConsumerConfig{Durable: "someone-else"}) require.ErrorIs(t, err, ErrConsumerNotFound) _, err = e.CreateConsumer(t.Context(), ConsumerConfig{Durable: DefaultNATSIngestConsumer, AckWait: time.Minute}) @@ -436,6 +437,10 @@ func TestExternalNATS_CreateConsumerChecksTheDurable(t *testing.T) { require.ErrorIs(t, err, ErrConsumerNotFound) _, err = f.admin.Consumer(t.Context(), shippedPartition(0), "wh-ingest-3") require.ErrorIs(t, err, jetstream.ErrConsumerNotFound, "the broker must not recreate the durable") + + require.NoError(t, f.admin.DeleteStream(t.Context(), shippedPartition(1))) + _, err = e.CreateConsumer(t.Context(), ConsumerConfig{Durable: workerDurable, Units: []string{shippedPartition(1) + "/wh-ingest-0"}}) + require.ErrorIs(t, err, ErrConsumerNotFound, "a durable whose stream is gone is gone") } // The shipped permissions refuse the wavehouse user everything that would @@ -845,13 +850,21 @@ func TestExternalNATS_ReleaseUnpinsOnlyItsOwnPin(t *testing.T) { assert.Less(t, time.Since(released), 5*time.Second, "at once, not after the pinned ttl") } +// quietAfter shortens how recently a unit must have been active to count +// as someone's, the durable's pinned_ttl otherwise. +func quietAfter(d time.Duration) func(*NATSConfig) { + return func(cfg *NATSConfig) { cfg.activeWithin = d } +} + // A unit whose holder is gone with rows unacked: ResetOrphaned leaves it -// alone while a client still holds the pin, and once none does, redelivers -// the held rows to the next owner at once, not after ack_wait. +// alone while a client still holds the pin, or was active on it within the +// window, and once neither, redelivers the held rows to the next owner at +// once, not after ack_wait. func TestExternalNATS_ResetOrphaned(t *testing.T) { t.Parallel() f := shippedFixture(t) - a, b := f.broker(t, nil), f.broker(t, nil) + const quiet = time.Second + a, b := f.broker(t, nil), f.broker(t, quietAfter(quiet)) topic := Topic{Tenant: "acme", Table: "events"} p, s := natsRoute(topic, 4, 8) unit := shippedPartition(p) + "/" + natsShardDurable("wh-ingest", s) @@ -871,6 +884,10 @@ func TestExternalNATS_ResetOrphaned(t *testing.T) { require.NoError(t, err) require.NoError(t, st.UnpinConsumer(t.Context(), natsShardDurable("wh-ingest", s), natsPriorityGroup), "stands in for the pin lapsing") reset, err = b.ResetOrphaned(t.Context(), unit) + require.ErrorIs(t, err, ErrUnitHeld, "no pin alone is no proof: it delivered just now") + assert.False(t, reset) + time.Sleep(quiet) + reset, err = b.ResetOrphaned(t.Context(), unit) require.NoError(t, err) require.True(t, reset) _, _, gotB := unitConsumer(t, b, unit, nil) @@ -927,8 +944,10 @@ func TestExternalNATS_WholeConsumerTakesOrphansOver(t *testing.T) { st, err := f.admin.Stream(t.Context(), shippedPartition(p)) require.NoError(t, err) require.NoError(t, st.UnpinConsumer(t.Context(), natsShardDurable("wh-ingest", s), natsPriorityGroup), "stands in for the pin lapsing") + const quiet = time.Second + time.Sleep(quiet) // and for its owner's silence since - b := f.broker(t, nil) + b := f.broker(t, quietAfter(quiet)) c, err := b.CreateConsumer(t.Context(), ConsumerConfig{Durable: workerDurable}) require.NoError(t, err) got := make(chan string, 4) @@ -991,3 +1010,270 @@ func TestExternalNATS_Unowned(t *testing.T) { require.NoError(t, err) assert.Zero(t, n, "held by the consumer that received it") } + +// One unit's consume throughput: a backlog of rows on one shard, drained by +// one consumer that acks each row as it arrives. +func TestExternalNATS_MeasureUnitConsumeThroughput(t *testing.T) { + measure(t) + f := shippedFixture(t) + e := f.broker(t, nil) + topic := Topic{Tenant: "acme", Table: "t"} + payload := make([]byte, 256) + const n = 20_000 + var wg sync.WaitGroup + for w := range 32 { + wg.Go(func() { + for i := w; i < n; i += 32 { + assert.NoError(t, e.Publish(t.Context(), topic, payload)) + } + }) + } + wg.Wait() + p, s := natsRoute(topic, 4, 8) + c, err := e.CreateConsumer(t.Context(), ConsumerConfig{Durable: workerDurable, Units: []string{shippedPartition(p) + "/" + natsShardDurable("wh-ingest", s)}}) + require.NoError(t, err) + var got atomic.Int64 + done := make(chan struct{}) + start := time.Now() + stop, _, err := c.Consume(func(m *Message) { + _ = m.Ack() + if got.Add(1) == n { + close(done) + } + }, 500) + require.NoError(t, err) + t.Cleanup(stop) + select { + case <-done: + case <-time.After(2 * time.Minute): + t.Fatalf("received %d of %d", got.Load(), n) + } + elapsed := time.Since(start) + t.Logf("one unit drained %d rows of %d bytes in %s: %.0f rows/s", n, len(payload), elapsed.Round(time.Millisecond), float64(n)/elapsed.Seconds()) +} + +// A shard durable deleted while its consumer's handler is busy and no pull +// of it is waiting ends delivery with failed, not a silent stall: the next +// pin-keeping pull gets no responders, and the durable is looked up. +func TestExternalNATS_DurableDeletedWhileBusyEndsDelivery(t *testing.T) { + t.Parallel() + f := shippedFixture(t) + e := f.broker(t, nil) + topic := Topic{Tenant: "acme", Table: "events"} + p, s := natsRoute(topic, 4, 8) + stream, durable := shippedPartition(p), natsShardDurable("wh-ingest", s) + c, err := e.CreateConsumer(t.Context(), ConsumerConfig{ + Durable: workerDurable, Units: []string{stream + "/" + durable}, + MaxHeld: func() int { return 2 }, + }) + require.NoError(t, err) + busy := make(chan struct{}) + t.Cleanup(func() { close(busy) }) + entered := make(chan struct{}, 8) + stop, failed, err := c.Consume(func(*Message) { entered <- struct{}{}; <-busy }, 16) + require.NoError(t, err) + t.Cleanup(stop) + for i := range 5 { + require.NoError(t, e.Publish(t.Context(), topic, []byte(strconv.Itoa(i)))) + } + <-entered // the handler holds the first row; the unit is at its cap + + require.Eventually(t, func() bool { + cons, err := f.admin.Consumer(t.Context(), stream, durable) + return err == nil && cons.CachedInfo().NumAckPending == 2 && cons.CachedInfo().NumWaiting == 0 + }, 10*time.Second, 5*time.Millisecond, "a moment with no pull waiting") + require.NoError(t, f.admin.DeleteConsumer(t.Context(), stream, durable)) + deleted := time.Now() + select { + case err := <-failed: + assert.ErrorIs(t, err, ErrDeliveryEnded) + assert.ErrorIs(t, err, ErrConsumerNotFound) + t.Logf("failed %s after the delete: %v", time.Since(deleted).Round(10*time.Millisecond), err) + case <-time.After(20 * time.Second): + t.Fatal("a durable deleted under a busy handler stalled silently") + } +} + +// Rows NAKed with a delay that come due while their unit is at its cap come +// back in order. A pin renewal that held a due row back would have the server +// requeue it behind the others (measured with max_bytes-1 renewals: 2 3 0 1), +// so at the cap the renewal takes the row instead. +func TestExternalNATS_RedeliveryOrderKeptAtTheCap(t *testing.T) { + t.Parallel() + f := shippedFixture(t) + e := f.broker(t, nil) + topic := Topic{Tenant: "acme", Table: "events"} + p, s := natsRoute(topic, 4, 8) + c, err := e.CreateConsumer(t.Context(), ConsumerConfig{ + Durable: workerDurable, + Units: []string{shippedPartition(p) + "/" + natsShardDurable("wh-ingest", s)}, MaxHeld: func() int { return 4 }, + }) + require.NoError(t, err) + var mu sync.Mutex + var got []string + var held []*Message + stop, _, err := c.Consume(func(m *Message) { + mu.Lock() + got = append(got, string(m.Data)) + held = append(held, m) + mu.Unlock() + }, 16) + require.NoError(t, err) + t.Cleanup(stop) + for i := range 8 { + require.NoError(t, e.Publish(t.Context(), topic, []byte(strconv.Itoa(i)))) + } + take := func(n int) []*Message { + require.Eventually(t, func() bool { mu.Lock(); defer mu.Unlock(); return len(held) >= n }, 20*time.Second, 10*time.Millisecond) + mu.Lock() + defer mu.Unlock() + out := held[:n] + held = held[n:] + return out + } + for _, m := range take(4) { // 0..3 come due in 3s, while 4..7 fill the cap + require.NoError(t, m.NakWithDelay(3*time.Second)) + } + later := take(4) + time.Sleep(14 * time.Second) // two renewals at least + for _, m := range later { + require.NoError(t, m.Ack()) + } + redelivered := take(4) + var order []string + for _, m := range redelivered { + order = append(order, string(m.Data)) + _ = m.Ack() + } + mu.Lock() + first := slices.Clone(got[:8]) + mu.Unlock() + assert.Equal(t, []string{"0", "1", "2", "3", "4", "5", "6", "7"}, first) + assert.Equal(t, []string{"0", "1", "2", "3"}, order, "redelivered in order across the renewals") +} + +// A unit whose handler is blocked, at its cap, keeps its pin, and takes no +// more rows to renew it than one ack_wait's worth of renewals (shortened to +// 10s, so 2 here); past that it renews without taking any. +func TestExternalNATS_BlockedHandlerTakesABoundedOvershoot(t *testing.T) { + t.Parallel() + f := shippedFixture(t) + e := f.broker(t, nil) + topic := Topic{Tenant: "acme", Table: "events"} + p, s := natsRoute(topic, 4, 8) + stream, durable := shippedPartition(p), natsShardDurable("wh-ingest", s) + c, err := e.CreateConsumer(t.Context(), ConsumerConfig{ + Durable: workerDurable, Units: []string{stream + "/" + durable}, + MaxHeld: func() int { return 2 }, + }) + require.NoError(t, err) + c.(*externalConsumer).parts[0].ackWait = 2 * renewEvery + busy := make(chan struct{}) + entered := make(chan struct{}, 16) + stop, _, err := c.Consume(func(*Message) { entered <- struct{}{}; <-busy }, 16) + require.NoError(t, err) + t.Cleanup(stop) + t.Cleanup(func() { close(busy) }) // first, so stop's cleanup is not left waiting + for i := range 10 { + require.NoError(t, e.Publish(t.Context(), topic, []byte(strconv.Itoa(i)))) + } + <-entered + state := func() (string, uint64) { + cons, err := f.admin.Consumer(t.Context(), stream, durable) + require.NoError(t, err) + return pinnedClient(cons.CachedInfo()), cons.CachedInfo().Delivered.Consumer + } + require.Eventually(t, func() bool { _, d := state(); return d >= 2 }, 5*time.Second, 10*time.Millisecond) + pin, _ := state() + require.NotEmpty(t, pin) + // The row in the handler stops counting at its ack_wait (10s here), as + // the server would redeliver it then, so one more is taken; after that + // nothing, however many renewals follow. + time.Sleep(18 * time.Second) // the last row-taking renewal is at about 15s + _, settled := state() + time.Sleep(11 * time.Second) // two more renewals + now, delivered := state() + assert.Equal(t, pin, now, "the pin was kept") + assert.Equal(t, settled, delivered, "renewals past the bound take no rows") + assert.Equal(t, uint64(5), delivered, "the cap of 2, one ack_wait (10s) of renewals at 5s, and the expired row's slot") +} + +// A halted unit keeps its pin for its ack_wait counted from when what it +// fetched has reached the handler, not from the halt: a slow drain does not +// eat the time a handover then waits for those rows to settle. +func TestExternalNATS_HaltKeepsThePinPastASlowDrain(t *testing.T) { + t.Parallel() + f := shippedFixture(t) + e := f.broker(t, nil) + topic := Topic{Tenant: "acme", Table: "events"} + p, s := natsRoute(topic, 4, 8) + c, err := e.CreateConsumer(t.Context(), ConsumerConfig{ + Durable: workerDurable, + Units: []string{shippedPartition(p) + "/" + natsShardDurable("wh-ingest", s)}, + }) + require.NoError(t, err) + c.(*externalConsumer).parts[0].ackWait = 2 * renewEvery // 10s + release := make(chan struct{}) + var once sync.Once + stop, _, err := c.Consume(func(*Message) { once.Do(func() { <-release }) }, 16) + require.NoError(t, err) + t.Cleanup(stop) + t.Cleanup(func() { closeOnceMQ(release) }) + for i := range 3 { + require.NoError(t, e.Publish(t.Context(), topic, []byte(strconv.Itoa(i)))) + } + require.Eventually(t, func() bool { return unitPin(t, f, topic) != "" }, 5*time.Second, 10*time.Millisecond) + pin := unitPin(t, f, topic) + halted := make(chan struct{}) + go func() { c.(Halter).Halt(); close(halted) }() + time.Sleep(15 * time.Second) // the drain outlasts the ack_wait + closeOnceMQ(release) + <-halted + time.Sleep(10 * time.Second) // within ack_wait of the drain, past it from the halt + assert.Equal(t, pin, unitPin(t, f, topic), "the pin is kept for ack_wait after the drain") +} + +func closeOnceMQ(ch chan struct{}) { + select { + case <-ch: + default: + close(ch) + } +} + +// The same bound holds where the prefetch share, not MaxHeld, is the cap: a +// blocked unit's renewal rows stop one ack_wait of renewals past its share. +func TestExternalNATS_BlockedHandlerOvershootBoundedByTheShare(t *testing.T) { + t.Parallel() + f := shippedFixture(t) + e := f.broker(t, nil) + topic := Topic{Tenant: "acme", Table: "events"} + p, s := natsRoute(topic, 4, 8) + stream, durable := shippedPartition(p), natsShardDurable("wh-ingest", s) + c, err := e.CreateConsumer(t.Context(), ConsumerConfig{ + Durable: workerDurable, Units: []string{stream + "/" + durable}, + MaxHeld: func() int { return 1000 }, + }) + require.NoError(t, err) + c.(*externalConsumer).parts[0].ackWait = 2 * renewEvery + busy := make(chan struct{}) + stop, _, err := c.Consume(func(*Message) { <-busy }, 2) + require.NoError(t, err) + t.Cleanup(stop) + t.Cleanup(func() { close(busy) }) + for i := range 12 { + require.NoError(t, e.Publish(t.Context(), topic, []byte(strconv.Itoa(i)))) + } + delivered := func() uint64 { + cons, err := f.admin.Consumer(t.Context(), stream, durable) + require.NoError(t, err) + return cons.CachedInfo().Delivered.Consumer + } + time.Sleep(16 * time.Second) + settled := delivered() + time.Sleep(11 * time.Second) // two more renewals + assert.Equal(t, settled, delivered(), "renewals past the bound take no rows") + // One in the handler, a share of 2 queued, and 2 renewals' worth past it; + // the handled row's expiry frees no room here, the queue being the cap. + assert.Equal(t, uint64(5), settled) +} diff --git a/internal/mq/mq.go b/internal/mq/mq.go index e27229478..cff6d042f 100644 --- a/internal/mq/mq.go +++ b/internal/mq/mq.go @@ -256,6 +256,13 @@ type ConsumerConfig struct { // pauses when its unacked messages hit it (backpressure), and no other // unit's does. MaxAckPending int + // MaxHeld, when set, caps the messages each unit of a Sharded broker's + // queue has delivered to this process and not yet had settled: at the + // cap the consumer fetches no more from that unit than it takes to keep + // its hold on the unit, until one settles. It is read before every fetch, so + // the cap may change while consuming. Nil is no cap but the broker's own. + // A broker that does not implement Sharded ignores it. + MaxHeld func() int } // Sharded is implemented by a broker whose ingest queue is split into units @@ -268,14 +275,18 @@ type Sharded interface { // holding rows, as last read, which are drained like the others. IngestUnits() (configured, extra []string) // ResetOrphaned redelivers at once what a unit's previous owner received - // and never settled, when no consumer holds the unit and rows are - // awaiting a settlement, and reports whether it did; ErrUnitHeld while a - // consumer still holds it. Call it before consuming a unit taken over - // from an owner that died, never while consuming it: a reset also - // redelivers what the caller itself holds. + // and never settled, when no consumer holds the unit, none has received + // from it or settled a message of it for a while, and rows are awaiting + // a settlement; it reports whether it did, and ErrUnitHeld while a + // consumer holds the unit or was recently active on it. A missing hold + // alone is no proof the owner is gone. Call it only for a unit whose + // owner the caller has other evidence is dead (the membership view), and + // before consuming it, never while consuming it: a reset also redelivers + // what the caller itself holds. ResetOrphaned(ctx context.Context, unit string) (bool, error) - // Unowned counts the units with rows waiting that no consumer holds: - // rows nobody is writing. + // Unowned counts the units with rows waiting that no consumer holds and + // none has received from or settled a message of for a while: rows + // nobody is writing. Unowned(ctx context.Context) (int, error) } @@ -288,9 +299,27 @@ type Releaser interface { Release(ctx context.Context) error } -// ErrUnitHeld is ResetOrphaned's answer while a consumer still holds the unit. +// Halter is implemented by a Consumer that can stop fetching ahead of its +// stop, so that its caller can write what it holds before the consumer lets +// its units go. Halt returns once no handler invocation is running and none +// will: everything already fetched has reached the handler. The consumer +// keeps its hold on its units until stop, but not indefinitely: a Sharded +// broker's for at least its ack wait after the halt has handed on what it +// fetched. +type Halter interface { + Halt() +} + +// ErrUnitHeld is ResetOrphaned's answer while a consumer still holds the +// unit, or has received from it or settled a message of it too recently to +// be judged gone. var ErrUnitHeld = errors.New("unit held by a consumer") +// ErrConsumerMismatch is CreateConsumer's answer when a consumer the broker's +// operator owns no longer fits the ConsumerConfig: its ack_wait is shorter, +// or its max_ack_pending unset. +var ErrConsumerMismatch = errors.New("consumer does not match its config") + // ErrUnitsUnsupported is CreateConsumer's answer to a ConsumerConfig naming // units on a broker that does not implement Sharded, or naming one it does // not have. @@ -303,7 +332,8 @@ type Consumer interface { // it: one per unit, so a tenant's messages arrive in order, one at a // time, while different units' arrive concurrently — handler must be // safe for that. A handler that blocks holds back its unit's delivery — - // that is the backpressure the ingest worker relies on. About prefetch + // that is the backpressure the ingest worker relies on; a Sharded + // broker's consumer keeps its hold on the unit meanwhile. About prefetch // messages are fetched ahead across the units together: the units when // delivery starts split it, and a queue joined later fetches ahead its // share of it at that point, at least one message each (0 = the client diff --git a/internal/mq/nats_manifests.go b/internal/mq/nats_manifests.go index 5319380ff..33b48d077 100644 --- a/internal/mq/nats_manifests.go +++ b/internal/mq/nats_manifests.go @@ -240,7 +240,7 @@ func WriteNATSManifests(w io.Writer, o NATSManifestOptions) error { return err } if o.PinnedTTL < minPinnedTTL { - return fmt.Errorf("pinned ttl %s must be at least %s, twice the pull expiry", o.PinnedTTL, minPinnedTTL) + return fmt.Errorf("pinned ttl %s must be at least %s, twice the time between pulls", o.PinnedTTL, minPinnedTTL) } if _, err := fmt.Fprintf(w, `# WaveHouse's JetStream topology as nack (jetstream.nats.io/v1beta2) resources: # %d ingest partition(s) with work-queue retention, each split into %d shards diff --git a/internal/mq/nats_topology.go b/internal/mq/nats_topology.go index 762650b80..011003916 100644 --- a/internal/mq/nats_topology.go +++ b/internal/mq/nats_topology.go @@ -167,13 +167,18 @@ func (t NATSTopology) dedupeDuplicateWindow() time.Duration { // puller. const natsPriorityGroup = "wavehouse" -// natsPullExpiry is how long one pull request of a shard waits. The server -// renews a pin only when its holder sends a new pull, so a live idle owner -// keeps its pin only while this is well under the durable's pinned TTL. +// natsPullExpiry is the most time between two pulls of a shard that is at +// its cap or halted, which are there to keep its pin (at the cap, taking one +// row each for a while). Every pull itself waits a +// second at most; boot still requires max_expires to allow a pull this long, +// as headroom for longer pulls. The server renews a pin only when its +// holder sends a new pull, so a live owner keeps its pin only while this is +// well under the durable's pinned TTL. const natsPullExpiry = 5 * time.Second -// minPinnedTTL is the shortest pinned TTL a shard durable may have: two pull -// expiries, so a live owner re-pulls at least once before its pin lapses. +// minPinnedTTL is the shortest pinned TTL a shard durable may have: twice +// the time between pulls, so a live owner pulls at least once before its pin +// lapses. const minPinnedTTL = 2 * natsPullExpiry // FindingSeverity says whether a finding stops WaveHouse from serving. @@ -538,7 +543,7 @@ func (v *topologyVerifier) durable(ctx context.Context, s jetstream.Stream, name req("inactive_threshold", "is %s; a durable must not expire", cfg.InactiveThreshold) } if cfg.MaxRequestExpires != 0 && cfg.MaxRequestExpires < natsPullExpiry { - req("max_expires", "is %s; must be unset or at least %s, WaveHouse's pull expiry, or the server refuses every pull", cfg.MaxRequestExpires, natsPullExpiry) + req("max_expires", "is %s; must be unset or at least %s, or the server may refuse WaveHouse's pulls", cfg.MaxRequestExpires, natsPullExpiry) } if cfg.MaxRequestBatch != 0 && cfg.MaxRequestBatch < t.Prefetch { req("max_request_batch", "is %d; must be 0 or at least %d, the prefetch of a worker that owns this one shard", cfg.MaxRequestBatch, t.Prefetch) @@ -554,7 +559,7 @@ func (v *topologyVerifier) durable(ctx context.Context, s jetstream.Stream, name case cfg.PriorityPolicy != jetstream.PriorityPolicyPinned: case cfg.PinnedTTL < minPinnedTTL: // The server renews a pin only on a new pull from its holder. - req("priority_timeout", "is %s; must be at least %s, twice WaveHouse's %s pull expiry, or a live idle owner loses its pin", cfg.PinnedTTL, minPinnedTTL, natsPullExpiry) + req("priority_timeout", "is %s; must be at least %s, twice the %s between WaveHouse's pulls of a shard at its cap, or a live owner loses its pin", cfg.PinnedTTL, minPinnedTTL, natsPullExpiry) case cfg.PinnedTTL >= defaultLeaseDuration: v.add(FindingRecommended, obj, "priority_timeout", "is %s; keep it under %s, the lease after which the other processes count a dead holder gone, so its pin has lapsed by then", cfg.PinnedTTL, defaultLeaseDuration) } diff --git a/tests/integration/shard_order_test.go b/tests/integration/shard_order_test.go new file mode 100644 index 000000000..f3827bbf9 --- /dev/null +++ b/tests/integration/shard_order_test.go @@ -0,0 +1,408 @@ +//go:build integration + +package tests + +import ( + "bufio" + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "slices" + "strconv" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/Wave-RF/WaveHouse/internal/chconn" + "github.com/Wave-RF/WaveHouse/internal/ingest" + "github.com/Wave-RF/WaveHouse/internal/mq" + "github.com/Wave-RF/WaveHouse/internal/mq/natstest" + "github.com/Wave-RF/WaveHouse/internal/tenant" + "github.com/Wave-RF/WaveHouse/internal/testutil" +) + +// insertLog is a stub ClickHouse shared by every process: the rows each +// table received, in the order their inserts completed, and who sent them. +type insertLog struct { + mu sync.Mutex + rows map[string][]int + writers map[string][]string + // first and last are when each process's first and last insert of each + // table completed: proc → table → time. + first, last map[string]map[string]time.Time + total int +} + +// server is proc's ClickHouse. gate, when set, runs before an insert is +// recorded, as a slow server would take its time. +func (l *insertLog) server(t *testing.T, proc string, gate func(table string)) *httptest.Server { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + table := r.URL.Query().Get("param_target_table") + var seqs []int + sc := bufio.NewScanner(r.Body) + for sc.Scan() { + var row []int + if err := json.Unmarshal(sc.Bytes(), &row); err != nil || len(row) != 1 { + http.Error(w, "bad row", http.StatusBadRequest) + return + } + seqs = append(seqs, row[0]) + } + if gate != nil { + gate(table) + } + l.mu.Lock() + if l.rows == nil { + l.rows, l.writers = map[string][]int{}, map[string][]string{} + l.first, l.last = map[string]map[string]time.Time{}, map[string]map[string]time.Time{} + } + if l.first[proc] == nil { + l.first[proc], l.last[proc] = map[string]time.Time{}, map[string]time.Time{} + } + now := time.Now() + if _, ok := l.first[proc][table]; !ok { + l.first[proc][table] = now + } + l.last[proc][table] = now + l.rows[table] = append(l.rows[table], seqs...) + if w := l.writers[table]; len(w) == 0 || w[len(w)-1] != proc { + l.writers[table] = append(w, proc) + } + l.total += len(seqs) + l.mu.Unlock() + w.WriteHeader(http.StatusOK) + })) + t.Cleanup(srv.Close) + return srv +} + +func (l *insertLog) count() int { + l.mu.Lock() + defer l.mu.Unlock() + return l.total +} + +func (l *insertLog) table(name string) []int { + l.mu.Lock() + defer l.mu.Unlock() + return slices.Clone(l.rows[name]) +} + +// inOrder reports whether seqs is exactly 0..n-1: every row once, in order. +func inOrder(seqs []int, n int) bool { + if len(seqs) != n { + return false + } + for i, s := range seqs { + if s != i { + return false + } + } + return true +} + +func seqEnvelope(t *testing.T, table string, seq int) []byte { + t.Helper() + out, err := json.Marshal(ingest.EventMessage{ + TableName: table, Format: ingest.FormatJSONCompactEachRow, + Columns: []string{"seq"}, Row: json.RawMessage("[" + strconv.Itoa(seq) + "]"), + }) + require.NoError(t, err) + return out +} + +// workerProc is one ingest process: the real worker over shard claims, as +// the restricted wavehouse user, writing to its own stub ClickHouse. +type workerProc struct { + id string + broker *mq.ExternalNATS + stop func(context.Context) error + failed <-chan error +} + +func startWorkerProc(t *testing.T, url, id string, lease time.Duration, cfg ingest.ClaimConfig, chURL string) *workerProc { + t.Helper() + ctx := t.Context() + p := &workerProc{id: id, broker: shardBroker(t, url)} + leases, err := p.broker.Leases(ctx, natstest.CoordBucket, id, mq.WithLeaseTimings(lease, lease*7/10, 100*time.Millisecond)) + require.NoError(t, err) + q, err := ingest.ClaimShards(p.broker, leases, cfg) + require.NoError(t, err) + p.stop, p.failed, err = ingest.StartIngestWorker(context.WithoutCancel(ctx), q, &testutil.MockCache{}, + func(tenant.ID) chconn.Target { return chconn.Target{URL: chURL, Database: "db"} }, nil) + require.NoError(t, err) + t.Cleanup(func() { _ = p.stop(context.Background()) }) + return p +} + +func noWorkerFailure(t *testing.T, procs ...*workerProc) { + t.Helper() + for _, p := range procs { + select { + case err := <-p.failed: + t.Fatalf("%s failed: %v", p.id, err) + default: + } + } +} + +// unitOf publishes data to topic before anything consumes it, and returns +// the unit whose durable the row waits on. +func unitOf(t *testing.T, srv *natstest.Server, b mq.Broker, topic mq.Topic, data []byte) string { + t.Helper() + before := pendingByUnit(t, srv) + require.NoError(t, b.Publish(t.Context(), topic, data)) + for u, n := range pendingByUnit(t, srv) { + if n > before[u] { + return u + } + } + t.Fatalf("no unit took %v's row", topic) + return "" +} + +func pendingByUnit(t *testing.T, srv *natstest.Server) map[string]uint64 { + t.Helper() + out := map[string]uint64{} + for p := range claimPartitions { + for s := range claimShards { + stream, durable := "WH_INGEST_"+strconv.Itoa(p), "wh-ingest-"+strconv.Itoa(s) + c, err := srv.Operator.JetStream().Consumer(t.Context(), stream, durable) + require.NoError(t, err) + out[stream+"/"+durable] = c.CachedInfo().NumPending + } + } + return out +} + +// unitState is what the server says of a unit: its pin, and its rows +// delivered and awaiting an ack. +func unitState(t *testing.T, srv *natstest.Server, unit string) (pin string, delivered uint64, ackPending int) { + t.Helper() + stream, durable, _ := strings.Cut(unit, "/") + c, err := srv.Operator.JetStream().Consumer(t.Context(), stream, durable) + require.NoError(t, err) + info := c.CachedInfo() + for _, g := range info.PriorityGroups { + pin = g.PinnedClientID + } + return pin, info.Delivered.Consumer, info.NumAckPending +} + +// An owner whose ClickHouse takes longer than the durable's pinned_ttl (10s) +// to answer, with its unit's share of held rows full, keeps its pin all the +// while: it takes at most one row past its share per 5s renewal, the unit +// never reads as unowned, and no other process may reset it. Once the insert returns, the rest of +// the table's rows follow, in order. +func TestShardClaims_BlockedOwnerKeepsItsShard(t *testing.T) { + srv := natstest.Start(t) + pub := shardBroker(t, srv.URL()) + const rows, share = 20, 8 + unit := unitOf(t, srv, pub, mq.Topic{Tenant: "acme", Table: "stuck"}, seqEnvelope(t, "stuck", 0)) + for i := 1; i < rows; i++ { + require.NoError(t, pub.Publish(t.Context(), mq.Topic{Tenant: "acme", Table: "stuck"}, seqEnvelope(t, "stuck", i))) + } + + entered, unblock := make(chan struct{}), make(chan struct{}) + var first sync.Once + log := &insertLog{} + ch := log.server(t, "proc-a", func(string) { + first.Do(func() { close(entered); <-unblock }) + }) + a := startWorkerProc(t, srv.URL(), "proc-a", 3*time.Second, ingest.ClaimConfig{Every: 100 * time.Millisecond, MaxHeld: share}, ch.URL) + t.Cleanup(func() { closeOnce(unblock) }) // first, should the test end early + select { + case <-entered: + case <-time.After(30 * time.Second): + t.Fatal("the first insert never started") + } + pin, delivered, ackPending := unitState(t, srv, unit) + require.NotEmpty(t, pin) + require.GreaterOrEqual(t, ackPending, share, "the unit's share is full") + + other := shardBroker(t, srv.URL()) + blocked := time.Now() + for time.Since(blocked) < 13*time.Second { + time.Sleep(500 * time.Millisecond) + now, d, _ := unitState(t, srv, unit) + require.Equal(t, pin, now, "the pin moved %s into the block", time.Since(blocked).Round(time.Millisecond)) + require.LessOrEqual(t, d-delivered, uint64(time.Since(blocked)/(5*time.Second))+1, "at most one row past the share per renewal") + if time.Since(blocked) > 11*time.Second { + n, err := a.broker.Unowned(t.Context()) + require.NoError(t, err) + assert.Zero(t, n, "a blocked owner's unit is not unowned") + _, err = other.ResetOrphaned(t.Context(), unit) + assert.ErrorIs(t, err, mq.ErrUnitHeld, "nor may another process reset it") + } + } + _, d, ackPending := unitState(t, srv, unit) + assert.Equal(t, int(d), ackPending, "every row delivered is still the owner's, none reset") + t.Logf("%d rows past the share delivered over %s of renewals", d-delivered, time.Since(blocked).Round(time.Second)) + + blockedFor := time.Since(blocked) + closeOnce(unblock) + resumed := time.Now() + require.Eventually(t, func() bool { return len(log.table("stuck")) == rows }, 30*time.Second, 50*time.Millisecond) + t.Logf("blocked %s with its pin kept throughout; the other %d rows followed %s after the insert returned", + blockedFor.Round(100*time.Millisecond), rows-share, time.Since(resumed).Round(10*time.Millisecond)) + assert.True(t, inOrder(log.table("stuck"), rows), "%v", log.table("stuck")) + noWorkerFailure(t, a) +} + +// A table whose ClickHouse hangs fills only its own unit's share of held +// rows: a table on another unit of the same process keeps flowing. +func TestShardClaims_StuckShardDoesNotStallTheOthers(t *testing.T) { + srv := natstest.Start(t) + pub := shardBroker(t, srv.URL()) + stuck := mq.Topic{Tenant: "acme", Table: "stuck"} + stuckUnit := unitOf(t, srv, pub, stuck, seqEnvelope(t, "stuck", 0)) + for i := 1; i < 20; i++ { + require.NoError(t, pub.Publish(t.Context(), stuck, seqEnvelope(t, "stuck", i))) + } + var healthy mq.Topic + for i := 0; healthy.Table == ""; i++ { + topic := mq.Topic{Tenant: "acme", Table: "healthy" + strconv.Itoa(i)} + if unitOf(t, srv, pub, topic, seqEnvelope(t, topic.Table, 0)) != stuckUnit { + healthy = topic + } + } + + entered, unblock := make(chan struct{}), make(chan struct{}) + var first sync.Once + log := &insertLog{} + ch := log.server(t, "proc-a", func(table string) { + if table == "stuck" { + first.Do(func() { close(entered) }) + <-unblock + } + }) + a := startWorkerProc(t, srv.URL(), "proc-a", 3*time.Second, ingest.ClaimConfig{Every: 100 * time.Millisecond, MaxHeld: 8}, ch.URL) + t.Cleanup(func() { closeOnce(unblock) }) // first, should the test end early + select { + case <-entered: + case <-time.After(30 * time.Second): + t.Fatal("the stuck table's insert never started") + } + _, _, held := unitState(t, srv, stuckUnit) + require.GreaterOrEqual(t, held, 8, "the stuck unit's share is full") + + const more = 30 + for i := 1; i <= more; i++ { + require.NoError(t, pub.Publish(t.Context(), healthy, seqEnvelope(t, healthy.Table, i))) + } + began := time.Now() + require.Eventually(t, func() bool { return len(log.table(healthy.Table)) == more+1 }, 20*time.Second, 50*time.Millisecond, + "the healthy table stalled behind the stuck one: %v", log.table(healthy.Table)) + t.Logf("with %s's share full, %s's %d rows were written %s after they were published", + stuck.Table, healthy.Table, more, time.Since(began).Round(10*time.Millisecond)) + assert.True(t, inOrder(log.table(healthy.Table), more+1)) + assert.Empty(t, log.table("stuck"), "the stuck insert has not returned") + closeOnce(unblock) + noWorkerFailure(t, a) +} + +func closeOnce(ch chan struct{}) { + select { + case <-ch: + default: + close(ch) + } +} + +// Under continuous publishing, every table's rows reach ClickHouse exactly +// once and in order while a second process joins (half the units hand over) +// and the first then stops cleanly (the rest move): the process giving a +// unit up writes what it holds before the next owner receives anything. +// The first process's ClickHouse takes longer than a batch window, so a +// unit released before its rows were written would have the next owner's +// rows land first. And a clean stop leaves nothing unsettled behind, so the +// next owner binds its units at once rather than waiting to judge them +// orphaned. +func TestShardClaims_TableOrderAcrossHandoverAndStop(t *testing.T) { + srv := natstest.Start(t) + pub := shardBroker(t, srv.URL()) + log := &insertLog{} + chA := log.server(t, "proc-a", func(string) { time.Sleep(6 * time.Second) }) + chB := log.server(t, "proc-b", nil) + const lease, tables = 3 * time.Second, 16 + cfg := ingest.ClaimConfig{Every: 100 * time.Millisecond} + + a := startWorkerProc(t, srv.URL(), "proc-a", lease, cfg, chA.URL) + var published atomic.Int64 + quit := make(chan struct{}) + var publisher sync.WaitGroup + publisher.Go(func() { + for seq := 0; ; seq++ { + select { + case <-quit: + return + default: + } + for i := range tables { + table := "t" + strconv.Itoa(i) + if err := pub.Publish(context.Background(), mq.Topic{Tenant: "acme", Table: table}, seqEnvelope(t, table, seq)); err != nil { + t.Errorf("publish %s/%d: %v", table, seq, err) + return + } + } + published.Store(int64(seq + 1)) + time.Sleep(20 * time.Millisecond) + } + }) + + time.Sleep(3 * time.Second) + b := startWorkerProc(t, srv.URL(), "proc-b", lease, cfg, chB.URL) + time.Sleep(15 * time.Second) // membership moves, and each moved unit's rows are written first + stopping := time.Now() + stopCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + require.NoError(t, a.stop(stopCtx)) + stopped := time.Now() + t.Logf("proc-a stopped cleanly in %s", stopped.Sub(stopping).Round(10*time.Millisecond)) + time.Sleep(6 * time.Second) + close(quit) + publisher.Wait() + + rows := int(published.Load()) + require.Eventually(t, func() bool { return log.count() >= rows*tables }, 45*time.Second, 100*time.Millisecond, + "inserted %d of %d rows", log.count(), rows*tables) + time.Sleep(time.Second) // anything written twice would show now + moved := 0 + for i := range tables { + table := "t" + strconv.Itoa(i) + assert.True(t, inOrder(log.table(table), rows), "%s: %d rows, want 0..%d in order", table, len(log.table(table)), rows-1) + log.mu.Lock() + if len(log.writers[table]) > 1 { + moved++ + } + assert.LessOrEqual(t, len(log.writers[table]), 3, "%s changed writer more than a handover and a stop explain: %v", table, log.writers[table]) + log.mu.Unlock() + } + t.Logf("%d rows in each of %d tables; %d tables changed writer", rows, tables, moved) + assert.Positive(t, moved, "some table moved between the processes") + + // The tables proc-a still wrote while stopping moved at the stop: proc-b's + // first write of each follows within its batch window (5s), not after a + // wait for the unit to count as orphaned (pinned_ttl, 10s) on top. + log.mu.Lock() + defer log.mu.Unlock() + var gaps []time.Duration + for i := range tables { + table := "t" + strconv.Itoa(i) + if last, ok := log.last["proc-a"][table]; ok && !last.Before(stopping) { + gaps = append(gaps, log.first["proc-b"][table].Sub(stopped)) + } + } + require.NotEmpty(t, gaps, "some tables moved at the stop") + slices.Sort(gaps) + t.Logf("%d tables moved at the stop; proc-b's first write of each came %s to %s after it", len(gaps), gaps[0].Round(10*time.Millisecond), gaps[len(gaps)-1].Round(10*time.Millisecond)) + assert.Less(t, gaps[len(gaps)-1], 7*time.Second, "a unit the stop gave up waited to be judged orphaned") + noWorkerFailure(t, b) +}