Skip to content
21 changes: 21 additions & 0 deletions AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -310,6 +310,27 @@ Environments and pre-release checks: [`.context/release-safety-playbook.md`](.co
EZID is the registrar (ADR 0007 — not Zenodo, whatever `.context/research.md` says).
DOIs are permanent and require explicit confirmation.
- **Zarr.** A derived, latest-only serving copy, not a source of truth.
`zarr.nemar.org` is the stable contract for it (issue #1061, epic #1181 phase 6):
`index.json` stays proxied, edge-cached, and D1-gated. Phase 2's `catalog.json` is a
SEPARATE route (no `<id>/zarr/` segment, so it can never match the redirect predicate
at all) that stays proxied and edge-cached too, but has no D1 gate of its own — its
source document only ever lists already-public datasets. A plain `GET` for a store
object with no allowlisted browser `Origin` — libraries, HPC jobs, agents — 302s
straight to the public S3 object instead of streaming through the Worker, so every
request is still counted without this Worker carrying the bytes (Cloudflare's terms
restrict proxying large files at this scale on a non-Enterprise plan). `HEAD` is
never redirected regardless of Origin: fsspec's `info()` and rclone's sync both probe
with HEAD, and rclone's HTTP backend does not follow HEAD redirects. The redirect
branch skips the D1 visibility gate entirely, and validates the dataset id itself
(`isValidDatasetId`) rather than trusting the path shape — the bucket's own
`NotResource` deny-list (`backend/src/services/bucket-policy.ts`) is the real
enforcement point, so a redirect that 403s at S3 for a private dataset leaks nothing
the proxied 404 does not. The rate limiter counts these hits toward the SAME shared
data-ip bucket a proxied request from that IP would be enforced against, without ever
blocking the redirect itself (`rateLimiter`'s `observeOnly` option,
`backend/src/middleware/rateLimit.ts`) — it logs at most one `console.warn` per
window, the first time the bucket crosses the point enforcement would have tripped,
never a per-request log.
The converter is `scripts/zarr/` **in this repo** and runs on the SDSC Hallu
cron (`scripts/zarr/hallu-zarr.sh`, hourly at `:30`), never in GitHub Actions —
Actions cannot finish a large dataset inside the 120-minute cap (ADR 0029).
Expand Down
68 changes: 64 additions & 4 deletions backend/src/middleware/rateLimit.ts
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@

import type { Context, Next } from "hono";
import { hashApiKey } from "../services/token";
import { hashIp } from "../services/web-session";
import type { Bindings, Variables } from "../types/bindings";

// Rate limit configuration
Expand Down Expand Up @@ -242,7 +243,31 @@ async function isPrivilegedToken(env: Bindings, hashedApiKey: string): Promise<b
}
}

export async function rateLimiter(c: RateLimitContext, next: Next) {
export interface RateLimiterOptions {
/**
* Count this request against its bucket's cache entry -- the SAME
* IP-keyed bucket a proxied request from the same client is enforced
* against -- without ever blocking THIS request (#1181 phase 6 / issue
* #1061). Built for the zarr sub-app's redirect branch: a 302 costs a
* fraction of a millisecond of CPU and zero bytes through this Worker, so
* it must never itself 429, but the counter is still the SAME shared
* bucket -- a client hammering an IP hard enough to matter is still
* visible, and a later PROXIED request from that IP correctly 429s once
* the shared bucket is exhausted, observed traffic included.
*
* Logs exactly ONE `console.warn` per window, the first time the count
* reaches the point enforcement would have blocked -- never per request,
* which would be the same noise this option replaces (an unconditional
* per-request log the zarr sub-app used to emit for every exempted hit).
*/
observeOnly?: boolean;
}

export async function rateLimiter(
c: RateLimitContext,
next: Next,
options: RateLimiterOptions = {},
) {
// Skip rate limiting in development
if (c.env.ENVIRONMENT === "development") {
await next();
Expand Down Expand Up @@ -290,13 +315,48 @@ export async function rateLimiter(c: RateLimitContext, next: Next) {
try {
const cache = caches.default;

// Get current count from cache
// Get current count (and, for the observe-only path, whether this
// window already warned once) from cache.
const cached = await cache.match(cacheKey);
let count = 0;
let warned = false;

if (cached) {
const data = (await cached.json()) as { count: number };
const data = (await cached.json()) as { count: number; warned?: boolean };
count = data.count;
warned = data.warned === true;
}

if (options.observeOnly) {
// Never block -- just record the hit against the shared bucket and
// warn once if it crossed the trip point. `warned` rides in the same
// cache record as `count` so it resets on the same window TTL.
if (count >= maxRequests && !warned) {
// Hashed, not raw: this is a log line, not the enforcement key
// itself (which stays keyed on the raw IP, same as before) --
// avoids putting a raw client IP into structured logs.
// Same helper web-session.ts uses for privacy-preserving IP storage, so
// every IP hash in the codebase is produced by one named function.
const ipHash = await hashIp(rawKey);
console.warn("[rate-limit] observe-only bucket would have tripped", {
ipHash,
path,
count: count + 1,
});
warned = true;
}
const newCount = count + 1;
await cache.put(
cacheKey,
new Response(JSON.stringify({ count: newCount, warned }), {
headers: {
"Content-Type": "application/json",
"Cache-Control": `max-age=${WINDOW_SIZE}`,
},
}),
);
await next();
return;
}

if (count >= maxRequests) {
Expand All @@ -322,7 +382,7 @@ export async function rateLimiter(c: RateLimitContext, next: Next) {

// Increment counter and store in cache with TTL
const newCount = count + 1;
const response = new Response(JSON.stringify({ count: newCount }), {
const response = new Response(JSON.stringify({ count: newCount, warned }), {
headers: {
"Content-Type": "application/json",
"Cache-Control": `max-age=${WINDOW_SIZE}`,
Expand Down
Loading
Loading