From bddffe04b41c0c413f463a6fcd7d714d861cb61f Mon Sep 17 00:00:00 2001 From: Seyed Yahya Shirazi Date: Wed, 2 Sep 2026 10:30:51 -0700 Subject: [PATCH 1/9] feat: redirect zarr bytes for non-browser clients Non-browser GET requests to a zarr store object (no allowlisted Origin) now 302 to the public S3 object instead of proxying through the Worker, never touching D1, the edge cache, or an upstream fetch. index.json stays proxied/edge-cached/D1-gated, HEAD is never redirected, and the rate limiter exempts redirect candidates from the data-ip bucket via the same predicate serve() uses to route. Closes #1061. Part of epic #1181. --- AGENTS.md | 14 + backend/src/routes/zarr-data.ts | 175 ++++++- backend/test/zarr-data-cache.test.ts | 682 ++++++++++++++++++++++++--- 3 files changed, 792 insertions(+), 79 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 36d89b793..97b8fddf2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -310,6 +310,20 @@ Environments and pre-release checks: [`.context/release-safety-playbook.md`](.co EZID is the registrar (ADR 0007 — not Zenodo, whatever `.context/research.md` says). DOIs are permanent and require explicit confirmation. - **Zarr.** A derived, latest-only serving copy, not a source of truth. + `zarr.nemar.org` is the stable contract for it (issue #1061, epic #1181 phase 6): + `index.json` (and phase 2's `catalog.json`) stays proxied, edge-cached, and D1-gated, + but a plain `GET` for a store object with no allowlisted browser `Origin` — libraries, + HPC jobs, agents — 302s straight to the public S3 object instead of streaming through + the Worker, so every request is still counted without this Worker carrying the bytes + (Cloudflare's terms restrict proxying large files at this scale on a non-Enterprise + plan). `HEAD` is never redirected regardless of Origin: fsspec's `info()` and rclone's + sync both probe with HEAD, and rclone's HTTP backend does not follow HEAD redirects. + The redirect branch skips the D1 visibility gate entirely — the bucket's own + `NotResource` deny-list (`backend/src/services/bucket-policy.ts`) is the real + enforcement point, so a redirect that 403s at S3 for a private dataset leaks nothing + the proxied 404 does not — and the rate limiter exempts these hits from the data-ip + bucket rather than throttling traffic that never touches D1, the edge cache, or S3 + through the Worker. The converter is `scripts/zarr/` **in this repo** and runs on the SDSC Hallu cron (`scripts/zarr/hallu-zarr.sh`, hourly at `:30`), never in GitHub Actions — Actions cannot finish a large dataset inside the 120-minute cap (ADR 0029). diff --git a/backend/src/routes/zarr-data.ts b/backend/src/routes/zarr-data.ts index e6e9fe155..48d0b09dc 100644 --- a/backend/src/routes/zarr-data.ts +++ b/backend/src/routes/zarr-data.ts @@ -1,10 +1,12 @@ /** - * zarr.nemar.org data plane (epic #684, Stream D enablement). + * zarr.nemar.org data plane (epic #684, Stream D enablement; redirects added + * #1178 phase 6 / issue #1061, epic #1181). * - * The authoritative browser gateway for the per-recording Zarr serving copies. - * The viewer (nemarOrg/website) reads stores from this host with zarrita; this - * sub-app proxies the public S3 objects under `s3:////zarr/...` and - * adds the three things a browser needs that raw S3 does not give us here: + * The authoritative gateway for the per-recording Zarr serving copies -- but + * "gateway" no longer means "proxy" for every request. `index.json` (the + * contract entry point) and any request carrying an allowlisted browser + * `Origin` are proxied, exactly as before phase 6, and get the three things + * a browser needs that raw S3 does not give us here: * * 1. CORS restricted to NEMAR origins (so a third-party site can't cross-origin * stream our chunks in a browser) -- the S3 data stays openly downloadable, @@ -18,9 +20,30 @@ * with a near-total hit rate). Before phase 1, every Range request * (which level-0 always is) skipped the edge entirely. * - * Only PUBLIC datasets are served (private data is never browser-streamable). - * Mounted on the `zarr.nemar.org` host fork in index.ts and path-mounted at - * `/zarrproxy` for workers.dev/dev access. + * A plain GET for a store object with no allowlisted Origin -- libraries, HPC + * jobs, agents: the overwhelming majority of request volume -- gets none of + * that. It 302s straight to the public S3 object instead (phase 6): + * Cloudflare's terms restrict proxying large files at this scale on a + * non-Enterprise plan, and every request is still counted whether this + * Worker carries the bytes or not. The redirect branch runs BEFORE the D1 + * visibility gate and never touches the edge cache or an upstream fetch -- + * see isRedirectCandidate() and the redirect branch in serve() for why + * that's safe (short version: the bucket's own NotResource deny-list, see + * services/bucket-policy.ts, is the real enforcement point, so a redirect + * that 403s at S3 for a private dataset leaks nothing the proxied 404 + * does not). + * + * HEAD is never redirected, regardless of Origin -- always answered by the + * proxied path. This is load-bearing: fsspec's `info()` and rclone's sync + * both probe with HEAD, and rclone's HTTP backend does not follow HEAD + * redirects (data.ts's fileOrIndexHandler documents the identical rule on + * the archive route). + * + * Only PUBLIC datasets are gated on the proxied branches (private data is + * never browser-streamable; the redirect branch relies on the bucket policy + * instead, see above). Mounted on the `zarr.nemar.org` host fork in + * index.ts and path-mounted at `/zarrproxy` for workers.dev/dev access -- + * the same code serves both entry points, redirect included. */ import type { Context } from "hono"; @@ -62,6 +85,77 @@ export function corsHeaders(origin: string | null): Record { return headers; } +/** + * Path shape for a zarr object request, tolerant of both entry points: + * `//zarr/` (zarr.nemar.org host fork) + * `/zarrproxy//zarr/` (path-mounted on api.nemar.org / workers.dev) + * Mirrors ZARR_PATH_RE in middleware/rateLimit.ts (kept as an independent + * literal rather than imported from there -- this file already imports + * `rateLimiter` FROM rateLimit.ts, so importing the other direction too + * would create a cycle). + */ +const ZARR_OBJECT_PATH_RE = /^(?:\/zarrproxy)?\/[a-z]{2}\d+\/zarr\/?(.*)$/; + +/** + * True when a request to this sub-app WILL take the redirect branch in + * serve() below, rather than being proxied: a `GET` (never `HEAD` -- see + * the module doc comment, that rule is load-bearing) for a store object -- + * not the always-proxied `index.json`, and not an empty/malformed path, + * which the proxied branch 404s either way -- with no allowlisted browser + * `Origin`. + * + * Called from exactly two places: serve()'s own routing decision, and the + * rate-limit middleware below that exempts these hits from the data-ip + * bucket (#1181 phase 6 / issue #1061). Both call sites hand it the same + * raw `method` / `path` / `origin` triple a Hono middleware already has, + * so routing and rate-limiting can never disagree about which requests + * actually reach D1/the edge cache/S3 through this Worker -- there is + * exactly one rule, not two hand-written copies of it. + */ +export function isRedirectCandidate(method: string, path: string, origin: string | null): boolean { + if (method !== "GET") return false; + if (allowedOrigin(origin)) return false; + const match = ZARR_OBJECT_PATH_RE.exec(path); + if (!match) return false; + let rest: string | null; + try { + rest = normalizeBidsPath(decodeURIComponent(match[1])); + } catch { + // Malformed percent-encoding: fall through to the proxied path, which + // hits the same decodeURIComponent call and 500s via onError -- the + // same failure mode as today, not a new one. + return false; + } + // index.json is always top-level (the producer never nests a store under + // that name) and stays proxied/edge-cached/D1-gated -- the contract entry + // point (#1061). Phase 2's catalog.json will need the same exemption once + // it exists. + return Boolean(rest) && rest !== "index.json"; +} + +/** + * Bytes to report for a redirect telemetry point, computed from the + * client's Range header rather than a blanket 0 -- unlike the archive + * route's 302 (which redirects the WHOLE file and never sees a Range), this + * one redirects a request that named its own slice, so the exact count is + * knowable without asking S3. A bounded (`A-B`) or suffix (`-N`) range has + * a known exact length; an open-ended (`A-`) range, a missing header, or + * anything parseCacheableRange() doesn't recognise as a single range does + * not (the total object size is unknown without an upstream fetch, which + * the redirect branch deliberately never makes) and falls back to 0 -- + * never guessed (#1181 phase 6 / issue #1061). + */ +export function bytesFromRangeHeader(rangeHeader: string | null): number { + if (!rangeHeader) return 0; + const normalized = parseCacheableRange(rangeHeader); + if (!normalized) return 0; + const bounded = /^bytes=(\d+)-(\d+)$/.exec(normalized); + if (bounded) return Number(bounded[2]) - Number(bounded[1]) + 1; + const suffix = /^bytes=-(\d+)$/.exec(normalized); + if (suffix) return Number(suffix[1]); + return 0; // open-ended `bytes=A-`: length unknown without hitting S3 +} + function s3PublicUrl(env: Bindings, key: string, base?: string): string { // Test seam (#1178 phase 1): a real Bun.serve() upstream stands in for S3 // in tests via deps.s3Base, instead of parsing env vars for a fake host. @@ -334,6 +428,52 @@ async function serve(c: Context<{ Bindings: Bindings }>, isHead: boolean, deps: } const key = `${datasetId}/zarr/${rest}`; + // Redirect branch (#1181 phase 6 / issue #1061): a non-browser GET for a + // store object never touches D1, the edge cache, or an upstream fetch + // through this Worker -- see the module doc comment for why that's safe. + // isRedirectCandidate is the SAME predicate the rate-limit middleware + // below uses to decide whether to exempt the request from the data-ip + // bucket, so routing and rate-limiting can't disagree about which + // requests actually reach S3 through this Worker. `!isHead` is redundant + // with the predicate's own method check (HEAD can never be a redirect + // candidate) but is left explicit here: the HEAD-never-redirects rule is + // load-bearing enough (fsspec/rclone probe with HEAD; rclone's HTTP + // backend does not follow HEAD redirects) that it should be readable at + // the call site, not only inside the predicate. + if (!isHead && isRedirectCandidate(c.req.method, c.req.path, origin)) { + recordAccess(c.env, { + datasetId, + source: "zarr", + // Blob slots are fixed (buildAccessDataPoint always writes exactly + // three); a redirect isn't a new AccessSource, it's a marker in the + // existing `detail` slot that zarrObjectType() never produces on its + // own, so the dashboard can tell a redirected chunk apart from a + // proxied one without a schema change. + detail: "chunk-redirect", + bytes: bytesFromRangeHeader(c.req.header("range") ?? null), + }); + return new Response(null, { + status: 302, + headers: { + Location: s3PublicUrl(c.env, key, deps.s3Base), + // The redirect TARGET is stable for a given key (bucket/region/key + // never change shape), unlike the proxied branch's per-object-class + // TTL from cacheControlFor() -- an hour is a reasonable middle + // ground for a value that in practice never changes. + "Cache-Control": "public, max-age=3600", + // Documents that the SAME URL answers differently for an + // allowlisted browser Origin (proxied bytes, not a redirect), even + // though nothing about this particular response is cached by us. + Vary: "Origin", + // No Content-Length (RFC 9110 SS8.6: it would describe the empty + // redirect body, not the S3 target) and no + // Access-Control-Allow-Origin (nothing of ours is exposed here -- + // see data.ts's fileOrIndexHandler for the identical archive-route + // convention). + }, + }); + } + // Gate FIRST, before ANY cache lookup, on every request regardless of hit // or miss (#1181 review item 1): a dataset flipped to private must stop // being served immediately, not up to a day later when the last-primed @@ -347,6 +487,13 @@ async function serve(c: Context<{ Bindings: Bindings }>, isHead: boolean, deps: // missing-object 404 negative cache below is safe precisely because this // gate now runs first: a cache lookup can only ever be reached once we've // reconfirmed the dataset is CURRENTLY public. + // + // This gate does NOT run on the redirect branch above (#1181 phase 6 / + // issue #1061): a redirect never touches D1, the edge cache, or an + // upstream fetch, so there is no cache entry or D1 staleness to protect + // here. The bucket's own NotResource deny-list (services/bucket-policy.ts) + // is the enforcement point on that path instead -- a redirect to a URL + // that 403s at S3 for a private dataset leaks nothing this 404 does not. if (!(await isPublicDataset(c.env, datasetId))) { return c.body(null, 404, { ...cors, "Cache-Control": NOT_FOUND_CACHE_CONTROL }); } @@ -598,6 +745,18 @@ export function createZarrDataRoutes( // Variables slot, all present at runtime on any Hono context. app.use("*", async (c, next) => { if (c.req.method === "OPTIONS") return next(); + // Redirect candidates (#1181 phase 6 / issue #1061) cost a fraction of a + // millisecond of CPU and zero bytes through this Worker -- the IP-keyed + // data-ip bucket exists to bound a runaway PROXIED loop, not to punish a + // cluster behind one NAT address for traffic that never touches D1, the + // edge cache, or S3 through here. Exempt them and log a counter instead + // of enforcing, per the issue. Uses the SAME predicate serve() uses to + // pick its response branch (see isRedirectCandidate's doc comment), so + // this can never exempt a request that serve() actually proxies. + if (isRedirectCandidate(c.req.method, c.req.path, c.req.header("origin") ?? null)) { + console.info("[zarr-data] rate-limit exempt (redirect candidate)", { path: c.req.path }); + return next(); + } const res = await rateLimiter(c as unknown as Parameters[0], next); // rateLimiter returns a bare 429 (no zarr CORS) when the bucket is exhausted; // without ACAO the browser sees an opaque failure instead of a readable 429 + diff --git a/backend/test/zarr-data-cache.test.ts b/backend/test/zarr-data-cache.test.ts index 38125b189..662aaa1e6 100644 --- a/backend/test/zarr-data-cache.test.ts +++ b/backend/test/zarr-data-cache.test.ts @@ -42,14 +42,17 @@ import type { Database } from "bun:sqlite"; import { afterAll, beforeAll, beforeEach, describe, expect, test } from "bun:test"; import type { Server } from "bun"; -import type { Hono } from "hono"; +import { Hono } from "hono"; +import { __limits } from "../src/middleware/rateLimit"; import { type CacheLike, FULL_OBJECT_CACHE_MAX_BYTES, RANGE_CACHE_MAX_BYTES, type ZarrDataDeps, + bytesFromRangeHeader, canonicalCacheUrl, createZarrDataRoutes, + isRedirectCandidate, parseCacheableRange, } from "../src/routes/zarr-data"; import type { Bindings } from "../src/types/bindings"; @@ -462,15 +465,32 @@ describe("parseCacheableRange", () => { const CHUNK_PATH = `/${PUBLIC_ID}/zarr/store.zarr/c/0/0`; const CHUNK_KEY = `${PUBLIC_ID}/zarr/store.zarr/c/0/0`; +// Phase 6 (#1181 phase 6 / issue #1061) redirects a store-object GET with no +// allowlisted Origin instead of proxying it. Every phase-1 test below that +// exercises the proxied/edge-cache machinery on a store-object path (not +// index.json, which stays proxied regardless of Origin) needs an allowed +// Origin to keep reaching that machinery -- exactly what a real browser +// viewer request looks like, and the ONLY thing that changed about how +// these paths reach the proxy branch post-phase-6. +const BROWSER_ORIGIN = "https://nemar.org"; + describe("range-keyed cache entries (#1178 item 3)", () => { + // Phase 6: a bare (no-Origin) chunk GET now redirects instead of proxying + // (see the "routing fork" describe block below), so every request here + // needs an allowed browser Origin to keep exercising the edge-cache + // machinery these tests are actually about. test("(a) ranged GET miss then identical ranged GET hit", async () => { - const first = await request(CHUNK_PATH, { headers: { Range: "bytes=0-99" } }); + const first = await request(CHUNK_PATH, { + headers: { Range: "bytes=0-99", Origin: BROWSER_ORIGIN }, + }); expect(first.status).toBe(206); expect(first.headers.get("content-range")).toBe(`bytes 0-99/${CHUNK_BYTES.length}`); const firstBytes = new Uint8Array(await first.arrayBuffer()); expect(countUpstream(CHUNK_KEY, "bytes=0-99")).toBe(1); - const second = await request(CHUNK_PATH, { headers: { Range: "bytes=0-99" } }); + const second = await request(CHUNK_PATH, { + headers: { Range: "bytes=0-99", Origin: BROWSER_ORIGIN }, + }); expect(second.status).toBe(206); expect(second.headers.get("content-range")).toBe(`bytes 0-99/${CHUNK_BYTES.length}`); const secondBytes = new Uint8Array(await second.arrayBuffer()); @@ -483,11 +503,13 @@ describe("range-keyed cache entries (#1178 item 3)", () => { }); test("(b) a different range is a separate miss", async () => { - await request(CHUNK_PATH, { headers: { Range: "bytes=0-99" } }); + await request(CHUNK_PATH, { headers: { Range: "bytes=0-99", Origin: BROWSER_ORIGIN } }); expect(countUpstream(CHUNK_KEY, "bytes=0-99")).toBe(1); expect(countUpstream(CHUNK_KEY, "bytes=100-199")).toBe(0); - const res = await request(CHUNK_PATH, { headers: { Range: "bytes=100-199" } }); + const res = await request(CHUNK_PATH, { + headers: { Range: "bytes=100-199", Origin: BROWSER_ORIGIN }, + }); expect(res.status).toBe(206); expect(res.headers.get("content-range")).toBe(`bytes 100-199/${CHUNK_BYTES.length}`); expect(countUpstream(CHUNK_KEY, "bytes=100-199")).toBe(1); @@ -496,11 +518,11 @@ describe("range-keyed cache entries (#1178 item 3)", () => { }); test("(c) full GET after a ranged GET is a separate miss", async () => { - await request(CHUNK_PATH, { headers: { Range: "bytes=0-99" } }); + await request(CHUNK_PATH, { headers: { Range: "bytes=0-99", Origin: BROWSER_ORIGIN } }); expect(countUpstream(CHUNK_KEY, "bytes=0-99")).toBe(1); expect(countUpstream(CHUNK_KEY, "")).toBe(0); - const res = await request(CHUNK_PATH); + const res = await request(CHUNK_PATH, { headers: { Origin: BROWSER_ORIGIN } }); expect(res.status).toBe(200); const bytes = new Uint8Array(await res.arrayBuffer()); expect(bytes.length).toBe(CHUNK_BYTES.length); @@ -509,7 +531,9 @@ describe("range-keyed cache entries (#1178 item 3)", () => { }); test("(d) a client-supplied __cr on a full GET is ignored for keying", async () => { - const res = await request(`${CHUNK_PATH}?__cr=bytes%3D0-9`); + const res = await request(`${CHUNK_PATH}?__cr=bytes%3D0-9`, { + headers: { Origin: BROWSER_ORIGIN }, + }); expect(res.status).toBe(200); const bytes = new Uint8Array(await res.arrayBuffer()); expect(bytes.length).toBe(CHUNK_BYTES.length); @@ -520,17 +544,21 @@ describe("range-keyed cache entries (#1178 item 3)", () => { // A second, plain full GET is a cache HIT sharing the same canonical // entry -- proving __cr was dropped from the key on the priming request. - const second = await request(CHUNK_PATH); + const second = await request(CHUNK_PATH, { headers: { Origin: BROWSER_ORIGIN } }); expect(second.status).toBe(200); expect(countUpstream(CHUNK_KEY, "")).toBe(1); }); test("(e) multi-range is passed through and never cached", async () => { - const first = await request(CHUNK_PATH, { headers: { Range: "bytes=0-1,4-5" } }); + const first = await request(CHUNK_PATH, { + headers: { Range: "bytes=0-1,4-5", Origin: BROWSER_ORIGIN }, + }); expect(first.status).toBe(200); // fake upstream ignores the unsupported spec expect(countUpstream(CHUNK_KEY, "bytes=0-1,4-5")).toBe(1); - const second = await request(CHUNK_PATH, { headers: { Range: "bytes=0-1,4-5" } }); + const second = await request(CHUNK_PATH, { + headers: { Range: "bytes=0-1,4-5", Origin: BROWSER_ORIGIN }, + }); expect(second.status).toBe(200); // Hit upstream again -- never cached. expect(countUpstream(CHUNK_KEY, "bytes=0-1,4-5")).toBe(2); @@ -538,12 +566,16 @@ describe("range-keyed cache entries (#1178 item 3)", () => { test("(f) 416 is returned to the client, not mapped to 502, and never cached (#1181 review item 5)", async () => { const outOfRange = "bytes=999999-1000010"; - const first = await request(CHUNK_PATH, { headers: { Range: outOfRange } }); + const first = await request(CHUNK_PATH, { + headers: { Range: outOfRange, Origin: BROWSER_ORIGIN }, + }); expect(first.status).toBe(416); expect(first.headers.get("content-range")).toBe(`bytes */${CHUNK_BYTES.length}`); expect(countUpstream(CHUNK_KEY, outOfRange)).toBe(1); - const second = await request(CHUNK_PATH, { headers: { Range: outOfRange } }); + const second = await request(CHUNK_PATH, { + headers: { Range: outOfRange, Origin: BROWSER_ORIGIN }, + }); expect(second.status).toBe(416); // Hit upstream again -- the 416 was never written to the cache. expect(countUpstream(CHUNK_KEY, outOfRange)).toBe(2); @@ -552,12 +584,19 @@ describe("range-keyed cache entries (#1178 item 3)", () => { describe("Cache-Control by tokening (#1178 item 4 / #1035, #1181 review item 9)", () => { test("(g) zarr.json: tokened gets 86400, untokened gets 60", async () => { - const tokened = await request(`/${PUBLIC_ID}/zarr/store.zarr/zarr.json?v=abc123`); + // Phase 6: a per-store zarr.json is a store object, not the top-level + // index.json, so a bare request redirects -- an allowed Origin is + // needed to reach the proxied Cache-Control logic under test here. + const tokened = await request(`/${PUBLIC_ID}/zarr/store.zarr/zarr.json?v=abc123`, { + headers: { Origin: BROWSER_ORIGIN }, + }); expect(tokened.headers.get("cache-control")).toBe( "public, max-age=86400, stale-while-revalidate=86400", ); - const untokened = await request(`/${PUBLIC_ID}/zarr/store.zarr/zarr.json`); + const untokened = await request(`/${PUBLIC_ID}/zarr/store.zarr/zarr.json`, { + headers: { Origin: BROWSER_ORIGIN }, + }); expect(untokened.headers.get("cache-control")).toBe( "public, max-age=60, stale-while-revalidate=300", ); @@ -578,11 +617,13 @@ describe("Cache-Control by tokening (#1178 item 4 / #1035, #1181 review item 9)" }); test("chunks are unchanged: 86400 regardless of tokening", async () => { - const untokened = await request(CHUNK_PATH); + const untokened = await request(CHUNK_PATH, { headers: { Origin: BROWSER_ORIGIN } }); expect(untokened.headers.get("cache-control")).toBe( "public, max-age=86400, stale-while-revalidate=86400", ); - const tokened = await request(`${CHUNK_PATH}?v=abc123`); + const tokened = await request(`${CHUNK_PATH}?v=abc123`, { + headers: { Origin: BROWSER_ORIGIN }, + }); expect(tokened.headers.get("cache-control")).toBe( "public, max-age=86400, stale-while-revalidate=86400", ); @@ -592,20 +633,21 @@ describe("Cache-Control by tokening (#1178 item 4 / #1035, #1181 review item 9)" describe("v-token round trip (#1181 review item 14)", () => { test("same tokened URL is a hit; different v values are separate entries; untokened is a third", async () => { const path = `/${PUBLIC_ID}/zarr/store.zarr/zarr.json`; + const headers = { Origin: BROWSER_ORIGIN }; - const v1a = await request(`${path}?v=aaa`); + const v1a = await request(`${path}?v=aaa`, { headers }); expect(v1a.status).toBe(200); expect(countUpstream(`${PUBLIC_ID}/zarr/store.zarr/zarr.json`, "")).toBe(1); - const v1b = await request(`${path}?v=aaa`); + const v1b = await request(`${path}?v=aaa`, { headers }); expect(v1b.status).toBe(200); expect(countUpstream(`${PUBLIC_ID}/zarr/store.zarr/zarr.json`, "")).toBe(1); // hit - const v2 = await request(`${path}?v=bbb`); + const v2 = await request(`${path}?v=bbb`, { headers }); expect(v2.status).toBe(200); expect(countUpstream(`${PUBLIC_ID}/zarr/store.zarr/zarr.json`, "")).toBe(2); // separate entry - const untokened = await request(path); + const untokened = await request(path, { headers }); expect(untokened.status).toBe(200); expect(countUpstream(`${PUBLIC_ID}/zarr/store.zarr/zarr.json`, "")).toBe(3); // a third entry }); @@ -618,8 +660,12 @@ describe("HEAD never touches the cache", () => { expect(countUpstream(`${PUBLIC_ID}/zarr/store.zarr/zarr.json`, "")).toBe(1); // A GET for the same object afterwards is still a miss -- HEAD wrote - // nothing to the cache. - const get = await request(`/${PUBLIC_ID}/zarr/store.zarr/zarr.json`); + // nothing to the cache. An allowed Origin keeps this GET on the proxied + // path (phase 6): a bare GET would redirect instead of reaching the + // cache at all. + const get = await request(`/${PUBLIC_ID}/zarr/store.zarr/zarr.json`, { + headers: { Origin: BROWSER_ORIGIN }, + }); expect(get.status).toBe(200); expect(countUpstream(`${PUBLIC_ID}/zarr/store.zarr/zarr.json`, "")).toBe(2); @@ -650,44 +696,62 @@ describe("CORS reapplication on a cache hit (#1178)", () => { const BLOCKED = "https://evil.example.org"; test("(i) an allowed origin gets ACAO on a hit, a blocked one does not", async () => { - const path = `/${PUBLIC_ID}/zarr/store.zarr/zarr.json`; + // Phase 6: a store object (store.zarr/zarr.json) with a blocked Origin + // now redirects rather than reaching the cache at all, which would make + // this test pass for the wrong reason (a redirect never carries ACAO + // either). index.json is never a redirect candidate, so a blocked + // Origin genuinely reaches the shared cache entry here -- the scenario + // this test is actually about: the Cache API doesn't honour + // Vary: Origin, so CORS has to be reapplied per requester on the way out. + const path = `/${PUBLIC_ID}/zarr/index.json`; + const key = `${PUBLIC_ID}/zarr/index.json`; // Prime the cache from the allowed origin. const primed = await request(path, { headers: { Origin: ALLOWED } }); expect(primed.headers.get("access-control-allow-origin")).toBe(ALLOWED); - expect(countUpstream(`${PUBLIC_ID}/zarr/store.zarr/zarr.json`, "")).toBe(1); + expect(countUpstream(key, "")).toBe(1); // A blocked origin hits the SAME cache entry (Vary: Origin is not // honoured by the Cache API) but must not get the priming origin's ACAO. const blockedHit = await request(path, { headers: { Origin: BLOCKED } }); - expect(countUpstream(`${PUBLIC_ID}/zarr/store.zarr/zarr.json`, "")).toBe(1); // still a hit + expect(blockedHit.status).toBe(200); + expect(countUpstream(key, "")).toBe(1); // still a hit expect(blockedHit.headers.get("access-control-allow-origin")).toBeNull(); // A subsequent allowed-origin request gets its ACAO back. const allowedHit = await request(path, { headers: { Origin: ALLOWED } }); - expect(countUpstream(`${PUBLIC_ID}/zarr/store.zarr/zarr.json`, "")).toBe(1); // still a hit + expect(countUpstream(key, "")).toBe(1); // still a hit expect(allowedHit.headers.get("access-control-allow-origin")).toBe(ALLOWED); }); test("a range hit also gets CORS reapplied per requester", async () => { - await request(CHUNK_PATH, { headers: { Range: "bytes=0-9", Origin: ALLOWED } }); - const blockedHit = await request(CHUNK_PATH, { - headers: { Range: "bytes=0-9", Origin: BLOCKED }, + // Same reasoning as above: index.json stays proxied for a blocked + // Origin (phase 6), so this exercises the range-cache CORS-reapplication + // path rather than the redirect branch a store object would now take. + const path = `/${PUBLIC_ID}/zarr/index.json`; + const key = `${PUBLIC_ID}/zarr/index.json`; + await request(path, { headers: { Range: "bytes=0-5", Origin: ALLOWED } }); + const blockedHit = await request(path, { + headers: { Range: "bytes=0-5", Origin: BLOCKED }, }); expect(blockedHit.status).toBe(206); expect(blockedHit.headers.get("access-control-allow-origin")).toBeNull(); - expect(countUpstream(CHUNK_KEY, "bytes=0-9")).toBe(1); + expect(countUpstream(key, "bytes=0-5")).toBe(1); }); }); describe("404 caching (#1178 item 4)", () => { test("(j) missing-object 404 carries max-age=60 and is served from cache", async () => { + // An allowed Origin keeps this on the proxied path (phase 6) -- a bare + // request to a store object now redirects instead of reaching the + // missing-object 404/negative-cache logic under test here. const missing = `/${PUBLIC_ID}/zarr/nope.zarr/zarr.json`; - const first = await request(missing); + const headers = { Origin: BROWSER_ORIGIN }; + const first = await request(missing, { headers }); expect(first.status).toBe(404); expect(first.headers.get("cache-control")).toBe("public, max-age=60"); expect(countUpstream(`${PUBLIC_ID}/zarr/nope.zarr/zarr.json`, "")).toBe(1); - const second = await request(missing); + const second = await request(missing, { headers }); expect(second.status).toBe(404); expect(second.headers.get("cache-control")).toBe("public, max-age=60"); // Zero additional upstream hits -- served from the edge. @@ -695,14 +759,20 @@ describe("404 caching (#1178 item 4)", () => { }); test("(j) private dataset 404 is not cached", async () => { + // An allowed Origin keeps this on the proxied/D1-gated path. A bare + // request to this same store-object path is ALSO still a 302 today + // (phase 6 redirects regardless of visibility, see the "no D1 read on + // the redirect path" describe block below) -- this test is specifically + // about the D1 gate a BROWSER request still goes through. const path = `/${PRIVATE_ID}/zarr/store.zarr/zarr.json`; - const first = await request(path); + const headers = { Origin: BROWSER_ORIGIN }; + const first = await request(path, { headers }); expect(first.status).toBe(404); expect(first.headers.get("cache-control")).toBe("public, max-age=60"); // The D1 gate rejects before any upstream fetch. expect(countUpstream(`${PRIVATE_ID}/zarr/store.zarr/zarr.json`)).toBe(0); - const second = await request(path); + const second = await request(path, { headers }); expect(second.status).toBe(404); // Still gated by D1 on every request -- never short-circuited by a // cached entry (the private dataset's visibility can flip). @@ -710,6 +780,8 @@ describe("404 caching (#1178 item 4)", () => { }); test("bad-path 404 carries max-age=60 and is not cached", async () => { + // Empty rest 404s before the redirect decision is even made (serve()'s + // own null/empty-rest check runs first), so no Origin is needed here. const first = await request(`/${PUBLIC_ID}/zarr/`); expect(first.status).toBe(404); expect(first.headers.get("cache-control")).toBe("public, max-age=60"); @@ -721,7 +793,9 @@ describe("404 caching (#1178 item 4)", () => { expect(head.status).toBe(404); expect(countUpstream(`${PUBLIC_ID}/zarr/also-missing.zarr/zarr.json`, "")).toBe(1); - const get = await request(missing); + // An allowed Origin keeps this GET on the proxied path (phase 6); a + // bare GET here would redirect rather than hit the missing-object 404. + const get = await request(missing, { headers: { Origin: BROWSER_ORIGIN } }); expect(get.status).toBe(404); // The HEAD did not populate the cache -- this GET still hits upstream. expect(countUpstream(`${PUBLIC_ID}/zarr/also-missing.zarr/zarr.json`, "")).toBe(2); @@ -731,12 +805,13 @@ describe("404 caching (#1178 item 4)", () => { const missing = `/${PUBLIC_ID}/zarr/still-missing.zarr/zarr.json`; const missingKey = `${PUBLIC_ID}/zarr/still-missing.zarr/zarr.json`; const range = "bytes=0-9"; + const headers = { Origin: BROWSER_ORIGIN }; - const first = await request(missing, { headers: { Range: range } }); + const first = await request(missing, { headers: { Range: range, ...headers } }); expect(first.status).toBe(404); expect(countUpstream(missingKey, range)).toBe(1); - const second = await request(missing, { headers: { Range: range } }); + const second = await request(missing, { headers: { Range: range, ...headers } }); expect(second.status).toBe(404); // The second request's range-key lookup misses (each range mints its // own key) but falls back to the canonical key, which the first @@ -745,7 +820,7 @@ describe("404 caching (#1178 item 4)", () => { // A DIFFERENT range on the same missing object also converges on the // same canonical negative entry. - const third = await request(missing, { headers: { Range: "bytes=10-19" } }); + const third = await request(missing, { headers: { Range: "bytes=10-19", ...headers } }); expect(third.status).toBe(404); expect(countUpstream(missingKey, "bytes=10-19")).toBe(0); }); @@ -753,14 +828,21 @@ describe("404 caching (#1178 item 4)", () => { describe("visibility gate runs before every cache lookup (#1181 review item 1)", () => { test("flipping a dataset private stops both a primed full-object entry and a primed range entry immediately", async () => { + // An allowed Origin keeps both paths on the proxied/D1-gated branch + // (phase 6): a bare request to either would redirect regardless of + // visibility, which is a different (and separately tested, see "no D1 + // read on the redirect path" below) guarantee than the one under test + // here -- that the PROXIED path's D1 gate runs before every cache + // lookup, not just on a miss. + const origin = { Origin: BROWSER_ORIGIN }; const fullPath = `/${PUBLIC_ID}/zarr/store.zarr/zarr.json`; const fullKey = `${PUBLIC_ID}/zarr/store.zarr/zarr.json`; const range = "bytes=0-9"; // Prime both a full-object entry and a range entry while public. - const primedFull = await request(fullPath); + const primedFull = await request(fullPath, { headers: origin }); expect(primedFull.status).toBe(200); - const primedRange = await request(CHUNK_PATH, { headers: { Range: range } }); + const primedRange = await request(CHUNK_PATH, { headers: { Range: range, ...origin } }); expect(primedRange.status).toBe(206); expect(countUpstream(fullKey, "")).toBe(1); expect(countUpstream(CHUNK_KEY, range)).toBe(1); @@ -768,9 +850,9 @@ describe("visibility gate runs before every cache lookup (#1181 review item 1)", // Flip visibility to private directly in D1. db.run("UPDATE datasets SET visibility = 'private' WHERE dataset_id = ?", [PUBLIC_ID]); - const flippedFull = await request(fullPath); + const flippedFull = await request(fullPath, { headers: origin }); expect(flippedFull.status).toBe(404); - const flippedRange = await request(CHUNK_PATH, { headers: { Range: range } }); + const flippedRange = await request(CHUNK_PATH, { headers: { Range: range, ...origin } }); expect(flippedRange.status).toBe(404); // No new upstream calls either -- the gate rejected BEFORE either cache // entry was even consulted, so a stale cached response never left the @@ -781,9 +863,9 @@ describe("visibility gate runs before every cache lookup (#1181 review item 1)", // Flip back to public: the primed cache entries are untouched by any of // this and are served again without a new upstream hit. db.run("UPDATE datasets SET visibility = 'public' WHERE dataset_id = ?", [PUBLIC_ID]); - const restoredFull = await request(fullPath); + const restoredFull = await request(fullPath, { headers: origin }); expect(restoredFull.status).toBe(200); - const restoredRange = await request(CHUNK_PATH, { headers: { Range: range } }); + const restoredRange = await request(CHUNK_PATH, { headers: { Range: range, ...origin } }); expect(restoredRange.status).toBe(206); expect(countUpstream(fullKey, "")).toBe(1); expect(countUpstream(CHUNK_KEY, range)).toBe(1); @@ -801,7 +883,7 @@ describe("a 206 with no Content-Range fails closed (#1181 review item 3)", () => warnings.push(args); }; try { - const first = await request(path, { headers: { Range: range } }); + const first = await request(path, { headers: { Range: range, Origin: BROWSER_ORIGIN } }); expect(first.status).toBe(206); expect(first.headers.get("content-range")).toBeNull(); expect(countUpstream(key, range)).toBe(1); @@ -809,7 +891,7 @@ describe("a 206 with no Content-Range fails closed (#1181 review item 3)", () => expect(String(warnings[0][0])).toContain("without Content-Range"); expect(String(warnings[0][0])).toContain(key); - const second = await request(path, { headers: { Range: range } }); + const second = await request(path, { headers: { Range: range, Origin: BROWSER_ORIGIN } }); expect(second.status).toBe(206); // Never cached -- the second identical request hits upstream again. expect(countUpstream(key, range)).toBe(2); @@ -826,13 +908,13 @@ describe("size caps on cached entries (#1181 review item 4)", () => { const end = RANGE_CACHE_MAX_BYTES; // inclusive 0..end => cap+1 bytes, over the cap const range = `bytes=0-${end}`; - const first = await request(path, { headers: { Range: range } }); + const first = await request(path, { headers: { Range: range, Origin: BROWSER_ORIGIN } }); expect(first.status).toBe(206); const firstBuf = await first.arrayBuffer(); expect(firstBuf.byteLength).toBe(end + 1); expect(countUpstream(key, range)).toBe(1); - const second = await request(path, { headers: { Range: range } }); + const second = await request(path, { headers: { Range: range, Origin: BROWSER_ORIGIN } }); expect(second.status).toBe(206); // Not cached -- served fresh from upstream again. expect(countUpstream(key, range)).toBe(2); @@ -842,13 +924,13 @@ describe("size caps on cached entries (#1181 review item 4)", () => { const path = `/${PUBLIC_ID}/zarr/store.zarr/c/big/oversized`; const key = `${PUBLIC_ID}/zarr/store.zarr/c/big/oversized`; - const first = await request(path); + const first = await request(path, { headers: { Origin: BROWSER_ORIGIN } }); expect(first.status).toBe(200); const firstBuf = await first.arrayBuffer(); expect(firstBuf.byteLength).toBe(CAP_TEST_BYTES.length); expect(countUpstream(key, "")).toBe(1); - const second = await request(path); + const second = await request(path, { headers: { Origin: BROWSER_ORIGIN } }); expect(second.status).toBe(200); expect(countUpstream(key, "")).toBe(2); // not cached }, 20000); @@ -858,13 +940,13 @@ describe("size caps on cached entries (#1181 review item 4)", () => { const key = `${PUBLIC_ID}/zarr/store.zarr/c/quirk/no-length-header`; const range = "bytes=0-9"; - const first = await request(path, { headers: { Range: range } }); + const first = await request(path, { headers: { Range: range, Origin: BROWSER_ORIGIN } }); expect(first.status).toBe(206); expect(first.headers.get("content-length")).toBeNull(); expect(first.headers.get("content-range")).toBe(`bytes 0-9/${QUIRK_BYTES.length}`); expect(countUpstream(key, range)).toBe(1); - const second = await request(path, { headers: { Range: range } }); + const second = await request(path, { headers: { Range: range, Origin: BROWSER_ORIGIN } }); expect(second.status).toBe(206); expect(countUpstream(key, range)).toBe(2); // not cached -- size couldn't be checked }); @@ -876,11 +958,11 @@ describe("upstream can ignore a well-formed single range (#1181 review item 18)" const key = `${PUBLIC_ID}/zarr/store.zarr/c/quirk/ignored-range`; const range = "bytes=0-9"; - const first = await request(path, { headers: { Range: range } }); + const first = await request(path, { headers: { Range: range, Origin: BROWSER_ORIGIN } }); expect(first.status).toBe(200); expect(countUpstream(key, range)).toBe(1); - const second = await request(path, { headers: { Range: range } }); + const second = await request(path, { headers: { Range: range, Origin: BROWSER_ORIGIN } }); expect(second.status).toBe(200); // Never cached -- upstream hit again. expect(countUpstream(key, range)).toBe(2); @@ -891,7 +973,9 @@ describe("concurrent identical range requests (#1181 review item 16)", () => { test("four at once: all 206 with identical bytes; at most 4 upstream hits, then a hit", async () => { const range = "bytes=0-49"; const results = await Promise.all( - Array.from({ length: 4 }, () => request(CHUNK_PATH, { headers: { Range: range } })), + Array.from({ length: 4 }, () => + request(CHUNK_PATH, { headers: { Range: range, Origin: BROWSER_ORIGIN } }), + ), ); for (const res of results) { expect(res.status).toBe(206); @@ -905,7 +989,7 @@ describe("concurrent identical range requests (#1181 review item 16)", () => { expect(hitsAfterBurst).toBeGreaterThan(0); expect(hitsAfterBurst).toBeLessThanOrEqual(4); - const fifth = await request(CHUNK_PATH, { headers: { Range: range } }); + const fifth = await request(CHUNK_PATH, { headers: { Range: range, Origin: BROWSER_ORIGIN } }); expect(fifth.status).toBe(206); // No new upstream hit -- the burst left a usable cache entry. expect(countUpstream(CHUNK_KEY, range)).toBe(hitsAfterBurst); @@ -920,11 +1004,11 @@ describe("canonical key from the normalised S3 key (#1181 review item 8)", () => const encoded = `/${PUBLIC_ID}/zarr/store.zarr/zarr%2ejson`; const objectKey = `${PUBLIC_ID}/zarr/store.zarr/zarr.json`; - const first = await request(plain); + const first = await request(plain, { headers: { Origin: BROWSER_ORIGIN } }); expect(first.status).toBe(200); expect(countUpstream(objectKey, "")).toBe(1); - const second = await request(encoded); + const second = await request(encoded, { headers: { Origin: BROWSER_ORIGIN } }); expect(second.status).toBe(200); const secondBytes = new Uint8Array(await second.arrayBuffer()); expect(secondBytes).toEqual(ZARR_JSON); @@ -946,10 +1030,13 @@ describe("cache.put failures are logged, not fatal (#1181 review item 6)", () => calls.push(args); }; try { + // An allowed Origin keeps this on the proxied/cache-writing path + // (phase 6) -- a bare request would redirect and never call + // cache.put() at all. const res = await requestWith( rejectingApp, `/${PUBLIC_ID}/zarr/store.zarr/zarr.json`, - {}, + { headers: { Origin: BROWSER_ORIGIN } }, testEnv(), ); expect(res.status).toBe(200); @@ -975,7 +1062,7 @@ describe("cache.put failures are logged, not fatal (#1181 review item 6)", () => const res = await requestWith( rejectingApp, CHUNK_PATH, - { headers: { Range: "bytes=0-9" } }, + { headers: { Range: "bytes=0-9", Origin: BROWSER_ORIGIN } }, testEnv(), ); expect(res.status).toBe(206); @@ -999,7 +1086,7 @@ describe("cache.put failures are logged, not fatal (#1181 review item 6)", () => const res = await requestWith( throwingPutApp, `/${PUBLIC_ID}/zarr/store.zarr/zarr.json`, - {}, + { headers: { Origin: BROWSER_ORIGIN } }, testEnv(), ); expect(res.status).toBe(200); @@ -1025,7 +1112,7 @@ describe("cache.put failures are logged, not fatal (#1181 review item 6)", () => const res = await requestWith( throwingPutApp, CHUNK_PATH, - { headers: { Range: "bytes=0-9" } }, + { headers: { Range: "bytes=0-9", Origin: BROWSER_ORIGIN } }, testEnv(), ); expect(res.status).toBe(206); @@ -1065,6 +1152,11 @@ describe("onError applies CORS to an unhandled failure (#1181 review item 7)", ( }); test("a blocked origin gets 500 with no ACAO", async () => { + // Phase 6: a blocked-Origin request to a STORE object (store.zarr/ + // zarr.json) now redirects and never calls cache.match() at all, so it + // would no longer reach onError -- use index.json (never a redirect + // candidate) to keep exercising the throwing-cache/onError path with a + // blocked Origin. const throwingApp = createZarrDataRoutes({ ...testDeps(), cache: () => new ThrowingMatchCache(), @@ -1074,7 +1166,7 @@ describe("onError applies CORS to an unhandled failure (#1181 review item 7)", ( try { const res = await requestWith( throwingApp, - `/${PUBLIC_ID}/zarr/store.zarr/zarr.json`, + `/${PUBLIC_ID}/zarr/index.json`, { headers: { Origin: "https://evil.example.org" } }, testEnv(), ); @@ -1098,20 +1190,24 @@ describe("telemetry with a real binding-shaped collector (#1181 review item 15)" }, } as unknown as Bindings; - await requestWithEnv(CHUNK_PATH, { headers: { Range: "bytes=0-9" } }, env); // range MISS + // An allowed Origin keeps these on the proxied path (phase 6): this + // test is about recordAccess()'s bytes reporting on the PROXY branch, + // which has its own dedicated tests above for the redirect branch. + const origin = { Origin: BROWSER_ORIGIN }; + await requestWithEnv(CHUNK_PATH, { headers: { Range: "bytes=0-9", ...origin } }, env); // range MISS expect(points.length).toBe(1); expect(points[0].doubles[0]).toBe(10); // bytes 0-9 inclusive - await requestWithEnv(CHUNK_PATH, { headers: { Range: "bytes=0-9" } }, env); // range HIT + await requestWithEnv(CHUNK_PATH, { headers: { Range: "bytes=0-9", ...origin } }, env); // range HIT expect(points.length).toBe(2); expect(points[1].doubles[0]).toBe(10); const fullPath = `/${PUBLIC_ID}/zarr/store.zarr/zarr.json`; - await requestWithEnv(fullPath, {}, env); // full-object MISS (primes the entry) + await requestWithEnv(fullPath, { headers: origin }, env); // full-object MISS (primes the entry) expect(points.length).toBe(3); expect(points[2].doubles[0]).toBe(ZARR_JSON.length); - await requestWithEnv(fullPath, {}, env); // full-object HIT + await requestWithEnv(fullPath, { headers: origin }, env); // full-object HIT expect(points.length).toBe(4); expect(points[3].doubles[0]).toBe(ZARR_JSON.length); }); @@ -1120,10 +1216,454 @@ describe("telemetry with a real binding-shaped collector (#1181 review item 15)" // env.ANALYTICS is left undefined by testEnv() -- recordAccess() no-ops // in that case (see services/access-metrics.ts). This just proves the // route doesn't throw or block on it, for both a fresh fetch and a - // cache hit. - const fresh = await request(`/${PUBLIC_ID}/zarr/store.zarr/zarr.json`); + // cache hit. An allowed Origin keeps this on the proxied path. + const headers = { Origin: BROWSER_ORIGIN }; + const fresh = await request(`/${PUBLIC_ID}/zarr/store.zarr/zarr.json`, { headers }); expect(fresh.status).toBe(200); - const hit = await request(`/${PUBLIC_ID}/zarr/store.zarr/zarr.json`); + const hit = await request(`/${PUBLIC_ID}/zarr/store.zarr/zarr.json`, { headers }); + expect(hit.status).toBe(200); + }); +}); + +// --------------------------------------------------------------------------- +// Phase 6 (#1181 phase 6 / issue #1061): redirect the bytes for non-browser +// clients instead of proxying them. The sections below extend this file's +// existing harness (real Bun.serve() upstream, real in-memory edge cache, +// real D1) rather than forking it -- `app`, `db`, `cache`, `upstream`, +// `request()`, `requestWithEnv()`, `requestWith()`, `testEnv()`, `testDeps()`, +// `countUpstream()`, and the fixture constants above are all shared. +// --------------------------------------------------------------------------- + +describe("routing fork stays correct at the edges (#1181 phase 6 / issue #1061)", () => { + test("a non-browser GET (no Origin) of a chunk redirects to the exact public S3 URL", async () => { + const res = await request(CHUNK_PATH); + expect(res.status).toBe(302); + expect(res.headers.get("location")).toBe(`http://localhost:${upstream.port}/${CHUNK_KEY}`); + expect(res.headers.get("cache-control")).toBe("public, max-age=3600"); + expect(res.headers.get("vary")).toBe("Origin"); + expect(res.headers.get("content-length")).toBeNull(); + expect(res.headers.get("access-control-allow-origin")).toBeNull(); + // Never touched the fake upstream -- serve() builds the Location string, + // it never fetches it. + expect(countUpstream(CHUNK_KEY, "")).toBe(0); + }); + + test("a non-allowlisted Origin also redirects", async () => { + const res = await request(CHUNK_PATH, { headers: { Origin: "https://evil.example.org" } }); + expect(res.status).toBe(302); + expect(res.headers.get("location")).toBe(`http://localhost:${upstream.port}/${CHUNK_KEY}`); + expect(res.headers.get("access-control-allow-origin")).toBeNull(); + }); + + test("an allowlisted Origin is still proxied with bytes + CORS, exactly as phase 1", async () => { + const res = await request(CHUNK_PATH, { headers: { Origin: "https://nemar.org" } }); + expect(res.status).toBe(200); + expect(res.headers.get("access-control-allow-origin")).toBe("https://nemar.org"); + const bytes = new Uint8Array(await res.arrayBuffer()); + expect(bytes).toEqual(CHUNK_BYTES); + expect(countUpstream(CHUNK_KEY, "")).toBe(1); + }); + + test("index.json with no Origin is still proxied, edge-cached, and D1-gated -- never redirected", async () => { + const res = await request(`/${PUBLIC_ID}/zarr/index.json`); + expect(res.status).toBe(200); + expect(res.headers.get("location")).toBeNull(); + const body = new Uint8Array(await res.arrayBuffer()); + expect(body).toEqual(INDEX_JSON); + expect(countUpstream(`${PUBLIC_ID}/zarr/index.json`, "")).toBe(1); + + // Cached on the second hit -- the proxied path, unchanged. + const hit = await request(`/${PUBLIC_ID}/zarr/index.json`); expect(hit.status).toBe(200); + expect(countUpstream(`${PUBLIC_ID}/zarr/index.json`, "")).toBe(1); + }); + + test("index.json on a private dataset is still 404 (D1-gated), never redirected", async () => { + // index.json is never a redirect candidate, so the private dataset stays + // protected by the D1 gate exactly as before phase 6. + const res = await request(`/${PRIVATE_ID}/zarr/index.json`); + expect(res.status).toBe(404); + expect(res.headers.get("location")).toBeNull(); + }); + + test("HEAD is never redirected, with or without Origin, and stays D1-gated", async () => { + const noOrigin = await request(`/${PUBLIC_ID}/zarr/store.zarr/zarr.json`, { method: "HEAD" }); + expect(noOrigin.status).toBe(200); + expect(noOrigin.headers.get("location")).toBeNull(); + + const withOrigin = await request(`/${PUBLIC_ID}/zarr/store.zarr/zarr.json`, { + method: "HEAD", + headers: { Origin: "https://evil.example.org" }, + }); + expect(withOrigin.status).toBe(200); + expect(withOrigin.headers.get("location")).toBeNull(); + + const privateHead = await request(`/${PRIVATE_ID}/zarr/store.zarr/zarr.json`, { + method: "HEAD", + }); + expect(privateHead.status).toBe(404); + }); +}); + +function countingDb(inner: D1Database): { db: D1Database; prepareCalls: () => number } { + let calls = 0; + const wrapped = { + ...inner, + prepare: (sql: string) => { + calls++; + return inner.prepare(sql); + }, + } as D1Database; + return { db: wrapped, prepareCalls: () => calls }; +} + +describe("no D1 read on the redirect path (#1181 phase 6 / issue #1061)", () => { + test("a redirect-candidate GET performs zero D1 prepare calls and zero upstream requests", async () => { + const { db: countedDb, prepareCalls } = countingDb(realD1(db)); + const env = { ...testEnv(), DB: countedDb } as Bindings; + + const res = await requestWithEnv(CHUNK_PATH, {}, env); + expect(res.status).toBe(302); + expect(prepareCalls()).toBe(0); + expect(requestLog.length).toBe(0); + }); + + test("still redirects (no D1 read) for a PRIVATE dataset's chunk -- the bucket's own deny-list is the real gate on this path, not D1", async () => { + const { db: countedDb, prepareCalls } = countingDb(realD1(db)); + const env = { ...testEnv(), DB: countedDb } as Bindings; + const path = `/${PRIVATE_ID}/zarr/store.zarr/zarr.json`; + + const res = await requestWithEnv(path, {}, env); + expect(res.status).toBe(302); + expect(res.headers.get("location")).toBe( + `http://localhost:${upstream.port}/${PRIVATE_ID}/zarr/store.zarr/zarr.json`, + ); + expect(prepareCalls()).toBe(0); + expect(requestLog.length).toBe(0); + }); + + test("a redirect candidate with a Range header still performs zero D1 calls", async () => { + const { db: countedDb, prepareCalls } = countingDb(realD1(db)); + const env = { ...testEnv(), DB: countedDb } as Bindings; + + const res = await requestWithEnv(CHUNK_PATH, { headers: { Range: "bytes=0-9" } }, env); + expect(res.status).toBe(302); + expect(prepareCalls()).toBe(0); + }); +}); + +describe("/zarrproxy path-mount entry point (#1181 phase 6)", () => { + test("redirects identically through the path-mounted entry point", async () => { + // Mirrors index.ts's `app.route("/zarrproxy", zarrDataRoutes)` -- Hono + // prepends the mount prefix without stripping it from c.req.path inside + // the sub-app, which is exactly what serve()'s prefix search and + // isRedirectCandidate's ZARR_OBJECT_PATH_RE are both written to tolerate. + const mounted = new Hono<{ Bindings: Bindings }>(); + mounted.route("/zarrproxy", app); + + const res = await requestWith(mounted, `/zarrproxy${CHUNK_PATH}`, {}, testEnv()); + expect(res.status).toBe(302); + expect(res.headers.get("location")).toBe(`http://localhost:${upstream.port}/${CHUNK_KEY}`); + expect(countUpstream(CHUNK_KEY, "")).toBe(0); + }); + + test("index.json still proxies through the path-mounted entry point", async () => { + const mounted = new Hono<{ Bindings: Bindings }>(); + mounted.route("/zarrproxy", app); + + const res = await requestWith( + mounted, + `/zarrproxy/${PUBLIC_ID}/zarr/index.json`, + {}, + testEnv(), + ); + expect(res.status).toBe(200); + expect(countUpstream(`${PUBLIC_ID}/zarr/index.json`, "")).toBe(1); + }); +}); + +describe("telemetry bytes on the redirect path (#1181 phase 6 / issue #1061)", () => { + function collector(): { env: Bindings; points: Array<{ blobs: string[]; doubles: number[] }> } { + const points: Array<{ blobs: string[]; doubles: number[] }> = []; + const env = { + ...testEnv(), + ANALYTICS: { + writeDataPoint: (p: { blobs: string[]; doubles: number[] }) => { + points.push(p); + }, + }, + } as unknown as Bindings; + return { env, points }; + } + + test("a bounded range (bytes=0-999) reports exactly 1000 bytes, never a 0 fallback", async () => { + const { env, points } = collector(); + const res = await requestWithEnv(CHUNK_PATH, { headers: { Range: "bytes=0-999" } }, env); + expect(res.status).toBe(302); + expect(points.length).toBe(1); + expect(points[0].doubles[0]).toBe(1000); + expect(points[0].blobs).toEqual([PUBLIC_ID, "zarr", "chunk-redirect"]); + }); + + test("a suffix range (bytes=-500) reports exactly 500 bytes", async () => { + const { env, points } = collector(); + await requestWithEnv(CHUNK_PATH, { headers: { Range: "bytes=-500" } }, env); + expect(points[0].doubles[0]).toBe(500); + }); + + test("an open-ended range (bytes=500-) reports 0 -- total size unknown without hitting S3", async () => { + const { env, points } = collector(); + await requestWithEnv(CHUNK_PATH, { headers: { Range: "bytes=500-" } }, env); + expect(points[0].doubles[0]).toBe(0); + }); + + test("no Range header reports 0", async () => { + const { env, points } = collector(); + await requestWithEnv(CHUNK_PATH, {}, env); + expect(points[0].doubles[0]).toBe(0); + }); + + test("a multi-range header reports 0 (not reducible to one bounded count)", async () => { + const { env, points } = collector(); + await requestWithEnv(CHUNK_PATH, { headers: { Range: "bytes=0-1,4-5" } }, env); + expect(points[0].doubles[0]).toBe(0); + }); +}); + +describe("isRedirectCandidate (#1181 phase 6 / issue #1061)", () => { + test("GET to a store object with no Origin is a candidate", () => { + expect(isRedirectCandidate("GET", "/on000001/zarr/store.zarr/c/0/0", null)).toBe(true); + }); + + test("GET with an allowlisted Origin is not a candidate", () => { + expect(isRedirectCandidate("GET", "/on000001/zarr/store.zarr/c/0/0", "https://nemar.org")).toBe( + false, + ); + }); + + test("GET with a non-allowlisted Origin is still a candidate", () => { + expect( + isRedirectCandidate("GET", "/on000001/zarr/store.zarr/c/0/0", "https://evil.example.org"), + ).toBe(true); + }); + + test("HEAD is never a candidate, Origin or not", () => { + expect(isRedirectCandidate("HEAD", "/on000001/zarr/store.zarr/c/0/0", null)).toBe(false); + expect( + isRedirectCandidate("HEAD", "/on000001/zarr/store.zarr/c/0/0", "https://evil.example.org"), + ).toBe(false); + }); + + test("OPTIONS is never a candidate", () => { + expect(isRedirectCandidate("OPTIONS", "/on000001/zarr/store.zarr/c/0/0", null)).toBe(false); + }); + + test("index.json is never a candidate", () => { + expect(isRedirectCandidate("GET", "/on000001/zarr/index.json", null)).toBe(false); + }); + + test("an empty or malformed rest is never a candidate", () => { + expect(isRedirectCandidate("GET", "/on000001/zarr/", null)).toBe(false); + expect(isRedirectCandidate("GET", "/on000001/zarr", null)).toBe(false); + expect(isRedirectCandidate("GET", "/on000001/zarr/..", null)).toBe(false); + }); + + test("the /zarrproxy path-mount shape is recognised identically", () => { + expect(isRedirectCandidate("GET", "/zarrproxy/on000001/zarr/store.zarr/c/0/0", null)).toBe( + true, + ); + expect(isRedirectCandidate("GET", "/zarrproxy/on000001/zarr/index.json", null)).toBe(false); + }); + + test("a path outside the zarr shape is never a candidate", () => { + expect(isRedirectCandidate("GET", "/datasets", null)).toBe(false); + }); + + test("malformed percent-encoding falls through to 'not a candidate' rather than throwing", () => { + expect(isRedirectCandidate("GET", "/on000001/zarr/%", null)).toBe(false); + }); +}); + +describe("bytesFromRangeHeader (#1181 phase 6 / issue #1061)", () => { + test("bounded range: B - A + 1", () => { + expect(bytesFromRangeHeader("bytes=0-999")).toBe(1000); + expect(bytesFromRangeHeader("bytes=1000-1999")).toBe(1000); + }); + + test("suffix range: N", () => { + expect(bytesFromRangeHeader("bytes=-500")).toBe(500); + }); + + test("open-ended range: 0 (unknown without hitting S3)", () => { + expect(bytesFromRangeHeader("bytes=1000-")).toBe(0); + }); + + test("no header: 0", () => { + expect(bytesFromRangeHeader(null)).toBe(0); + }); + + test("multi-range / malformed: 0", () => { + expect(bytesFromRangeHeader("bytes=0-1,4-5")).toBe(0); + expect(bytesFromRangeHeader("garbage")).toBe(0); + }); +}); + +// --------------------------------------------------------------------------- +// Rate-limiter exemption for redirect candidates (#1181 phase 6 / issue +// #1061). rateLimiter's own request counter reads/writes the REAL +// `caches.default` global directly -- a different cache from the +// deps.cache() seam this file otherwise uses for zarr-data.ts's OWN edge +// cache, an unrelated concern. bun:test has no `caches` global at all, and +// every describe block above sidesteps that by keeping ENVIRONMENT: +// "development" in testEnv(), which is rateLimiter's own documented bypass +// (see middleware/rateLimit.ts). This block is the one place that needs the +// REAL, non-bypassed rateLimiter code path, so it installs a real in-memory +// CacheStorage stand-in for its own duration only -- the same technique +// rate-limit-buckets.test.ts uses for the same reason, restored in afterAll +// so no other suite sharing this process (root `bun test` runs test/ and +// backend/test/ together, see AGENTS.md) sees a stray global. +// --------------------------------------------------------------------------- + +class RateLimitCache implements Cache { + private store = new Map }>(); + + async match(req: RequestInfo | URL): Promise { + const entry = this.store.get(keyFor(req)); + return entry ? new Response(entry.body, { headers: entry.headers }) : undefined; + } + + async put(req: RequestInfo | URL, res: Response): Promise { + const headers: Record = {}; + res.headers.forEach((v, k) => { + headers[k] = v; + }); + this.store.set(keyFor(req), { body: await res.text(), headers }); + } + + async delete(): Promise { + return false; + } + async add(): Promise { + throw new Error("not implemented"); + } + async addAll(): Promise { + throw new Error("not implemented"); + } + async keys(): Promise { + return []; + } + async matchAll(): Promise { + return []; + } + + clear(): void { + this.store.clear(); + } + + /** Read back the counter rateLimiter itself wrote (`{"count": N}`), or + * undefined if this bucket key was never touched at all -- the strongest + * proof that a redirect candidate never reached the enforcement code. */ + count(url: string): number | undefined { + const entry = this.store.get(url); + return entry ? (JSON.parse(entry.body) as { count: number }).count : undefined; + } + + /** Seed a bucket directly at the real cap boundary, in the exact shape + * rateLimiter writes, so a test can prove enforcement AT the production + * DATA_MAX_REQUESTS cap without looping thousands of real requests + * through the full D1/cache-backed app. */ + seedCount(url: string, count: number): void { + this.store.set(url, { + body: JSON.stringify({ count }), + headers: { "Cache-Control": `max-age=${__limits.WINDOW_SIZE}` }, + }); + } +} + +describe("rate limiter exemption for redirect candidates (#1181 phase 6 / issue #1061)", () => { + const rlCache = new RateLimitCache(); + // biome-ignore lint/suspicious/noExplicitAny: test-only runtime patch, mirrors rate-limit-buckets.test.ts + let originalCaches: any; + + beforeAll(() => { + // biome-ignore lint/suspicious/noExplicitAny: test-only runtime patch + originalCaches = (globalThis as any).caches; + // biome-ignore lint/suspicious/noExplicitAny: test-only runtime patch + (globalThis as any).caches = { default: rlCache } as unknown as CacheStorage; + }); + + afterAll(() => { + // biome-ignore lint/suspicious/noExplicitAny: test-only runtime restore + (globalThis as any).caches = originalCaches; + }); + + beforeEach(() => { + rlCache.clear(); + }); + + // ENVIRONMENT must NOT be "development" here -- that's rateLimiter's own + // bypass, and this block exists specifically to exercise the real, + // non-bypassed code path. Computed lazily (per call, not per describe) so + // it always reads the CURRENT `db` the outer beforeEach just rebuilt. + function rlEnv(): Bindings { + return { ...testEnv(), ENVIRONMENT: "production" } as Bindings; + } + + test("a redirect candidate never touches the data-ip counter, no matter the IP or volume", async () => { + const ip = "203.0.113.10"; + for (let i = 0; i < 5; i++) { + const res = await requestWith( + app, + CHUNK_PATH, + { headers: { "CF-Connecting-IP": ip } }, + rlEnv(), + ); + expect(res.status).toBe(302); + } + // Never created -- proves the middleware skipped rateLimiter entirely + // for these hits, not just that they landed under a generous cap. + expect(rlCache.count(`https://rate-limit.internal/rl:data-ip:${ip}`)).toBeUndefined(); + }); + + test("a redirect candidate still succeeds even once the SAME IP's data-ip bucket is fully exhausted", async () => { + const ip = "203.0.113.11"; + rlCache.seedCount(`https://rate-limit.internal/rl:data-ip:${ip}`, __limits.DATA_MAX_REQUESTS); + + const res = await requestWith( + app, + CHUNK_PATH, + { headers: { "CF-Connecting-IP": ip } }, + rlEnv(), + ); + expect(res.status).toBe(302); + }); + + test("a proxied GET (index.json, no Origin) DOES charge the data-ip bucket and 429s at the real cap", async () => { + const ip = "203.0.113.12"; + const key = `https://rate-limit.internal/rl:data-ip:${ip}`; + rlCache.seedCount(key, __limits.DATA_MAX_REQUESTS - 1); + + const path = `/${PUBLIC_ID}/zarr/index.json`; + const first = await requestWith(app, path, { headers: { "CF-Connecting-IP": ip } }, rlEnv()); + expect(first.status).toBe(200); + expect(rlCache.count(key)).toBe(__limits.DATA_MAX_REQUESTS); + + const second = await requestWith(app, path, { headers: { "CF-Connecting-IP": ip } }, rlEnv()); + expect(second.status).toBe(429); + }); + + test("an allowlisted-Origin GET for a store chunk (proxied, not redirected) also charges the bucket", async () => { + const ip = "203.0.113.13"; + const key = `https://rate-limit.internal/rl:data-ip:${ip}`; + rlCache.seedCount(key, __limits.DATA_MAX_REQUESTS - 1); + + const res = await requestWith( + app, + CHUNK_PATH, + { headers: { "CF-Connecting-IP": ip, Origin: "https://nemar.org" } }, + rlEnv(), + ); + expect(res.status).toBe(200); + expect(rlCache.count(key)).toBe(__limits.DATA_MAX_REQUESTS); }); }); From da6c8f15aa7a8545e21eaf3939efa1d1e7e8d5d1 Mon Sep 17 00:00:00 2001 From: Seyed Yahya Shirazi Date: Wed, 2 Sep 2026 10:58:38 -0700 Subject: [PATCH 2/9] fix: encode S3 key segments in s3PublicUrl s3PublicUrl interpolated the decoded key unescaped, so a literal '#' truncated the redirect Location at the fragment, '?'/space broke the request line, and a non-Latin-1 character made workerd refuse to send the header at all (Bun tolerated it, so tests stayed green). It now shares canonicalCacheUrl's per-segment encodeURIComponent treatment via a new encodeS3KeyPath() helper, so the fetch target, the cache key, and the Location all agree. Exported s3PublicUrl for a direct unit test of its real (non-s3Base) branch. PR #1202 review item 1. --- backend/src/routes/zarr-data.ts | 43 +++++++++-- backend/test/zarr-data-cache.test.ts | 108 ++++++++++++++++++++++++++- 2 files changed, 142 insertions(+), 9 deletions(-) diff --git a/backend/src/routes/zarr-data.ts b/backend/src/routes/zarr-data.ts index 48d0b09dc..dc6f10345 100644 --- a/backend/src/routes/zarr-data.ts +++ b/backend/src/routes/zarr-data.ts @@ -156,13 +156,24 @@ export function bytesFromRangeHeader(rangeHeader: string | null): number { return 0; // open-ended `bytes=A-`: length unknown without hitting S3 } -function s3PublicUrl(env: Bindings, key: string, base?: string): string { +/** + * Build the public S3 URL for a key -- the upstream fetch target for the + * proxied branch AND the redirect `Location` for the redirect branch + * (#1181 phase 6). `key` is the already-decoded, normalised S3 key (see + * serve()'s `rest`), so it is re-encoded per path segment here via + * encodeS3KeyPath() -- the SAME encoding canonicalCacheUrl() uses for the + * edge-cache key -- rather than interpolated raw. Exported for a direct + * unit test of the real (non-`base`) branch; production code never imports + * it from outside this file. + */ +export function s3PublicUrl(env: Bindings, key: string, base?: string): string { + const encodedKey = encodeS3KeyPath(key); // Test seam (#1178 phase 1): a real Bun.serve() upstream stands in for S3 // in tests via deps.s3Base, instead of parsing env vars for a fake host. - if (base) return `${base}/${key}`; + if (base) return `${base}/${encodedKey}`; // Public-read object (deny-list policy makes a public dataset's prefix public). const region = env.AWS_REGION || "us-east-2"; - return `https://${env.S3_BUCKET}.s3.${region}.amazonaws.com/${key}`; + return `https://${env.S3_BUCKET}.s3.${region}.amazonaws.com/${encodedKey}`; } /** Cache-Control for the origin + edge. @@ -240,6 +251,26 @@ export type NormalizedRange = string & { readonly __brand: "NormalizedRange" }; * pathname. */ export type CanonicalCacheUrl = string & { readonly __brand: "CanonicalCacheUrl" }; +/** + * Percent-encode each path segment of an S3 key independently (never the + * whole key as one string, which would also escape the `/` separators). + * + * Shared by canonicalCacheUrl() (the edge-cache key) and s3PublicUrl() (the + * upstream fetch target AND the redirect Location, #1181 phase 6 review) so + * all three agree on the exact same encoded spelling for a given key. Before + * this was shared, s3PublicUrl() interpolated the raw (decoded) key + * unescaped: a literal `#` truncated the URL at the fragment, `?`/space + * broke the request line, and a non-Latin-1 character landed raw in a + * `Location` header, which workerd (unlike Bun, where tests ran) refuses to + * send at all. + */ +function encodeS3KeyPath(key: string): string { + return key + .split("/") + .map((segment) => encodeURIComponent(segment)) + .join("/"); +} + /** * Canonical edge-cache key for an object: origin + "/" + the normalised S3 * key, with each path segment individually `encodeURIComponent`'d, plus the @@ -257,11 +288,7 @@ export function canonicalCacheUrl( key: string, v: string | null, ): CanonicalCacheUrl { - const encodedPath = key - .split("/") - .map((segment) => encodeURIComponent(segment)) - .join("/"); - const url = new URL(`${origin}/${encodedPath}`); + const url = new URL(`${origin}/${encodeS3KeyPath(key)}`); if (v) url.searchParams.set("v", v); return url.toString() as CanonicalCacheUrl; } diff --git a/backend/test/zarr-data-cache.test.ts b/backend/test/zarr-data-cache.test.ts index 662aaa1e6..4e9b4eaf4 100644 --- a/backend/test/zarr-data-cache.test.ts +++ b/backend/test/zarr-data-cache.test.ts @@ -54,6 +54,7 @@ import { createZarrDataRoutes, isRedirectCandidate, parseCacheableRange, + s3PublicUrl, } from "../src/routes/zarr-data"; import type { Bindings } from "../src/types/bindings"; import { freshDb, realD1 } from "./helpers/d1"; @@ -77,6 +78,17 @@ const QUIRK_BYTES = new Uint8Array(64).fill(3); // exceeds RANGE_CACHE_MAX_BYTES. Real bytes, real transfer -- not mocked. const CAP_TEST_BYTES = new Uint8Array(FULL_OBJECT_CACHE_MAX_BYTES + 1024).fill(9); +// A store-object name with a space, '#', '?', '%', '+', and a non-ASCII +// character -- every character class the encoding fix (#1181 phase 6 +// review item 1) has to survive. Registered in OBJECTS under its RAW +// (decoded) spelling, exactly like every other fixture here; the fake +// upstream below decodes the incoming request path before looking it up, +// mirroring how S3 itself resolves a percent-encoded request path back to +// the raw object key. +const WEIRD_NAME = "weird #?%+café.bin"; +const WEIRD_KEY = `${PUBLIC_ID}/zarr/store.zarr/${WEIRD_NAME}`; +const WEIRD_BYTES = new Uint8Array([9, 8, 7, 6, 5]); + const OBJECTS: Record = { [`${PUBLIC_ID}/zarr/index.json`]: INDEX_JSON, [`${PUBLIC_ID}/zarr/store.zarr/zarr.json`]: ZARR_JSON, @@ -85,6 +97,7 @@ const OBJECTS: Record = { [`${PUBLIC_ID}/zarr/store.zarr/c/quirk/no-length-header`]: QUIRK_BYTES, [`${PUBLIC_ID}/zarr/store.zarr/c/quirk/ignored-range`]: QUIRK_BYTES, [`${PUBLIC_ID}/zarr/store.zarr/c/big/oversized`]: CAP_TEST_BYTES, + [WEIRD_KEY]: WEIRD_BYTES, [`${PRIVATE_ID}/zarr/store.zarr/zarr.json`]: ZARR_JSON, }; @@ -134,15 +147,39 @@ function countUpstream(path: string, range = ""): number { return requestLog.filter((r) => r.path === path && r.range === range).length; } +/** Same as countUpstream, but also asserts the HTTP method (#1181 phase 6 + * review item 9) -- countUpstream alone can't distinguish "the upstream + * never saw this key/range" from "it saw it, but as the wrong method" + * (e.g. a HEAD probe that accidentally streamed a GET's full body). */ +function countUpstreamByMethod(method: string, path: string, range = ""): number { + return requestLog.filter((r) => r.method === method && r.path === path && r.range === range) + .length; +} + beforeAll(() => { upstream = Bun.serve({ port: 0, fetch(req) { const url = new URL(req.url); - const key = url.pathname.slice(1); + // Decode the incoming (percent-encoded, as a real client would send + // it) request path back to the raw key -- mirrors how S3 itself + // resolves a percent-encoded request path to the raw object key, and + // lets OBJECTS/QUIRKS stay keyed by the natural/raw spelling like + // every other fixture here (#1181 phase 6 review item 1). + const key = decodeURIComponent(url.pathname.slice(1)); const range = req.headers.get("range") ?? ""; requestLog.push({ method: req.method, path: key, range }); + // Simulate the bucket's own NotResource deny-list (bucket-policy.ts): + // a private dataset's ENTIRE prefix is walled off from anonymous + // access at the bucket level, independent of whether the specific key + // exists -- this is the actual safety argument for the redirect + // branch skipping the D1 gate (#1181 phase 6 review item 10; see the + // "private dataset redirect safety" describe block below). + if (key.startsWith(`${PRIVATE_ID}/`)) { + return new Response(null, { status: 403 }); + } + const bytes = OBJECTS[key]; if (!bytes) return new Response(null, { status: 404 }); const etag = `"${key}"`; @@ -426,6 +463,28 @@ describe("canonicalCacheUrl", () => { }); }); +describe("s3PublicUrl (#1181 phase 6 review item 1)", () => { + test("the real (non-s3Base) branch builds a per-segment-encoded AWS URL", () => { + const env = { AWS_REGION: "us-east-2", S3_BUCKET: "nemar" } as unknown as Bindings; + expect(s3PublicUrl(env, "on000001/zarr/store.zarr/weird #?%+café.bin")).toBe( + "https://nemar.s3.us-east-2.amazonaws.com/on000001/zarr/store.zarr/weird%20%23%3F%25%2Bcaf%C3%A9.bin", + ); + }); + + test("the real branch defaults AWS_REGION when unset", () => { + const env = { S3_BUCKET: "nemar" } as unknown as Bindings; + expect(s3PublicUrl(env, "on000001/zarr/index.json")).toBe( + "https://nemar.s3.us-east-2.amazonaws.com/on000001/zarr/index.json", + ); + }); + + test("the s3Base (test-seam) branch also encodes, agreeing with the real branch", () => { + expect(s3PublicUrl({} as Bindings, "on000001/zarr/a b/c#d.bin", "http://localhost:1234")).toBe( + "http://localhost:1234/on000001/zarr/a%20b/c%23d.bin", + ); + }); +}); + describe("parseCacheableRange", () => { test("accepts bytes=A-B", () => { expect(parseCacheableRange("bytes=0-999")).toBe("bytes=0-999"); @@ -1303,6 +1362,53 @@ describe("routing fork stays correct at the edges (#1181 phase 6 / issue #1061)" }); expect(privateHead.status).toBe(404); }); + + test("a literal 'null' Origin header on a store object still redirects with no ACAO (#1181 phase 6 review item 11)", async () => { + // `Origin: null` (the literal 4-char string, sent by some sandboxed + // browsing contexts) is not a URL allowedOrigin() can parse, so it must + // fall through to "not allowlisted" exactly like any other unparseable + // or unrecognised origin -- never crash, never get treated as trusted. + const res = await request(CHUNK_PATH, { headers: { Origin: "null" } }); + expect(res.status).toBe(302); + expect(res.headers.get("access-control-allow-origin")).toBeNull(); + }); + + test("HEAD with a non-allowlisted Origin on a private dataset is 404, never redirected (#1181 phase 6 review item 11)", async () => { + const res = await request(`/${PRIVATE_ID}/zarr/store.zarr/zarr.json`, { + method: "HEAD", + headers: { Origin: "https://evil.example.org" }, + }); + expect(res.status).toBe(404); + expect(res.headers.get("location")).toBeNull(); + }); +}); + +describe("redirect Location and the proxied fetch target are both correctly encoded (#1181 phase 6 review item 1)", () => { + const WEIRD_REQUEST_PATH = `/${PUBLIC_ID}/zarr/store.zarr/${encodeURIComponent(WEIRD_NAME)}`; + const EXPECTED_ENCODED_KEY = WEIRD_KEY.split("/").map(encodeURIComponent).join("/"); + + test("the redirect branch: Location carries the exact per-segment-encoded key", async () => { + const res = await request(WEIRD_REQUEST_PATH); + expect(res.status).toBe(302); + expect(res.headers.get("location")).toBe( + `http://localhost:${upstream.port}/${EXPECTED_ENCODED_KEY}`, + ); + // The fake upstream never saw this request at all (redirect never + // fetches) -- confirms the Location is built, not derived from a probe. + expect(countUpstream(WEIRD_KEY, "")).toBe(0); + }); + + test("the proxied branch: the fake upstream receives the encoded path and serves the object", async () => { + const res = await request(WEIRD_REQUEST_PATH, { headers: { Origin: BROWSER_ORIGIN } }); + expect(res.status).toBe(200); + const bytes = new Uint8Array(await res.arrayBuffer()); + expect(bytes).toEqual(WEIRD_BYTES); + // countUpstream is keyed on the DECODED key the fake server resolves the + // encoded request path back to -- a hit here proves the wire request + // really carried the encoded spelling and the server correctly reversed + // it back to the exact same key serve() started from. + expect(countUpstream(WEIRD_KEY, "")).toBe(1); + }); }); function countingDb(inner: D1Database): { db: D1Database; prepareCalls: () => number } { From 8c2bde1abf9b44fa54dab09e4e4d9f8559aaaa92 Mon Sep 17 00:00:00 2001 From: Seyed Yahya Shirazi Date: Wed, 2 Sep 2026 11:00:45 -0700 Subject: [PATCH 3/9] fix: anchor redirect path regex and validate dataset id ZARR_OBJECT_PATH_RE's \/zarr\/?(.*)$ matched "bogus" as rest for /on000001/zarrbogus, exempting a path that never reaches serve()'s route from the rate limiter. Anchored on \/zarr(?:\/(.*))? like ZARR_PATH_RE, requiring end-of-string or a literal slash after zarr. isRedirectCandidate never validated the dataset id either, so zz000001 (wrong prefix), nm1234567 (too many digits), and nm999999 (over the 99999 cap) got redirected to a garbage S3 URL and wrote an invalid id into Analytics Engine, while the same request with a browser Origin correctly 404'd. The regex now captures the id and isValidDatasetId gates it; a malformed id falls through to the normal proxied 404 path instead of a separate D1-free branch. Also drops the stale "catalog.json will need the same exemption" comment: catalog.json is a separate route with no /zarr/ segment and can never match this regex at all. PR #1202 review items 2, 3, 7 (comment only; item 7's tests follow). --- backend/src/routes/zarr-data.ts | 37 ++++++++++++++++++++++------ backend/test/zarr-data-cache.test.ts | 34 +++++++++++++++++++++++++ 2 files changed, 64 insertions(+), 7 deletions(-) diff --git a/backend/src/routes/zarr-data.ts b/backend/src/routes/zarr-data.ts index dc6f10345..f6ddf81bf 100644 --- a/backend/src/routes/zarr-data.ts +++ b/backend/src/routes/zarr-data.ts @@ -89,20 +89,39 @@ export function corsHeaders(origin: string | null): Record { * Path shape for a zarr object request, tolerant of both entry points: * `//zarr/` (zarr.nemar.org host fork) * `/zarrproxy//zarr/` (path-mounted on api.nemar.org / workers.dev) + * Anchored on the `zarr` segment itself -- `(?:\/(.*))?$` requires either + * end-of-string right after `zarr` or a literal `/` before any rest, so + * `/on000001/zarrbogus` or `/on000001/zarr-x` cannot match (#1181 phase 6 + * review item 2: the old `\/zarr\/?(.*)$` treated "bogus" as `rest` for + * `/on000001/zarrbogus`, misclassifying it as a redirect candidate that + * never actually reaches serve()'s `/:datasetId/zarr/*` route at all). * Mirrors ZARR_PATH_RE in middleware/rateLimit.ts (kept as an independent * literal rather than imported from there -- this file already imports * `rateLimiter` FROM rateLimit.ts, so importing the other direction too - * would create a cycle). + * would create a cycle). The id is captured too (group 1) so + * isRedirectCandidate can validate it below -- the path shape alone + * (`[a-z]{2}\d+`) doesn't enforce the real id contract (exactly 6 digits, + * an `nm`/`xx`/`on` prefix, `<= 99999`; see isValidDatasetId). */ -const ZARR_OBJECT_PATH_RE = /^(?:\/zarrproxy)?\/[a-z]{2}\d+\/zarr\/?(.*)$/; +const ZARR_OBJECT_PATH_RE = /^(?:\/zarrproxy)?\/([a-z]{2}\d+)\/zarr(?:\/(.*))?$/; /** * True when a request to this sub-app WILL take the redirect branch in * serve() below, rather than being proxied: a `GET` (never `HEAD` -- see * the module doc comment, that rule is load-bearing) for a store object -- * not the always-proxied `index.json`, and not an empty/malformed path, - * which the proxied branch 404s either way -- with no allowlisted browser - * `Origin`. + * which the proxied branch 404s either way -- on a VALID dataset id, with + * no allowlisted browser `Origin`. + * + * The id check matters on its own (#1181 phase 6 review item 3): without + * it, `zz000001` (wrong prefix), `nm1234567` (too many digits), and + * `nm999999` (over the 99999 cap) all satisfied the old path-shape-only + * regex and got redirected to a garbage S3 URL -- with `recordAccess` + * writing an invalid dataset id into Analytics Engine -- while the exact + * same request with a browser Origin correctly 404'd via serve()'s + * isPublicDataset -> isValidDatasetId gate. A malformed id must fall + * through to that SAME 404 path regardless of Origin, not take a + * different, D1-free branch of its own. * * Called from exactly two places: serve()'s own routing decision, and the * rate-limit middleware below that exempts these hits from the data-ip @@ -117,9 +136,11 @@ export function isRedirectCandidate(method: string, path: string, origin: string if (allowedOrigin(origin)) return false; const match = ZARR_OBJECT_PATH_RE.exec(path); if (!match) return false; + const [, datasetId, rawRest] = match; + if (!isValidDatasetId(datasetId)) return false; let rest: string | null; try { - rest = normalizeBidsPath(decodeURIComponent(match[1])); + rest = normalizeBidsPath(decodeURIComponent(rawRest ?? "")); } catch { // Malformed percent-encoding: fall through to the proxied path, which // hits the same decodeURIComponent call and 500s via onError -- the @@ -128,8 +149,10 @@ export function isRedirectCandidate(method: string, path: string, origin: string } // index.json is always top-level (the producer never nests a store under // that name) and stays proxied/edge-cached/D1-gated -- the contract entry - // point (#1061). Phase 2's catalog.json will need the same exemption once - // it exists. + // point (#1061). catalog.json (phase 2, #1062) is a SEPARATE route + // (`GET /catalog.json`, no `/zarr/` segment at all) and can never + // match ZARR_OBJECT_PATH_RE in the first place, so it needs no exemption + // here -- see the "catalog.json can never match" unit tests. return Boolean(rest) && rest !== "index.json"; } diff --git a/backend/test/zarr-data-cache.test.ts b/backend/test/zarr-data-cache.test.ts index 4e9b4eaf4..f9fd73582 100644 --- a/backend/test/zarr-data-cache.test.ts +++ b/backend/test/zarr-data-cache.test.ts @@ -1411,6 +1411,20 @@ describe("redirect Location and the proxied fetch target are both correctly enco }); }); +describe("invalid dataset ids never redirect, even with a plausible path shape (#1181 phase 6 review item 3)", () => { + for (const id of ["zz000001", "nm1234567", "nm999999"]) { + test(`${id} with no Origin falls through to the normal 404, not a redirect to a garbage URL`, async () => { + const res = await request(`/${id}/zarr/store.zarr/c/0/0`); + expect(res.status).toBe(404); + expect(res.headers.get("location")).toBeNull(); + // isPublicDataset short-circuits on isValidDatasetId before ever + // reading D1 for a shape this obviously invalid -- same as it always + // did for these ids on the proxied path. + expect(countUpstream(`${id}/zarr/store.zarr/c/0/0`, "")).toBe(0); + }); + } +}); + function countingDb(inner: D1Database): { db: D1Database; prepareCalls: () => number } { let calls = 0; const wrapped = { @@ -1588,6 +1602,26 @@ describe("isRedirectCandidate (#1181 phase 6 / issue #1061)", () => { test("malformed percent-encoding falls through to 'not a candidate' rather than throwing", () => { expect(isRedirectCandidate("GET", "/on000001/zarr/%", null)).toBe(false); }); + + test("'zarr' as a prefix of a longer segment is not a candidate (#1181 phase 6 review item 2)", () => { + // The old `\/zarr\/?(.*)$` pattern matched "bogus" as `rest` here -- + // neither shape reaches serve()'s `/:datasetId/zarr/*` route at all. + expect(isRedirectCandidate("GET", "/on000001/zarrbogus", null)).toBe(false); + expect(isRedirectCandidate("GET", "/on000001/zarr-x", null)).toBe(false); + }); + + test("an invalid dataset id is never a candidate, even with a plausible path shape (#1181 phase 6 review item 3)", () => { + expect(isRedirectCandidate("GET", "/zz000001/zarr/store.zarr/c/0/0", null)).toBe(false); // wrong prefix + expect(isRedirectCandidate("GET", "/nm1234567/zarr/store.zarr/c/0/0", null)).toBe(false); // too many digits + expect(isRedirectCandidate("GET", "/nm999999/zarr/store.zarr/c/0/0", null)).toBe(false); // over the 99999 cap + }); + + test("catalog.json can never match the zarr object path shape, for any Origin (#1181 phase 6 review item 7)", () => { + for (const origin of [null, "https://nemar.org", "https://evil.example.org"]) { + expect(isRedirectCandidate("GET", "/catalog.json", origin)).toBe(false); + expect(isRedirectCandidate("GET", "/zarrproxy/catalog.json", origin)).toBe(false); + } + }); }); describe("bytesFromRangeHeader (#1181 phase 6 / issue #1061)", () => { From 9f6a4e32c4bf92eb1535c53f3abce1045489dae0 Mon Sep 17 00:00:00 2001 From: Seyed Yahya Shirazi Date: Wed, 2 Sep 2026 11:01:42 -0700 Subject: [PATCH 4/9] fix: bytesFromRangeHeader never returns negative bytes "bytes=100-50" is syntactically a valid single range (parseCacheableRange only checks digit shape, not end >= start), so it computed -49 and that landed in the Analytics Engine data point. Real S3 would 416 an inverted range like any other out-of-range request, so there is never a real byte count for one; fall back to the same "unknown" 0 as every other unmeasurable case. PR #1202 review item 4. --- backend/src/routes/zarr-data.ts | 13 ++++++++++++- backend/test/zarr-data-cache.test.ts | 5 +++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/backend/src/routes/zarr-data.ts b/backend/src/routes/zarr-data.ts index f6ddf81bf..1c783e87e 100644 --- a/backend/src/routes/zarr-data.ts +++ b/backend/src/routes/zarr-data.ts @@ -173,7 +173,18 @@ export function bytesFromRangeHeader(rangeHeader: string | null): number { const normalized = parseCacheableRange(rangeHeader); if (!normalized) return 0; const bounded = /^bytes=(\d+)-(\d+)$/.exec(normalized); - if (bounded) return Number(bounded[2]) - Number(bounded[1]) + 1; + if (bounded) { + // parseCacheableRange only checks the digits-A-digits SHAPE, not that + // end >= start -- "bytes=100-50" is syntactically a valid single range + // and would otherwise compute -49 here, landing a negative byte count + // in Analytics Engine (#1181 phase 6 review item 4). Real S3 answers an + // inverted range with 416, same as any other out-of-range request, so + // there is never a real number of bytes served for one -- fall back to + // the same "unknown" 0 as every other unmeasurable case here. + const start = Number(bounded[1]); + const end = Number(bounded[2]); + return end >= start ? end - start + 1 : 0; + } const suffix = /^bytes=-(\d+)$/.exec(normalized); if (suffix) return Number(suffix[1]); return 0; // open-ended `bytes=A-`: length unknown without hitting S3 diff --git a/backend/test/zarr-data-cache.test.ts b/backend/test/zarr-data-cache.test.ts index f9fd73582..847f656e5 100644 --- a/backend/test/zarr-data-cache.test.ts +++ b/backend/test/zarr-data-cache.test.ts @@ -1642,6 +1642,11 @@ describe("bytesFromRangeHeader (#1181 phase 6 / issue #1061)", () => { expect(bytesFromRangeHeader(null)).toBe(0); }); + test("an inverted range (end < start) returns 0, never a negative number (#1181 phase 6 review item 4)", () => { + expect(bytesFromRangeHeader("bytes=100-50")).toBe(0); + expect(bytesFromRangeHeader("bytes=1-0")).toBe(0); + }); + test("multi-range / malformed: 0", () => { expect(bytesFromRangeHeader("bytes=0-1,4-5")).toBe(0); expect(bytesFromRangeHeader("garbage")).toBe(0); From 94e26c71cd2a6f0539f591ca23e4701f79ea6f36 Mon Sep 17 00:00:00 2001 From: Seyed Yahya Shirazi Date: Wed, 2 Sep 2026 11:09:36 -0700 Subject: [PATCH 5/9] feat: observe-only rate-limit counting for zarr redirects Replaces the unconditional per-request console.info the zarr redirect exemption used to emit with an observe-only mode on rateLimiter itself (a new RateLimiterOptions.observeOnly). A redirect candidate now counts toward the SAME IP-keyed data-ip bucket a proxied request from that client would be enforced against, but never blocks -- so a later proxied request from a client that's been hammering the bucket via redirects still 429s correctly. Logs at most one console.warn per window, the first time the bucket crosses the point enforcement would have tripped, with a hash of the IP (not the raw address) rather than a log line per exempted request. Updates AGENTS.md's zarr paragraph to match, and corrects a related inaccuracy: catalog.json is proxied and edge-cached but has no D1 gate of its own (it lists only already-public datasets), unlike index.json. PR #1202 review item 5. --- AGENTS.md | 29 ++++--- backend/src/middleware/rateLimit.ts | 65 +++++++++++++++- backend/src/routes/zarr-data.ts | 27 ++++--- backend/test/zarr-data-cache.test.ts | 111 +++++++++++++++++++++------ 4 files changed, 185 insertions(+), 47 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 97b8fddf2..94aa77967 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -311,19 +311,26 @@ Environments and pre-release checks: [`.context/release-safety-playbook.md`](.co DOIs are permanent and require explicit confirmation. - **Zarr.** A derived, latest-only serving copy, not a source of truth. `zarr.nemar.org` is the stable contract for it (issue #1061, epic #1181 phase 6): - `index.json` (and phase 2's `catalog.json`) stays proxied, edge-cached, and D1-gated, - but a plain `GET` for a store object with no allowlisted browser `Origin` — libraries, - HPC jobs, agents — 302s straight to the public S3 object instead of streaming through - the Worker, so every request is still counted without this Worker carrying the bytes - (Cloudflare's terms restrict proxying large files at this scale on a non-Enterprise - plan). `HEAD` is never redirected regardless of Origin: fsspec's `info()` and rclone's - sync both probe with HEAD, and rclone's HTTP backend does not follow HEAD redirects. - The redirect branch skips the D1 visibility gate entirely — the bucket's own + `index.json` stays proxied, edge-cached, and D1-gated. Phase 2's `catalog.json` is a + SEPARATE route (no `/zarr/` segment, so it can never match the redirect predicate + at all) that stays proxied and edge-cached too, but has no D1 gate of its own — its + source document only ever lists already-public datasets. A plain `GET` for a store + object with no allowlisted browser `Origin` — libraries, HPC jobs, agents — 302s + straight to the public S3 object instead of streaming through the Worker, so every + request is still counted without this Worker carrying the bytes (Cloudflare's terms + restrict proxying large files at this scale on a non-Enterprise plan). `HEAD` is + never redirected regardless of Origin: fsspec's `info()` and rclone's sync both probe + with HEAD, and rclone's HTTP backend does not follow HEAD redirects. The redirect + branch skips the D1 visibility gate entirely, and validates the dataset id itself + (`isValidDatasetId`) rather than trusting the path shape — the bucket's own `NotResource` deny-list (`backend/src/services/bucket-policy.ts`) is the real enforcement point, so a redirect that 403s at S3 for a private dataset leaks nothing - the proxied 404 does not — and the rate limiter exempts these hits from the data-ip - bucket rather than throttling traffic that never touches D1, the edge cache, or S3 - through the Worker. + the proxied 404 does not. The rate limiter counts these hits toward the SAME shared + data-ip bucket a proxied request from that IP would be enforced against, without ever + blocking the redirect itself (`rateLimiter`'s `observeOnly` option, + `backend/src/middleware/rateLimit.ts`) — it logs at most one `console.warn` per + window, the first time the bucket crosses the point enforcement would have tripped, + never a per-request log. The converter is `scripts/zarr/` **in this repo** and runs on the SDSC Hallu cron (`scripts/zarr/hallu-zarr.sh`, hourly at `:30`), never in GitHub Actions — Actions cannot finish a large dataset inside the 120-minute cap (ADR 0029). diff --git a/backend/src/middleware/rateLimit.ts b/backend/src/middleware/rateLimit.ts index 3c7e1b656..c409622bc 100644 --- a/backend/src/middleware/rateLimit.ts +++ b/backend/src/middleware/rateLimit.ts @@ -242,7 +242,31 @@ async function isPrivilegedToken(env: Bindings, hashedApiKey: string): Promise= maxRequests && !warned) { + // Hashed, not raw: this is a log line, not the enforcement key + // itself (which stays keyed on the raw IP, same as before) -- + // avoids putting a raw client IP into structured logs. + const ipHash = await hashApiKey(rawKey); + console.warn("[rate-limit] observe-only bucket would have tripped", { + ipHash, + path, + count: count + 1, + }); + warned = true; + } + const newCount = count + 1; + await cache.put( + cacheKey, + new Response(JSON.stringify({ count: newCount, warned }), { + headers: { + "Content-Type": "application/json", + "Cache-Control": `max-age=${WINDOW_SIZE}`, + }, + }), + ); + await next(); + return; } if (count >= maxRequests) { @@ -322,7 +379,7 @@ export async function rateLimiter(c: RateLimitContext, next: Next) { // Increment counter and store in cache with TTL const newCount = count + 1; - const response = new Response(JSON.stringify({ count: newCount }), { + const response = new Response(JSON.stringify({ count: newCount, warned }), { headers: { "Content-Type": "application/json", "Cache-Control": `max-age=${WINDOW_SIZE}`, diff --git a/backend/src/routes/zarr-data.ts b/backend/src/routes/zarr-data.ts index 1c783e87e..0f245c6b1 100644 --- a/backend/src/routes/zarr-data.ts +++ b/backend/src/routes/zarr-data.ts @@ -810,15 +810,24 @@ export function createZarrDataRoutes( // millisecond of CPU and zero bytes through this Worker -- the IP-keyed // data-ip bucket exists to bound a runaway PROXIED loop, not to punish a // cluster behind one NAT address for traffic that never touches D1, the - // edge cache, or S3 through here. Exempt them and log a counter instead - // of enforcing, per the issue. Uses the SAME predicate serve() uses to - // pick its response branch (see isRedirectCandidate's doc comment), so - // this can never exempt a request that serve() actually proxies. - if (isRedirectCandidate(c.req.method, c.req.path, c.req.header("origin") ?? null)) { - console.info("[zarr-data] rate-limit exempt (redirect candidate)", { path: c.req.path }); - return next(); - } - const res = await rateLimiter(c as unknown as Parameters[0], next); + // edge cache, or S3 through here. Observe-only (#1181 phase 6 review + // item 5): rateLimiter still counts these against the SAME shared + // bucket a proxied request from this IP would be enforced against, but + // never blocks this one, and logs at most one console.warn per window + // instead of the unconditional per-request log this used to be. Uses + // the SAME predicate serve() uses to pick its response branch (see + // isRedirectCandidate's doc comment), so this can never observe-only a + // request that serve() actually proxies. + const observeOnly = isRedirectCandidate( + c.req.method, + c.req.path, + c.req.header("origin") ?? null, + ); + const res = await rateLimiter( + c as unknown as Parameters[0], + next, + observeOnly ? { observeOnly: true } : undefined, + ); // rateLimiter returns a bare 429 (no zarr CORS) when the bucket is exhausted; // without ACAO the browser sees an opaque failure instead of a readable 429 + // retry_after. Re-apply the zarr CORS headers so the viewer can back off. diff --git a/backend/test/zarr-data-cache.test.ts b/backend/test/zarr-data-cache.test.ts index 847f656e5..9924a3354 100644 --- a/backend/test/zarr-data-cache.test.ts +++ b/backend/test/zarr-data-cache.test.ts @@ -1654,19 +1654,20 @@ describe("bytesFromRangeHeader (#1181 phase 6 / issue #1061)", () => { }); // --------------------------------------------------------------------------- -// Rate-limiter exemption for redirect candidates (#1181 phase 6 / issue -// #1061). rateLimiter's own request counter reads/writes the REAL -// `caches.default` global directly -- a different cache from the -// deps.cache() seam this file otherwise uses for zarr-data.ts's OWN edge -// cache, an unrelated concern. bun:test has no `caches` global at all, and -// every describe block above sidesteps that by keeping ENVIRONMENT: -// "development" in testEnv(), which is rateLimiter's own documented bypass -// (see middleware/rateLimit.ts). This block is the one place that needs the -// REAL, non-bypassed rateLimiter code path, so it installs a real in-memory -// CacheStorage stand-in for its own duration only -- the same technique -// rate-limit-buckets.test.ts uses for the same reason, restored in afterAll -// so no other suite sharing this process (root `bun test` runs test/ and -// backend/test/ together, see AGENTS.md) sees a stray global. +// Observe-only rate limiting for redirect candidates (#1181 phase 6 / issue +// #1061 / PR #1202 review item 5). rateLimiter's own request counter +// reads/writes the REAL `caches.default` global directly -- a different +// cache from the deps.cache() seam this file otherwise uses for +// zarr-data.ts's OWN edge cache, an unrelated concern. bun:test has no +// `caches` global at all, and every describe block above sidesteps that by +// keeping ENVIRONMENT: "development" in testEnv(), which is rateLimiter's +// own documented bypass (see middleware/rateLimit.ts). This block is the +// one place that needs the REAL, non-bypassed rateLimiter code path, so it +// installs a real in-memory CacheStorage stand-in for its own duration only +// -- the same technique rate-limit-buckets.test.ts uses for the same +// reason, restored in afterAll so no other suite sharing this process (root +// `bun test` runs test/ and backend/test/ together, see AGENTS.md) sees a +// stray global. // --------------------------------------------------------------------------- class RateLimitCache implements Cache { @@ -1705,9 +1706,8 @@ class RateLimitCache implements Cache { this.store.clear(); } - /** Read back the counter rateLimiter itself wrote (`{"count": N}`), or - * undefined if this bucket key was never touched at all -- the strongest - * proof that a redirect candidate never reached the enforcement code. */ + /** Read back the counter rateLimiter itself wrote (`{"count": N, ...}`), + * or undefined if this bucket key was never touched at all. */ count(url: string): number | undefined { const entry = this.store.get(url); return entry ? (JSON.parse(entry.body) as { count: number }).count : undefined; @@ -1754,9 +1754,10 @@ describe("rate limiter exemption for redirect candidates (#1181 phase 6 / issue return { ...testEnv(), ENVIRONMENT: "production" } as Bindings; } - test("a redirect candidate never touches the data-ip counter, no matter the IP or volume", async () => { + test("a redirect candidate counts toward the shared data-ip bucket, but never 429s (observe-only, #1181 phase 6 review item 5)", async () => { const ip = "203.0.113.10"; - for (let i = 0; i < 5; i++) { + const key = `https://rate-limit.internal/rl:data-ip:${ip}`; + for (let i = 1; i <= 5; i++) { const res = await requestWith( app, CHUNK_PATH, @@ -1764,15 +1765,17 @@ describe("rate limiter exemption for redirect candidates (#1181 phase 6 / issue rlEnv(), ); expect(res.status).toBe(302); + // The counter increments on every hit -- observe-only COUNTS against + // the same shared bucket a proxied request would be enforced + // against, it just never blocks the redirect itself. + expect(rlCache.count(key)).toBe(i); } - // Never created -- proves the middleware skipped rateLimiter entirely - // for these hits, not just that they landed under a generous cap. - expect(rlCache.count(`https://rate-limit.internal/rl:data-ip:${ip}`)).toBeUndefined(); }); - test("a redirect candidate still succeeds even once the SAME IP's data-ip bucket is fully exhausted", async () => { + test("a redirect candidate still succeeds (never 429s) even once the SAME IP's bucket is fully exhausted, and keeps counting", async () => { const ip = "203.0.113.11"; - rlCache.seedCount(`https://rate-limit.internal/rl:data-ip:${ip}`, __limits.DATA_MAX_REQUESTS); + const key = `https://rate-limit.internal/rl:data-ip:${ip}`; + rlCache.seedCount(key, __limits.DATA_MAX_REQUESTS); const res = await requestWith( app, @@ -1781,6 +1784,68 @@ describe("rate limiter exemption for redirect candidates (#1181 phase 6 / issue rlEnv(), ); expect(res.status).toBe(302); + expect(rlCache.count(key)).toBe(__limits.DATA_MAX_REQUESTS + 1); + }); + + test("N+1 redirects crossing the cap produce exactly one console.warn, never one per request", async () => { + const ip = "203.0.113.14"; + const key = `https://rate-limit.internal/rl:data-ip:${ip}`; + // Seed one below the cap so the SECOND request below is the one that + // crosses it -- proves the boundary behaviour at the real production + // DATA_MAX_REQUESTS cap without looping thousands of requests. + rlCache.seedCount(key, __limits.DATA_MAX_REQUESTS - 1); + + const originalWarn = console.warn; + const warnings: unknown[][] = []; + console.warn = (...args: unknown[]) => { + warnings.push(args); + }; + try { + // Request #1: reads count = DATA_MAX_REQUESTS - 1, still under cap -- + // no warn yet, but it does push the counter up to the cap. + const first = await requestWith( + app, + CHUNK_PATH, + { headers: { "CF-Connecting-IP": ip } }, + rlEnv(), + ); + expect(first.status).toBe(302); + expect(warnings.length).toBe(0); + expect(rlCache.count(key)).toBe(__limits.DATA_MAX_REQUESTS); + + // Request #2 (the "N+1"th for this window): reads count = + // DATA_MAX_REQUESTS, >= cap for the first time -- exactly one warn, + // never a 429. + const second = await requestWith( + app, + CHUNK_PATH, + { headers: { "CF-Connecting-IP": ip } }, + rlEnv(), + ); + expect(second.status).toBe(302); + expect(warnings.length).toBe(1); + expect(String(warnings[0][0])).toContain("would have tripped"); + const payload = warnings[0][1] as { ipHash: string; path: string; count: number }; + expect(payload.path).toBe(CHUNK_PATH); + // The logged count is post-increment (this request's own hit included). + expect(payload.count).toBe(__limits.DATA_MAX_REQUESTS + 1); + // Hashed, not the raw client IP, in the log line. + expect(payload.ipHash).not.toBe(ip); + expect(typeof payload.ipHash).toBe("string"); + + // A third, still-over-cap redirect in the SAME window does not warn + // again -- "never per request". + const third = await requestWith( + app, + CHUNK_PATH, + { headers: { "CF-Connecting-IP": ip } }, + rlEnv(), + ); + expect(third.status).toBe(302); + expect(warnings.length).toBe(1); + } finally { + console.warn = originalWarn; + } }); test("a proxied GET (index.json, no Origin) DOES charge the data-ip bucket and 429s at the real cap", async () => { From e1002b3a28fd6b3a589d98a5177e50b18d650459 Mon Sep 17 00:00:00 2001 From: Seyed Yahya Shirazi Date: Wed, 2 Sep 2026 11:09:59 -0700 Subject: [PATCH 6/9] docs: document chunk-redirect in access-metrics detail contract AccessEvent.detail's zarr contract listed only index/metadata/chunk; the redirect branch's "chunk-redirect" marker (#1181 phase 6) was undocumented at the source of truth for the field. PR #1202 review item 6. --- backend/src/services/access-metrics.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/backend/src/services/access-metrics.ts b/backend/src/services/access-metrics.ts index e02a64dc5..6cf076733 100644 --- a/backend/src/services/access-metrics.ts +++ b/backend/src/services/access-metrics.ts @@ -25,7 +25,14 @@ export interface AccessEvent { /** * Source-specific discriminator stored as a blob: * archive -> the version string (e.g. "1.0.0") - * zarr -> object class ("index" | "metadata" | "chunk") + * zarr -> object class ("index" | "metadata" | "chunk"), or + * "chunk-redirect" for a store object answered with a 302 + * to the public S3 object rather than proxied bytes + * (#1181 phase 6 / issue #1061) -- `bytes` on that event + * reflects only what the client's Range header asked for, + * never what S3 actually returned, so the dashboard should + * treat it as a distinct series rather than folding it into + * a served-bytes total with the other three. */ detail: string; /** From e8180f90e4d053de7fad58343c70c4bcadcfca6a Mon Sep 17 00:00:00 2001 From: Seyed Yahya Shirazi Date: Wed, 2 Sep 2026 11:11:27 -0700 Subject: [PATCH 7/9] test: catalog.json is proxied, not a redirect candidate Route-level companion to the isRedirectCandidate unit tests added earlier: a bare GET /catalog.json still gets a 200 from S3 via serveCatalog, never a 302 -- this route has no /zarr/ segment and never consults isRedirectCandidate at all. PR #1202 review item 7 (tests). --- backend/test/zarr-catalog-route.test.ts | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/backend/test/zarr-catalog-route.test.ts b/backend/test/zarr-catalog-route.test.ts index b5a874732..230857be5 100644 --- a/backend/test/zarr-catalog-route.test.ts +++ b/backend/test/zarr-catalog-route.test.ts @@ -214,6 +214,17 @@ describe("GET /catalog.json: success, miss then hit", () => { await requestCatalog(); expect(prepareCount).toBe(0); }); + + test("no Origin is still proxied, never redirected -- catalog.json can never match the zarr redirect path shape (#1181 phase 6 review item 7)", async () => { + // Unlike a store object under //zarr/, this route has no + // /zarr/ segment at all, so isRedirectCandidate's path regex can + // never match it -- there is no Origin-dependent fork here, only the + // one path serveCatalog() implements. + const res = await requestCatalog(); + expect(res.status).toBe(200); + expect(res.headers.get("location")).toBeNull(); + expect(upstreamRequests).toBe(1); + }); }); describe("GET /catalog.json: absence and failure", () => { From 8b35ec7aa753d330c4f67af62f3e0128a0858d95 Mon Sep 17 00:00:00 2001 From: Seyed Yahya Shirazi Date: Wed, 2 Sep 2026 11:22:28 -0700 Subject: [PATCH 8/9] test: cover redirect edge cases from review - /zarrproxy + allowed Origin + store object: proxied bytes with CORS (item 8) - HEAD tests assert the fake upstream actually received method HEAD, not just a matching path/range (item 9) - Private dataset redirect safety: follow the 302 Location for real and confirm S3 itself 403s an unauthenticated request under the private prefix (item 10) - 8 concurrent redirect-candidate GETs: all 302, zero upstream, zero D1 (item 13) - Re-verify range-key hit/miss, the full-object size cap, and the 206-without-Content-Range fail-closed path via a no-Origin index.json request, since those mechanics moved to allowed-Origin store objects earlier in this PR (item 12) Items 1-7 and 11 landed in earlier commits on this branch. PR #1202 review items 8, 9, 10, 12, 13. --- backend/test/zarr-data-cache.test.ts | 143 +++++++++++++++++++++++++-- 1 file changed, 137 insertions(+), 6 deletions(-) diff --git a/backend/test/zarr-data-cache.test.ts b/backend/test/zarr-data-cache.test.ts index 9924a3354..5de7b0971 100644 --- a/backend/test/zarr-data-cache.test.ts +++ b/backend/test/zarr-data-cache.test.ts @@ -714,25 +714,34 @@ describe("v-token round trip (#1181 review item 14)", () => { describe("HEAD never touches the cache", () => { test("(h) HEAD is never cached and never puts", async () => { - const head = await request(`/${PUBLIC_ID}/zarr/store.zarr/zarr.json`, { method: "HEAD" }); + const key = `${PUBLIC_ID}/zarr/store.zarr/zarr.json`; + const head = await request(`/${key}`, { method: "HEAD" }); expect(head.status).toBe(200); - expect(countUpstream(`${PUBLIC_ID}/zarr/store.zarr/zarr.json`, "")).toBe(1); + expect(countUpstream(key, "")).toBe(1); + // The fake upstream really received method HEAD, not a GET that just + // happened to match path/range (#1181 phase 6 review item 9) -- a + // regression that sent GET here would waste bandwidth pulling a full + // body for a HEAD probe. + expect(countUpstreamByMethod("HEAD", key, "")).toBe(1); + expect(countUpstreamByMethod("GET", key, "")).toBe(0); // A GET for the same object afterwards is still a miss -- HEAD wrote // nothing to the cache. An allowed Origin keeps this GET on the proxied // path (phase 6): a bare GET would redirect instead of reaching the // cache at all. - const get = await request(`/${PUBLIC_ID}/zarr/store.zarr/zarr.json`, { + const get = await request(`/${key}`, { headers: { Origin: BROWSER_ORIGIN }, }); expect(get.status).toBe(200); - expect(countUpstream(`${PUBLIC_ID}/zarr/store.zarr/zarr.json`, "")).toBe(2); + expect(countUpstream(key, "")).toBe(2); + expect(countUpstreamByMethod("GET", key, "")).toBe(1); // And a second HEAD, after that GET primed a cache entry, still goes // upstream -- HEAD never reads the cache either. - const head2 = await request(`/${PUBLIC_ID}/zarr/store.zarr/zarr.json`, { method: "HEAD" }); + const head2 = await request(`/${key}`, { method: "HEAD" }); expect(head2.status).toBe(200); - expect(countUpstream(`${PUBLIC_ID}/zarr/store.zarr/zarr.json`, "")).toBe(3); + expect(countUpstream(key, "")).toBe(3); + expect(countUpstreamByMethod("HEAD", key, "")).toBe(2); }); test("(17) HEAD with a Range header is never cached, never puts, and forwards the Range upstream", async () => { @@ -740,6 +749,7 @@ describe("HEAD never touches the cache", () => { const first = await request(CHUNK_PATH, { method: "HEAD", headers: { Range: range } }); expect(first.status).toBe(206); expect(countUpstream(CHUNK_KEY, range)).toBe(1); + expect(countUpstreamByMethod("HEAD", CHUNK_KEY, range)).toBe(1); const second = await request(CHUNK_PATH, { method: "HEAD", headers: { Range: range } }); expect(second.status).toBe(206); @@ -747,6 +757,8 @@ describe("HEAD never touches the cache", () => { // a cache entry. The upstream log entry only exists because the Range // header really was forwarded (the fake server keys its log on it). expect(countUpstream(CHUNK_KEY, range)).toBe(2); + expect(countUpstreamByMethod("HEAD", CHUNK_KEY, range)).toBe(2); + expect(countUpstreamByMethod("GET", CHUNK_KEY, range)).toBe(0); }); }); @@ -1470,6 +1482,42 @@ describe("no D1 read on the redirect path (#1181 phase 6 / issue #1061)", () => expect(res.status).toBe(302); expect(prepareCalls()).toBe(0); }); + + test("8 concurrent redirect-candidate GETs: all 302, zero upstream requests, zero D1 queries (#1181 phase 6 review item 13)", async () => { + const { db: countedDb, prepareCalls } = countingDb(realD1(db)); + const env = { ...testEnv(), DB: countedDb } as Bindings; + + const results = await Promise.all( + Array.from({ length: 8 }, () => requestWithEnv(CHUNK_PATH, {}, env)), + ); + for (const res of results) { + expect(res.status).toBe(302); + } + expect(requestLog.length).toBe(0); + expect(prepareCalls()).toBe(0); + }); +}); + +describe("private dataset redirect safety: the bucket's own deny-list, not this route, is what blocks it (#1181 phase 6 review item 10)", () => { + test("following the 302 Location for a private dataset's chunk gets 403 directly from S3", async () => { + const path = `/${PRIVATE_ID}/zarr/store.zarr/zarr.json`; + const res = await request(path); + expect(res.status).toBe(302); + const location = res.headers.get("location"); + expect(location).not.toBeNull(); + + // This IS the actual safety argument for the redirect branch skipping + // the D1 gate (decision 2 in the phase 6 brief, and the module doc + // comment above): the bucket's NotResource deny-list + // (services/bucket-policy.ts) denies ANONYMOUS s3:GetObject on a + // private dataset's ENTIRE prefix at the bucket level, independent of + // this Worker or D1. Fetch the Location directly -- exactly what a + // real client following the redirect would do -- and confirm S3 + // itself, not this route, is what actually keeps the object private. A + // redirect that 403s here leaks nothing a proxied 404 does not. + const upstreamRes = await fetch(location as string); + expect(upstreamRes.status).toBe(403); + }); }); describe("/zarrproxy path-mount entry point (#1181 phase 6)", () => { @@ -1500,6 +1548,89 @@ describe("/zarrproxy path-mount entry point (#1181 phase 6)", () => { expect(res.status).toBe(200); expect(countUpstream(`${PUBLIC_ID}/zarr/index.json`, "")).toBe(1); }); + + test("an allowed Origin through the path-mounted entry point is proxied with bytes + CORS, not redirected (#1181 phase 6 review item 8)", async () => { + const mounted = new Hono<{ Bindings: Bindings }>(); + mounted.route("/zarrproxy", app); + + const res = await requestWith( + mounted, + `/zarrproxy${CHUNK_PATH}`, + { headers: { Origin: BROWSER_ORIGIN } }, + testEnv(), + ); + expect(res.status).toBe(200); + expect(res.headers.get("access-control-allow-origin")).toBe(BROWSER_ORIGIN); + expect(res.headers.get("location")).toBeNull(); + const bytes = new Uint8Array(await res.arrayBuffer()); + expect(bytes).toEqual(CHUNK_BYTES); + expect(countUpstream(CHUNK_KEY, "")).toBe(1); + }); +}); + +describe("edge-cache mechanics re-verified for a no-Origin index.json path (#1181 phase 6 review item 12)", () => { + // index.json is never a redirect candidate, so a BARE (no-Origin) request + // to it is still proxied and still runs the SAME edge-cache machinery the + // phase-1 tests above now exercise via an allowed-Origin store object. + // These re-confirm the same mechanics reached through the no-Origin path, + // which phase 6 made the common case for every OTHER object class. + const INDEX_PATH = `/${PUBLIC_ID}/zarr/index.json`; + const INDEX_KEY = `${PUBLIC_ID}/zarr/index.json`; + + test("range-key miss then hit", async () => { + const range = "bytes=0-5"; + const first = await request(INDEX_PATH, { headers: { Range: range } }); + expect(first.status).toBe(206); + expect(countUpstream(INDEX_KEY, range)).toBe(1); + + const second = await request(INDEX_PATH, { headers: { Range: range } }); + expect(second.status).toBe(206); + expect(countUpstream(INDEX_KEY, range)).toBe(1); // hit, not a second fetch + }); + + test("a different range is a separate miss", async () => { + await request(INDEX_PATH, { headers: { Range: "bytes=0-5" } }); + const res = await request(INDEX_PATH, { headers: { Range: "bytes=6-10" } }); + expect(res.status).toBe(206); + expect(countUpstream(INDEX_KEY, "bytes=6-10")).toBe(1); + expect(countUpstream(INDEX_KEY, "bytes=0-5")).toBe(1); // untouched + }); + + test("size cap: a full object over FULL_OBJECT_CACHE_MAX_BYTES is served correctly and not cached", async () => { + // index.json's own fixture is small; swap in the oversized fixture for + // this one test only, restoring it in `finally` so no other test in + // this file (which shares OBJECTS at module scope) sees the swap. + const original = OBJECTS[INDEX_KEY]; + OBJECTS[INDEX_KEY] = CAP_TEST_BYTES; + try { + const first = await request(INDEX_PATH); + expect(first.status).toBe(200); + expect(countUpstream(INDEX_KEY, "")).toBe(1); + + const second = await request(INDEX_PATH); + expect(second.status).toBe(200); + expect(countUpstream(INDEX_KEY, "")).toBe(2); // not cached -- over the cap + } finally { + OBJECTS[INDEX_KEY] = original; + } + }, 20000); + + test("a 206 with no Content-Range fails closed, uncached", async () => { + QUIRKS.set(INDEX_KEY, "no-content-range"); + try { + const range = "bytes=0-5"; + const first = await request(INDEX_PATH, { headers: { Range: range } }); + expect(first.status).toBe(206); + expect(first.headers.get("content-range")).toBeNull(); + expect(countUpstream(INDEX_KEY, range)).toBe(1); + + const second = await request(INDEX_PATH, { headers: { Range: range } }); + expect(second.status).toBe(206); + expect(countUpstream(INDEX_KEY, range)).toBe(2); // never cached + } finally { + QUIRKS.delete(INDEX_KEY); + } + }); }); describe("telemetry bytes on the redirect path (#1181 phase 6 / issue #1061)", () => { From 23f4b7a356898df64641d02852f4fcaffe2ac963 Mon Sep 17 00:00:00 2001 From: Seyed Yahya Shirazi Date: Wed, 2 Sep 2026 11:31:17 -0700 Subject: [PATCH 9/9] refactor: hash observe-only IPs with hashIp --- backend/src/middleware/rateLimit.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/backend/src/middleware/rateLimit.ts b/backend/src/middleware/rateLimit.ts index c409622bc..caab3d214 100644 --- a/backend/src/middleware/rateLimit.ts +++ b/backend/src/middleware/rateLimit.ts @@ -23,6 +23,7 @@ import type { Context, Next } from "hono"; import { hashApiKey } from "../services/token"; +import { hashIp } from "../services/web-session"; import type { Bindings, Variables } from "../types/bindings"; // Rate limit configuration @@ -334,7 +335,9 @@ export async function rateLimiter( // Hashed, not raw: this is a log line, not the enforcement key // itself (which stays keyed on the raw IP, same as before) -- // avoids putting a raw client IP into structured logs. - const ipHash = await hashApiKey(rawKey); + // Same helper web-session.ts uses for privacy-preserving IP storage, so + // every IP hash in the codebase is produced by one named function. + const ipHash = await hashIp(rawKey); console.warn("[rate-limit] observe-only bucket would have tripped", { ipHash, path,