Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 25 additions & 0 deletions backend/src/db/migrations/0043_dataset_archive_skip_reason.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
-- Migration 0043: archive_skip_reason for the >100GB skip-archive policy
-- (epic #749, Phase 3 / #752).
--
-- run-generate-archive.yml has a 60-min cap and no size guard, so large
-- datasets stream forever and get cancelled mid-zip (on007523 478GB cancelled
-- at ~24%; on005752 680GB / on007524 321GB never produced a zip). Phase 3
-- preflights the version manifest totals and, over threshold, SKIPS the zip and
-- steers users to the range-resumable per-file direct download
-- (data.nemar.org/<id>/<v>/manifest.json bytes_url).
--
-- Representing "skipped": rather than widen the 0036 CHECK on archive_status
-- (which is `IN ('pending','ready','failed')` and cannot be ALTERed in place --
-- adding a value would require recreating the central `datasets` table along
-- with its datasets_fts external-content table + triggers, a high-risk op), we
-- add a nullable reason column. A dataset is "archive skipped" iff
-- archive_skip_reason IS NOT NULL; archive_status stays NULL for those rows.
--
-- archive_skip_reason human-readable reason the archive was skipped
-- (e.g. "dataset 680.0 GB exceeds 100 GB archive limit"),
-- set by /webhooks/archive-ready status='skipped' and the
-- admin archive-sweep. NULL = not skipped.
--
-- Plain ADD COLUMN like 0036/0040-0042; not in datasets_fts or the embed-dirty
-- trigger OF lists, so this touches neither the lexical index nor embeddings.
ALTER TABLE datasets ADD COLUMN archive_skip_reason TEXT;
54 changes: 37 additions & 17 deletions backend/src/routes/admin.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import { adminMiddleware, authMiddleware, ownerMiddleware } from "../middleware/
import { LIVE_DATASETS, isLiveDataset } from "../constants";
import { tombstoneUserStatement } from "../db/user-tombstone";
import { SYSTEM_USER_ID } from "../lib/constants";
import { shouldSkipArchive } from "../services/archive-policy";
import {
type RecipientGroup,
type RecipientGroupOrUser,
Expand Down Expand Up @@ -1136,11 +1137,13 @@ adminRoutes.post("/datasets/archive-sweep", async (c) => {
// concern and the data-plane download is public-only (loadPublishedDataset).
// A bare throw here (e.g. migration 0036 not yet applied) would surface a
// contextless 500, so handle it explicitly.
let candidates: { dataset_id: string }[];
let candidates: { dataset_id: string; file_size: number | null; total_files: number | null }[];
try {
const candidateRows = await db
.prepare(
`SELECT dataset_id FROM datasets
// file_size/total_files (migration 0020) let us mark an oversized dataset
// with no archive as 'skipped' (#752) instead of just 'absent'.
`SELECT dataset_id, file_size, total_files FROM datasets
WHERE owner_user_id != ${SYSTEM_USER_ID}
AND (is_sandbox = 0 OR is_sandbox IS NULL)
AND visibility = 'public'
Expand All @@ -1149,7 +1152,7 @@ adminRoutes.post("/datasets/archive-sweep", async (c) => {
LIMIT ?`,
)
.bind(limit)
.all<{ dataset_id: string }>();
.all<{ dataset_id: string; file_size: number | null; total_files: number | null }>();
candidates = candidateRows.results ?? [];
} catch (err) {
console.error("[archive-sweep] candidate query failed:", err);
Expand All @@ -1159,9 +1162,10 @@ adminRoutes.post("/datasets/archive-sweep", async (c) => {
const s3 = getS3Config(c.env);
let ready = 0;
let absent = 0;
let skipped = 0;
const errors: { dataset_id: string; error: string }[] = [];

for (const { dataset_id } of candidates) {
for (const { dataset_id, file_size, total_files } of candidates) {
// Separate try/catch per phase so an error is attributed to the operation
// that failed (S3 LIST vs D1 write), not lumped together.
let size: number;
Expand All @@ -1175,28 +1179,43 @@ adminRoutes.post("/datasets/archive-sweep", async (c) => {
if (size > 0) {
await db
.prepare(
"UPDATE datasets SET archive_status = 'ready', archive_size = ?, archive_checked_at = datetime('now') WHERE dataset_id = ?",
// Clear any stale archive_skip_reason: a real zip exists (#752).
"UPDATE datasets SET archive_status = 'ready', archive_size = ?, archive_checked_at = datetime('now'), archive_skip_reason = NULL WHERE dataset_id = ?",
)
.bind(size, dataset_id)
.run();
ready++;
} else {
// Checked, no archive on S3: stamp checked_at so the sweep won't rescan,
// but leave archive_status NULL (absence is not a 'failed' generation).
// The dashboard treats a published dataset with status != 'ready' as
// "missing archive".
await db
.prepare("UPDATE datasets SET archive_checked_at = datetime('now') WHERE dataset_id = ?")
.bind(dataset_id)
.run();
absent++;
// Checked, no archive on S3. If the dataset is over the size policy
// (#752), record WHY no zip exists (archive_skip_reason) so the UI shows
// the direct-download recipe instead of "missing archive". Otherwise it's
// genuinely absent: stamp checked_at, leave archive_status NULL.
const decision = shouldSkipArchive({ totalBytes: file_size, totalFiles: total_files });
if (decision.skip) {
await db
.prepare(
"UPDATE datasets SET archive_skip_reason = ?, archive_checked_at = datetime('now') WHERE dataset_id = ?",
)
.bind(decision.reason ?? "archive skipped (size policy)", dataset_id)
.run();
skipped++;
} else {
await db
.prepare(
"UPDATE datasets SET archive_checked_at = datetime('now') WHERE dataset_id = ?",
)
.bind(dataset_id)
.run();
absent++;
}
}
} catch (err) {
// S3 confirmed the size; only the D1 write failed. Note that so a re-run's
// duplicate entry is explicable.
// S3 confirmed the size; only the D1 write failed. Note the branch (ready
// vs skip/absent) + size so a re-run's duplicate entry is explicable and a
// dropped archive_skip_reason write is attributable, not masked as "ready".
errors.push({
dataset_id,
error: `d1 (s3 size=${size}): ${err instanceof Error ? err.message : String(err)}`,
error: `d1 write [${size > 0 ? "ready" : "skip/absent"}] (s3 size=${size}): ${err instanceof Error ? err.message : String(err)}`,
});
}
}
Expand Down Expand Up @@ -1227,6 +1246,7 @@ adminRoutes.post("/datasets/archive-sweep", async (c) => {
checked: candidates.length,
ready,
absent,
skipped,
errors,
remaining: remainingRow?.n ?? null,
});
Expand Down
20 changes: 17 additions & 3 deletions backend/src/routes/data.ts
Original file line number Diff line number Diff line change
Expand Up @@ -77,10 +77,16 @@ async function loadPublishedDataset(env: Bindings, datasetId: string) {
return null;
}
const row = await env.DB.prepare(
"SELECT dataset_id, visibility FROM datasets WHERE dataset_id = ?",
"SELECT dataset_id, visibility, archive_status, archive_size, archive_skip_reason FROM datasets WHERE dataset_id = ?",
)
.bind(datasetId)
.first<{ dataset_id: string; visibility: string }>();
.first<{
dataset_id: string;
visibility: string;
archive_status: string | null;
archive_size: number | null;
archive_skip_reason: string | null;
}>();
if (!row) {
console.log(`[data] reject: not in catalog datasetId=${datasetId}`);
return null;
Expand Down Expand Up @@ -1117,7 +1123,15 @@ async function datasetRootResponse(
if (!dataset) return notFound("Dataset not found");

const versionRows = await loadVersionRows(env, datasetId);
const payload = buildLandingPayload({ datasetId, versionRows });
const payload = buildLandingPayload({
datasetId,
versionRows,
archive: {
status: dataset.archive_status,
size: dataset.archive_size,
skip_reason: dataset.archive_skip_reason,
},
});

const accept = request.headers.get("accept");
const formatParam = new URL(request.url).searchParams.get("format");
Expand Down
34 changes: 29 additions & 5 deletions backend/src/routes/webhooks.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2713,12 +2713,16 @@ webhooks.post("/zarr-ready", async (c) => {
*/
interface ArchiveReadyBody {
dataset_id: string;
status?: "ready" | "failed";
status?: "ready" | "failed" | "skipped";
/** Bytes of the generated zip; persisted to datasets.archive_size on 'ready'. */
size?: number;
/** The published version the archive was built for (logged, not stored). */
version?: string;
error?: string;
/** Why archive generation was skipped (status='skipped', #752). Persisted to
* datasets.archive_skip_reason; its presence is what marks a dataset
* "archive skipped" (archive_status stays NULL). */
reason?: string;
}

webhooks.post("/archive-ready", async (c) => {
Expand Down Expand Up @@ -2746,8 +2750,8 @@ webhooks.post("/archive-ready", async (c) => {
}
// Require an explicit status: a missing/unknown value must not default to
// 'ready' (that would mark a failed generation as having an archive).
if (body.status !== "ready" && body.status !== "failed") {
return c.json({ error: "status must be 'ready' or 'failed'" }, 400);
if (body.status !== "ready" && body.status !== "failed" && body.status !== "skipped") {
return c.json({ error: "status must be 'ready', 'failed', or 'skipped'" }, 400);
}
const status = body.status;
if (status === "failed" && body.error) {
Expand All @@ -2765,16 +2769,34 @@ webhooks.post("/archive-ready", async (c) => {
try {
if (status === "ready") {
const result = await c.env.DB.prepare(
// Clear archive_skip_reason: a real zip now exists, so a stale skip from
// an earlier (larger) version must not keep the UI on the direct-download
// recipe (#752).
`UPDATE datasets
SET archive_status = 'ready',
archive_checked_at = datetime('now'),
archive_size = ?,
archive_retry_count = 0
archive_retry_count = 0,
archive_skip_reason = NULL
WHERE dataset_id = ?`,
)
.bind(typeof body.size === "number" ? body.size : null, body.dataset_id)
.run();
changed = result.meta.changes ?? 0;
} else if (status === "skipped") {
// Over the size/file-count policy (#752): the workflow built no zip and
// steers users to direct download. Record the reason; leave archive_status
// NULL (skipped is intentional, NOT a failed generation -> no auto-retry).
const result = await c.env.DB.prepare(
`UPDATE datasets
SET archive_skip_reason = ?,
archive_status = NULL,
archive_checked_at = datetime('now')
WHERE dataset_id = ?`,
)
.bind(body.reason ?? "archive skipped (size policy)", body.dataset_id)
.run();
changed = result.meta.changes ?? 0;
} else {
// Read the current dispatch count to decide whether to re-dispatch. The
// count is NOT advanced here -- it is incremented only after a successful
Expand Down Expand Up @@ -2808,7 +2830,9 @@ webhooks.post("/archive-ready", async (c) => {
// matches zero rows and persists nothing. Surface it as a 404 instead of a
// silent 200 -- mirrors the zarr-ready / prescreen-result guards.
if (changed === 0) {
console.error(`[archive-ready] UPDATE matched 0 rows dataset=${body.dataset_id} -- not in D1`);
console.error(
`[archive-ready] UPDATE matched 0 rows dataset=${body.dataset_id} status=${status} -- not in D1`,
);
return c.json({ error: "Dataset not found" }, 404);
}

Expand Down
59 changes: 59 additions & 0 deletions backend/src/services/archive-policy.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
/**
* Archive size policy (epic #749, Phase 3 / #752).
*
* Datasets over a size/file-count threshold skip the downloadable-zip build
* (run-generate-archive.yml's 60-min cap can't finish them) and steer users to
* the range-resumable per-file direct download instead. The thresholds and the
* decision live here so the Worker (archive-ready, admin sweep) and the CI
* preflight (run-generate-archive.yml mirrors these numbers in bash) agree.
*/

/** Bytes ceiling for building a zip archive. Over this -> skip + direct download.
* 100 GiB. The 2026-06-14 batch that blew the 60-min cap was 321-680 GB. */
export const ARCHIVE_MAX_BYTES = 100 * 1024 * 1024 * 1024;

/** File-count ceiling. Guards the pathological many-tiny-files case where bytes
* are modest but the zip's per-entry overhead still blows the cap. */
export const ARCHIVE_MAX_FILES = 200_000;

export interface ArchiveSizeInput {
/** Total dataset bytes (from the version manifest's cached total). */
totalBytes: number | null | undefined;
/** Total file count (from the version manifest). */
totalFiles?: number | null;
}

export interface ArchiveSkipDecision {
skip: boolean;
/** Human-readable reason when skip is true; undefined otherwise. */
reason?: string;
}

/** Format bytes as a compact GB string for the skip reason. */
function gb(bytes: number): string {
return `${(bytes / (1024 * 1024 * 1024)).toFixed(1)} GB`;
}

/**
* Decide whether to skip zip-archive generation for a dataset of the given
* size. Pure. Over the byte ceiling OR the file-count ceiling -> skip with a
* reason. Unknown total bytes is treated as NOT skipped (build attempted; the
* 60-min cap remains the backstop) so a missing manifest stat can't silently
* suppress every archive.
*/
export function shouldSkipArchive(input: ArchiveSizeInput): ArchiveSkipDecision {
const { totalBytes, totalFiles } = input;
if (typeof totalBytes === "number" && totalBytes > ARCHIVE_MAX_BYTES) {
return {
skip: true,
reason: `dataset ${gb(totalBytes)} exceeds ${gb(ARCHIVE_MAX_BYTES)} archive limit; use direct download`,
};
}
if (typeof totalFiles === "number" && totalFiles > ARCHIVE_MAX_FILES) {
return {
skip: true,
reason: `dataset ${totalFiles.toLocaleString()} files exceeds ${ARCHIVE_MAX_FILES.toLocaleString()} archive limit; use direct download`,
};
}
return { skip: false };
}
17 changes: 17 additions & 0 deletions backend/src/services/data-router.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1209,11 +1209,22 @@ export interface LandingVersion {
browse_url: string;
}

/** Latest-only downloadable-archive state (#752). `status` is 'ready'/'failed'/
* null; a non-null `skip_reason` means the zip was intentionally skipped by the
* size policy (status stays null) and the consumer should render the direct
* per-file download recipe instead of a zip button. */
export interface LandingArchive {
status: string | null;
size: number | null;
skip_reason: string | null;
}

export interface LandingPayload {
dataset_id: string;
latest: string | null;
metadata_url: string;
versions: LandingVersion[];
archive: LandingArchive;
}

/**
Expand All @@ -1224,6 +1235,7 @@ export interface LandingPayload {
export function buildLandingPayload(args: {
datasetId: string;
versionRows: DatasetVersionRow[];
archive?: { status?: string | null; size?: number | null; skip_reason?: string | null };
}): LandingPayload {
const { datasetId, versionRows } = args;
const versions: LandingVersion[] = versionRows.map((row) => {
Expand All @@ -1241,6 +1253,11 @@ export function buildLandingPayload(args: {
latest: versions.length > 0 ? versions[0].version : null,
metadata_url: `/${datasetId}/metadata.json`,
versions,
archive: {
status: args.archive?.status ?? null,
size: args.archive?.size ?? null,
skip_reason: args.archive?.skip_reason ?? null,
},
};
}

Expand Down
25 changes: 23 additions & 2 deletions backend/src/services/page-bundle.ts
Original file line number Diff line number Diff line change
Expand Up @@ -307,11 +307,32 @@ export async function buildPageBundle(
datasetId: string,
versionParam: string | null,
): Promise<PageBundle> {
// landing is cheap (one D1 query); compute it eagerly so we know which
// landing is cheap (two small D1 queries); compute it eagerly so we know which
// version to ask the summary endpoint for. D1 failure here MUST propagate
// — see loadVersionRowsForBundle docstring.
const versionRows = await loadVersionRowsForBundle(env, datasetId);
const landingPayload = buildLandingPayload({ datasetId, versionRows });
// Latest-only archive state (#752) so the bundle's landing payload carries the
// skip/ready signal, matching the /<id>/ landing route (data.ts).
const archiveRow = await env.DB.prepare(
"SELECT archive_status, archive_size, archive_skip_reason FROM datasets WHERE dataset_id = ?",
)
.bind(datasetId)
.first<{
archive_status: string | null;
archive_size: number | null;
archive_skip_reason: string | null;
}>();
const landingPayload = buildLandingPayload({
datasetId,
versionRows,
archive: archiveRow
? {
status: archiveRow.archive_status,
size: archiveRow.archive_size,
skip_reason: archiveRow.archive_skip_reason,
}
: undefined,
});
const resolvedVersion = pickVersion(versionParam, versionRows);

// Narrow `resolvedVersion` to a non-null local so the TS flow analyzer
Expand Down
Loading
Loading