From e74afff85493dc87b374d672321e8b0e3811bf6f Mon Sep 17 00:00:00 2001 From: Tim Nunamaker Date: Sun, 18 Oct 2026 00:24:18 -0500 Subject: [PATCH] fix(collector-runtime): stop failed spool writes leaving temp files A failed write to the blob spool (`LocalDeviceBlobSpool`, which keeps artifact bytes on local disk under their sha256) can leave its temp file in the spool's tmp/ directory. Nothing reads that file as an artifact, because the rename into place never runs, but it occupies disk space. No production code calls sweepTemp(), the method meant to reclaim leftover temp files, so a leftover stays in place. put(), the spool's write method, writes incoming bytes to a temp file and renames it to its sha256 path only after the whole body has been written. If reading the content throws partway, Node's pipeline() helper rejects as soon as that happens. It does not wait for the file's open, which Node runs asynchronously on its thread pool. put()'s catch block then deletes the temp file synchronously. When the open has not created the file yet, the delete finds nothing, and the file appears afterwards with nothing to remove it. The catch block now waits for the write stream's 'close' event before deleting. A file stream emits 'close' only after a pending open has settled and its file descriptor is closed, so the delete runs after the file has been created or the open has failed. The process sees this failure, so it removes the file now rather than leaving it for sweepTemp(). The success path is unchanged. The failure path now waits for one extra open and close before put() rejects. The regression test delays the file's open (by mocking fs.open, Node's file-open function) until the failure has reached put()'s catch block, then checks that the spool's tmp directory is empty. Without the fix it fails in 10 of 10 runs, leaving the temp file. With the fix it passes in 10 of 10. The existing test "leaves no addressable entry when the source fails mid-write" exercises the same race without delaying the open. It failed in 12 of 200 runs before the fix and in 0 of 200 after. With eight CPU-bound processes running alongside, it failed in 16 of 200 before the fix and in 0 of 200 after. Not verified: how often production writes take this path, how large a leftover file can get, and whether code outside the collector-runtime package depends on how quickly put() rejects. The one caller inside the package, captureBlobArtifact, which spools bytes and then queues an upload, propagates the error unchanged. The added latency on the failure path is not measured. Signed-off-by: Tim Nunamaker Assisted-by: AI --- .../src/local-device-blob-spool.test.ts | 57 ++++++++++++++++++- .../src/local-device-blob-spool.ts | 23 +++++++- 2 files changed, 76 insertions(+), 4 deletions(-) diff --git a/packages/collector-runtime/src/local-device-blob-spool.test.ts b/packages/collector-runtime/src/local-device-blob-spool.test.ts index 05c858249..5a791d127 100644 --- a/packages/collector-runtime/src/local-device-blob-spool.test.ts +++ b/packages/collector-runtime/src/local-device-blob-spool.test.ts @@ -3,7 +3,7 @@ import assert from "node:assert/strict"; import { createHash, randomBytes } from "node:crypto"; -import { createReadStream, mkdtempSync, readdirSync, rmSync, statSync, writeFileSync } from "node:fs"; +import fs, { createReadStream, mkdtempSync, readdirSync, rmSync, statSync, writeFileSync } from "node:fs"; import { readFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -94,6 +94,61 @@ describe("LocalDeviceBlobSpool", () => { assert.deepEqual(readdirSync(join(root, "tmp")), []); }); + it("leaves no temp file when the source fails before the sink has opened", async (t) => { + const root = makeRoot(); + const spool = new LocalDeviceBlobSpool({ root }); + const partial = randomBytes(8192); + const boom = new Error("source read failed"); + + // Hold the sink's open() until the failure has been handled. Unheld, the + // open can land after put()'s cleanup and leave a temp file behind. Under + // load that order happens by chance; holding the open forces it. + const realOpen = fs.open; + let releaseOpen!: () => void; + const released = new Promise((resolve) => { + releaseOpen = resolve; + }); + let openLanded!: () => void; + const landed = new Promise((resolve) => { + openLanded = resolve; + }); + let held = false; + t.mock.method( + fs, + "open", + (path: string, flags: string, mode: number, callback: (error: Error | null, fd: number) => void) => { + held = true; + released.then(() => + realOpen(path, flags, mode, (error, fd) => { + callback(error, fd); + openLanded(); + }) + ); + } + ); + + const outcome = spool + .put( + (async function* () { + yield await Promise.resolve(partial); + throw boom; + })() + ) + .then( + () => assert.fail("put must reject when the source fails"), + (error: unknown) => error + ); + + // Let the failure reach its cleanup while the open is still held. + await new Promise((resolve) => setImmediate(resolve)); + assert.ok(held, "the sink's open() must go through fs.open for this test to hold it"); + releaseOpen(); + await landed; + + assert.equal(await outcome, boom); + assert.deepEqual(readdirSync(join(root, "tmp")), [], "the failed write must not leave a temp file behind"); + }); + it("reports a missing body distinctly so the drain can stop retrying", () => { const root = makeRoot(); const spool = new LocalDeviceBlobSpool({ root }); diff --git a/packages/collector-runtime/src/local-device-blob-spool.ts b/packages/collector-runtime/src/local-device-blob-spool.ts index cd411c0e8..565b30470 100644 --- a/packages/collector-runtime/src/local-device-blob-spool.ts +++ b/packages/collector-runtime/src/local-device-blob-spool.ts @@ -15,6 +15,7 @@ import { } from "node:fs"; import { readdir, stat } from "node:fs/promises"; import { join } from "node:path"; +import type { Writable } from "node:stream"; import { pipeline } from "node:stream/promises"; /** @@ -174,6 +175,17 @@ async function* toByteChunks(content: LocalDeviceBlobSpoolContent): AsyncIterabl } } +/** + * Resolve once `stream` has emitted `close`, including when it emitted it earlier. + * A destroyed file stream closes only after its pending `open()` has settled. + */ +function waitForClose(stream: Writable): Promise { + if (stream.closed) { + return Promise.resolve(); + } + return new Promise((resolve) => stream.once("close", () => resolve())); +} + /** * Hard precondition for enabling spool reclamation: every queue that can name * a body in this spool must be represented in the reference set. @@ -276,6 +288,9 @@ export class LocalDeviceBlobSpool { const hash = createHash("sha256"); let sizeBytes = 0; const tempPath = join(this.#tempDir(), `spool-${process.pid}-${Date.now()}-${Math.random().toString(36).slice(2)}`); + // flush: true fsyncs the file content before close resolves. Without + // it the rename can publish a name over bytes still only in cache. + const sink = createWriteStream(tempPath, { flush: true }); try { await pipeline( (async function* (source: AsyncIterable) { @@ -285,11 +300,13 @@ export class LocalDeviceBlobSpool { yield chunk; } })(toByteChunks(content)), - // flush: true fsyncs the file content before close resolves. Without - // it the rename can publish a name over bytes still only in cache. - createWriteStream(tempPath, { flush: true }) + sink ); } catch (error) { + // pipeline rejects once the source fails, not once the sink has closed. + // The sink's open() may still be in flight: an unlink that runs before it + // lands finds nothing, and the file then appears with nothing to remove it. + await waitForClose(sink); this.#discardTemp(tempPath); throw error; }