From 4e865a05117e251bccbb36bddfec56d98c6deccc Mon Sep 17 00:00:00 2001 From: Pyseph <45090858+Pyseph@users.noreply.github.com> Date: Thu, 5 Nov 2026 02:33:05 +0100 Subject: [PATCH 1/3] perf(server): folder listings beyond four share one Git ignore check Expand all lists every folder and labels each folder's children with its own `git check-ignore` process. Every ignore check of a listing now counts toward a limit of four per workspace. A check that finds a free slot runs in the listing's own fiber, as it always did. Checks that arrive while four run wait and go out together in the next call (one Git root, at most 1,000 paths); identical checks that can't be split, such as nested-repository probes, share one call. A shared call that Git refuses or that times out is asked again folder by folder through the same limit. Folders reached through a symlink never share a call. Waiting checks whose listings all left never reach Git, and a call whose listings all left is interrupted. A call's finalizer frees its slot and completes its listings on any exit; a defect fails those listings, as it fails a listing's own call. Nothing is reused: every call starts after all of its checks arrived. --- .../src/workspace/WorkspaceEntries.test.ts | 190 ++++++++++++++ apps/server/src/workspace/WorkspaceEntries.ts | 50 ++-- .../workspace/workspaceIgnoreChecks.test.ts | 242 ++++++++++++++++++ .../src/workspace/workspaceIgnoreChecks.ts | 214 ++++++++++++++++ 4 files changed, 676 insertions(+), 20 deletions(-) create mode 100644 apps/server/src/workspace/workspaceIgnoreChecks.test.ts create mode 100644 apps/server/src/workspace/workspaceIgnoreChecks.ts diff --git a/apps/server/src/workspace/WorkspaceEntries.test.ts b/apps/server/src/workspace/WorkspaceEntries.test.ts index eedff4fc79c9..9dd76906fa94 100644 --- a/apps/server/src/workspace/WorkspaceEntries.test.ts +++ b/apps/server/src/workspace/WorkspaceEntries.test.ts @@ -2,7 +2,9 @@ import * as NodeFSP from "node:fs/promises"; import * as NodeServices from "@effect/platform-node/NodeServices"; import { FileFinder } from "@ff-labs/fff-node"; +import type { ProjectEntry } from "@t3tools/contracts"; import { it, afterEach, describe, expect } from "@effect/vitest"; +import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; @@ -15,6 +17,7 @@ import * as HostProcess from "@t3tools/shared/HostProcess"; import * as VcsProcess from "../vcs/VcsProcess.ts"; import * as WorkspaceEntries from "./WorkspaceEntries.ts"; import * as WorkspacePaths from "./WorkspacePaths.ts"; +import * as WorkspaceSearchIndex from "./WorkspaceSearchIndex.ts"; vi.mock("node:fs/promises", async (importOriginal) => { const actual = await importOriginal(); @@ -83,6 +86,58 @@ const searchWorkspaceEntries = (input: { return yield* workspaceEntries.search(input); }); +/** + * Lists folders at once, as "Expand all" does with a wide client. Git calls are held until four + * have started, so later folders find four checks running; records each call's cwd and stdin. + */ +const listAtOnce = (cwd: string, folders: ReadonlyArray) => + Effect.gen(function* () { + const fourStarted = yield* Deferred.make(); + const calls: Array<{ readonly cwd: string; readonly stdin: string }> = []; + let running = 0; + let mostRunning = 0; + const vcsProcess = Layer.effect( + VcsProcess.VcsProcess, + Effect.map(VcsProcess.VcsProcess, (inner) => + VcsProcess.VcsProcess.of({ + run: (input) => + input.args.includes("check-ignore") + ? Effect.suspend(() => { + calls.push({ cwd: input.cwd, stdin: input.stdin ?? "" }); + mostRunning = Math.max(mostRunning, ++running); + if (calls.length === 4) Deferred.doneUnsafe(fourStarted, Effect.void); + return Deferred.await(fourStarted).pipe( + Effect.andThen(inner.run(input)), + Effect.ensuring(Effect.sync(() => running--)), + ); + }) + : inner.run(input), + }), + ), + ).pipe(Layer.provide(VcsProcess.layer)); + const listings = yield* Effect.forEach( + folders, + (directoryPath) => + Effect.flatMap(WorkspaceEntries.WorkspaceEntries, (workspaceEntries) => + workspaceEntries.list({ cwd, directoryPath }), + ).pipe( + Effect.map(({ entries }) => + entries.toSorted((left, right) => left.path.localeCompare(right.path)), + ), + ), + { concurrency: "unbounded" }, + ).pipe( + Effect.provide( + Layer.effect(WorkspaceEntries.WorkspaceEntries, WorkspaceEntries.make).pipe( + Layer.provide(WorkspaceSearchIndex.WorkspaceSearchIndexMap.layer), + Layer.provide(vcsProcess), + Layer.provide(WorkspacePaths.layer), + ), + ), + ); + return { listings, calls, mostRunning }; + }); + const appendSeparator = (input: string) => Effect.map(HostProcess.Platform, (platform) => input.endsWith("/") || input.endsWith("\\") @@ -328,6 +383,141 @@ it.layer(layerTest, { excludeTestServices: true })("WorkspaceEntries", (it) => { }), ); + it.effect("labels folders listed beyond four at once with shared Git calls", () => + Effect.gen(function* () { + const cwd = yield* makeTempDir({ git: true }); + yield* writeTextFile(cwd, ".gitignore", "*.log\n"); + const folders = Array.from({ length: 12 }, (_, index) => `folder-${index}`); + for (const folder of folders) { + yield* writeTextFile(cwd, `${folder}/keep.txt`); + yield* writeTextFile(cwd, `${folder}/drop.log`); + } + + const { listings, calls, mostRunning } = yield* listAtOnce(cwd, folders); + expect(listings).toEqual( + folders.map((folder) => [ + { path: `${folder}/drop.log`, kind: "file", ignored: true }, + { path: `${folder}/keep.txt`, kind: "file" }, + ]), + ); + expect(mostRunning).toBe(4); + expect(calls.length).toBeLessThan(folders.length); + }), + ); + + it.effect("keeps the labels of folders checked with one that Git refuses", () => + Effect.gen(function* () { + const cwd = yield* makeTempDir({ git: true }); + yield* writeTextFile(cwd, ".gitignore", "*.log\n"); + const gitlink = yield* git(cwd, ["hash-object", ".gitignore"]); + // A submodule that was never checked out: Git refuses every path inside it, and with it + // the call it shares with the ordinary folders listed beside it. + const folders = Array.from({ length: 40 }, (_, index) => + index === 20 ? "ghost" : `folder-${index}`, + ); + for (const folder of folders) yield* writeTextFile(cwd, `${folder}/drop.log`); + yield* git(cwd, ["update-index", "--add", "--cacheinfo", `160000,${gitlink},ghost`]); + + const { listings, calls, mostRunning } = yield* listAtOnce(cwd, folders); + expect(listings).toEqual( + folders.map((folder) => [ + folder === "ghost" + ? { path: `${folder}/drop.log`, kind: "file" } + : { path: `${folder}/drop.log`, kind: "file", ignored: true }, + ]), + ); + expect( + calls.some(({ stdin }) => stdin.includes("ghost/") && stdin.includes("folder-")), + ).toBe(true); + // Asking the folders of the refused call again stays within the workspace's four calls. + expect(mostRunning).toBe(4); + }), + ); + + it.effect("keeps the Git calls of a folder that is not a repository within four", () => + Effect.gen(function* () { + const cwd = yield* makeTempDir(); + const folders = Array.from({ length: 24 }, (_, index) => `folder-${index}`); + for (const folder of folders) yield* writeTextFile(cwd, `${folder}/drop.log`); + + const { listings, mostRunning } = yield* listAtOnce(cwd, folders); + expect(listings).toEqual( + folders.map((folder) => [{ path: `${folder}/drop.log`, kind: "file" }]), + ); + expect(mostRunning).toBe(4); + }), + ); + + it.effect("checks a submodule's folders apart from the outer repository's", () => + Effect.gen(function* () { + const cwd = yield* makeTempDir({ git: true }); + yield* writeTextFile(cwd, ".gitignore", "*.log\n"); + yield* git(cwd, ["init", "sub"]); + yield* writeTextFile(cwd, "sub/.gitignore", "*.tmp\n"); + const gitlink = yield* git(cwd, ["hash-object", ".gitignore"]); + yield* git(cwd, ["update-index", "--add", "--cacheinfo", `160000,${gitlink},sub`]); + const outer = Array.from({ length: 6 }, (_, index) => `outer-${index}`); + const inner = Array.from({ length: 6 }, (_, index) => `sub/inner-${index}`); + const folders = outer.flatMap((folder, index) => [folder, inner[index]!]); + for (const folder of folders) { + yield* writeTextFile(cwd, `${folder}/drop.log`); + yield* writeTextFile(cwd, `${folder}/drop.tmp`); + } + + const { listings, calls, mostRunning } = yield* listAtOnce(cwd, folders); + expect(listings).toEqual( + folders.map((folder) => + folder.startsWith("sub/") + ? [ + { path: `${folder}/drop.log`, kind: "file" }, + { path: `${folder}/drop.tmp`, kind: "file", ignored: true }, + ] + : [ + { path: `${folder}/drop.log`, kind: "file", ignored: true }, + { path: `${folder}/drop.tmp`, kind: "file" }, + ], + ), + ); + // Calls never mix the two roots, and listings waiting to ask about `sub/.git` share one. + const path = yield* Path.Path; + const mixed = calls.filter(({ cwd: callCwd, stdin }) => { + const paths = stdin.split("\0").filter(Boolean); + return path.basename(callCwd) === "sub" + ? paths.some((checkPath) => !checkPath.startsWith("inner-")) + : paths.some((checkPath) => checkPath.startsWith("inner-")); + }); + expect(mixed).toEqual([]); + expect(calls.filter(({ stdin }) => stdin === "sub/.git\0").length).toBeLessThan( + inner.length, + ); + expect(mostRunning).toBe(4); + }), + ); + + it.effect("applies ignore rules edited between listings", () => + Effect.gen(function* () { + const cwd = yield* makeTempDir({ git: true }); + yield* writeTextFile(cwd, ".gitignore", "*.log\n"); + const folders = Array.from({ length: 8 }, (_, index) => `folder-${index}`); + for (const folder of folders) { + yield* writeTextFile(cwd, `${folder}/keep.txt`); + yield* writeTextFile(cwd, `${folder}/drop.log`); + } + yield* writeTextFile(cwd, "folder-7/.gitignore", "!drop.log\n"); + const labels = (listings: ReadonlyArray>) => + listings.flat().flatMap((entry) => (entry.ignored === true ? [entry.path] : [])); + + expect(labels((yield* listAtOnce(cwd, folders)).listings)).toEqual( + folders.slice(0, 7).map((folder) => `${folder}/drop.log`), + ); + yield* writeTextFile(cwd, ".gitignore", "*.txt\n"); + yield* writeTextFile(cwd, "folder-7/.gitignore", ""); + expect(labels((yield* listAtOnce(cwd, folders)).listings)).toEqual( + folders.map((folder) => `${folder}/keep.txt`), + ); + }), + ); + it.effect("lists a host folder and its parent, each from its own root", () => Effect.gen(function* () { const parent = yield* makeTempDir(); diff --git a/apps/server/src/workspace/WorkspaceEntries.ts b/apps/server/src/workspace/WorkspaceEntries.ts index d388170f93f5..8913a6d5595f 100644 --- a/apps/server/src/workspace/WorkspaceEntries.ts +++ b/apps/server/src/workspace/WorkspaceEntries.ts @@ -29,6 +29,7 @@ import { expandHomePath } from "@t3tools/provider-core/server/pathExpansion"; import * as VcsProcess from "../vcs/VcsProcess.ts"; import * as WorkspacePaths from "./WorkspacePaths.ts"; import * as WorkspaceSearchIndex from "./WorkspaceSearchIndex.ts"; +import { IGNORE_CHECK_CHUNK_SIZE, makeIgnoreCheckQueue } from "./workspaceIgnoreChecks.ts"; export class WorkspaceEntriesWindowsPathUnsupportedError extends Schema.TaggedError()( "WorkspaceEntriesWindowsPathUnsupportedError", @@ -114,9 +115,6 @@ export class WorkspaceEntries extends Context.Service< /** Ignored files offered for copying; more than this is a build folder, not config. */ const IGNORED_FILES_LIMIT = 200; -/** Paths sent to one `git check-ignore` call when labelling a folder's children. */ -const IGNORE_CHECK_CHUNK_SIZE = 1000; - /** Children classified at once when listing a folder; each symlink costs a realpath and a stat. */ const SYMLINK_RESOLVE_BATCH_SIZE = 64; @@ -155,6 +153,23 @@ export const make = Effect.gen(function* () { const workspaceSearchIndexes = yield* WorkspaceSearchIndex.WorkspaceSearchIndexMap; const vcsProcess = yield* VcsProcess.VcsProcess; + // stdin avoids argument limits; ignore classification is optional without git/repositories. + const runCheckIgnore = (gitCwd: string, paths: ReadonlyArray) => + vcsProcess.run({ + operation: "WorkspaceEntries.list", + command: "git", + args: ["-c", "core.fsmonitor=false", "check-ignore", "-z", "--stdin"], + cwd: gitCwd, + stdin: `${paths.join("\0")}\0`, + allowNonZeroExit: true, + timeoutMs: 10_000, + maxOutputBytes: 16 * 1024 * 1024, + }); + + // A Git process per folder costs far more than classifying its paths, so listings share calls + // once their workspace runs four. + const checkIgnore = yield* makeIgnoreCheckQueue(runCheckIgnore); + const normalizeWorkspaceRoot = Effect.fn("WorkspaceEntries.normalizeWorkspaceRoot")(function* ( cwd: string, ): Effect.fn.Return { @@ -407,23 +422,16 @@ export const make = Effect.gen(function* () { }); let gitRoot = ""; let gitPrefix = ""; - // stdin avoids argument limits; ignore classification is optional without git/repositories. - const checkIgnore = (paths: ReadonlyArray) => - vcsProcess - .run({ - operation: "WorkspaceEntries.list", - command: "git", - args: ["-c", "core.fsmonitor=false", "check-ignore", "-z", "--stdin"], - cwd: path.join(normalizedCwd, gitRoot), - stdin: `${paths.map((entryPath) => entryPath.slice(gitPrefix.length)).join("\0")}\0`, - allowNonZeroExit: true, - timeoutMs: 10_000, - maxOutputBytes: 16 * 1024 * 1024, - }) - .pipe(Effect.orElseSucceed(() => undefined)); + const checkPaths = (paths: ReadonlyArray, shareable: boolean) => + checkIgnore({ + workspace: normalizedCwd, + gitCwd: path.join(normalizedCwd, gitRoot), + paths: paths.map((entryPath) => entryPath.slice(gitPrefix.length)), + shareable, + }); // Git refuses paths inside a submodule (128) but still answers for nested clones. for (const nestedGitRoot of nestedGitRoots) { - if ((yield* checkIgnore([`${nestedGitRoot}/.git`]))?.exitCode !== 128) continue; + if ((yield* checkPaths([`${nestedGitRoot}/.git`], false))?.exitCode !== 128) continue; gitRoot = nestedGitRoot; gitPrefix = `${gitRoot}/`; } @@ -435,7 +443,7 @@ export const make = Effect.gen(function* () { directoryGitPath !== "" && !throughSymlink && !/^:|[*?[\\]/.test(directoryGitPath) && - (yield* checkIgnore([target.relativePath]))?.exitCode === 0 + (yield* checkPaths([target.relativePath], false))?.exitCode === 0 ) { return { entries: entries.map((entry) => ({ ...entry, ignored: true })), @@ -444,8 +452,10 @@ export const make = Effect.gen(function* () { } const ignored = new Set(); for (let offset = 0; offset < entries.length; offset += IGNORE_CHECK_CHUNK_SIZE) { - const result = yield* checkIgnore( + const result = yield* checkPaths( entries.slice(offset, offset + IGNORE_CHECK_CHUNK_SIZE).map((entry) => entry.path), + // Git refuses every path beyond a symlink, so such folders never share a call. + !throughSymlink, ); if (!result || (result.exitCode !== 0 && result.exitCode !== 1)) break; for (const ignoredPath of result.stdout.split("\0")) ignored.add(gitPrefix + ignoredPath); diff --git a/apps/server/src/workspace/workspaceIgnoreChecks.test.ts b/apps/server/src/workspace/workspaceIgnoreChecks.test.ts new file mode 100644 index 000000000000..8cac963529b3 --- /dev/null +++ b/apps/server/src/workspace/workspaceIgnoreChecks.test.ts @@ -0,0 +1,242 @@ +import { it, describe, expect } from "@effect/vitest"; +import * as Cause from "effect/Cause"; +import * as Deferred from "effect/Deferred"; +import * as Effect from "effect/Effect"; +import * as Exit from "effect/Exit"; +import * as Fiber from "effect/Fiber"; +import * as Queue from "effect/Queue"; +import * as Scope from "effect/Scope"; + +import { + IGNORE_CHECKS_PER_WORKSPACE, + type IgnoreCheckOutput, + makeIgnoreCheckQueue, +} from "./workspaceIgnoreChecks.ts"; + +type GitOutput = IgnoreCheckOutput & { readonly stdoutTruncated: boolean }; + +interface GitCall { + readonly paths: ReadonlyArray; + readonly outcome: Deferred.Deferred>; + interrupted: boolean; +} + +/** Git that ignores `*.log` and answers each call only when the test says so. */ +const makeGit = Effect.gen(function* () { + const started = yield* Queue.unbounded(); + const calls: Array = []; + let running = 0; + let mostRunning = 0; + const run = (_gitCwd: string, paths: ReadonlyArray) => + Effect.suspend(() => { + const call: GitCall = { paths, outcome: Deferred.makeUnsafe(), interrupted: false }; + calls.push(call); + mostRunning = Math.max(mostRunning, ++running); + Queue.offerUnsafe(started, call); + return Deferred.await(call.outcome).pipe( + Effect.flatten, + Effect.onInterrupt(() => + Effect.sync(() => { + call.interrupted = true; + }), + ), + Effect.ensuring(Effect.sync(() => running--)), + ); + }); + const answer = (call: GitCall, outcome?: Effect.Effect) => { + const ignored = call.paths.filter((checkPath) => checkPath.endsWith(".log")); + return Deferred.succeed( + call.outcome, + outcome ?? + Effect.succeed({ + exitCode: ignored.length > 0 ? 0 : 1, + stdout: ignored.map((ignoredPath) => `${ignoredPath}\0`).join(""), + stdoutTruncated: false, + }), + ); + }; + /** Waits for the next call to start. */ + const next = Queue.take(started); + /** Answers every call, started or still to come. */ + const answerAll = Effect.forEach(calls, (call) => answer(call), { discard: true }).pipe( + Effect.andThen( + Queue.take(started).pipe( + Effect.flatMap((call) => answer(call)), + Effect.forever, + Effect.forkScoped, + ), + ), + ); + const nextFour = Effect.forEach([1, 2, 3, 4], () => next); + return { run, calls, next, nextFour, answer, answerAll, mostRunning: () => mostRunning }; +}); + +const folder = (name: string, shareable = true) => ({ + workspace: "/repo", + gitCwd: "/repo", + paths: [`${name}/drop.log`, `${name}/keep.txt`], + shareable, +}); + +const labelsOf = (name: string): IgnoreCheckOutput => ({ + exitCode: 0, + stdout: `${name}/drop.log\0`, +}); + +describe("makeIgnoreCheckQueue", () => { + it.effect("frees the slots of calls whose listings all left, before or while they ran", () => + Effect.gen(function* () { + const git = yield* makeGit; + const check = yield* makeIgnoreCheckQueue(git.run); + const running = yield* Effect.forEach(["a", "b", "c", "d"], (name) => + check(folder(name)).pipe(Effect.forkChild({ startImmediately: true })), + ); + yield* git.nextFour; + const waiting = yield* Effect.forEach(["w1", "w2", "w3", "w4", "w5", "w6"], (name) => + check(folder(name)).pipe(Effect.forkChild({ startImmediately: true })), + ); + expect(git.calls.length).toBe(IGNORE_CHECKS_PER_WORKSPACE); + + // Waiting listings leave before any call starts for them; two running ones leave mid-call. + yield* Fiber.interruptAll(waiting); + yield* Fiber.interrupt(running[0]!); + yield* Fiber.interrupt(running[1]!); + expect(git.calls.map((call) => call.interrupted)).toEqual([true, true, false, false]); + + yield* git.answerAll; + const later = yield* Effect.forEach(["e", "f", "g", "h", "i", "j"], (name) => + check(folder(name)).pipe(Effect.forkChild({ startImmediately: true })), + ); + expect(yield* Fiber.joinAll([running[2]!, running[3]!, ...later])).toEqual( + ["c", "d", "e", "f", "g", "h", "i", "j"].map((name) => + expect.objectContaining(labelsOf(name)), + ), + ); + expect(git.calls.some((call) => call.paths.some((path) => path.startsWith("w")))).toBe(false); + expect(git.mostRunning()).toBe(IGNORE_CHECKS_PER_WORKSPACE); + }), + ); + + it.effect("fails the listings of a call that dies, and keeps serving the workspace", () => + Effect.gen(function* () { + const git = yield* makeGit; + const check = yield* makeIgnoreCheckQueue(git.run); + const running = yield* Effect.forEach(["a", "b", "c", "d"], (name) => + check(folder(name)).pipe(Effect.forkChild({ startImmediately: true })), + ); + const [first] = yield* git.nextFour; + const shared = yield* Effect.forEach(["e", "f", "g"], (name) => + check(folder(name)).pipe(Effect.forkChild({ startImmediately: true })), + ); + yield* git.answer(first!); + const sharedCall = yield* git.next; + expect(sharedCall.paths).toEqual(["e", "f", "g"].flatMap((name) => folder(name).paths)); + yield* git.answer(sharedCall, Effect.die("git crashed")); + + for (const fiber of shared) { + const exit = yield* Effect.exit(Fiber.join(fiber)); + expect(Exit.isFailure(exit) && Cause.hasDies(exit.cause)).toBe(true); + } + yield* git.answerAll; + expect(yield* Fiber.joinAll(running.slice(1))).toEqual( + ["b", "c", "d"].map((name) => expect.objectContaining(labelsOf(name))), + ); + expect(yield* check(folder("later"))).toEqual(expect.objectContaining(labelsOf("later"))); + }), + ); + + it.effect( + "asks each folder of a timed-out or refused shared call again alone, four at once", + () => + Effect.gen(function* () { + const failures: ReadonlyArray> = [ + Effect.fail("timed out"), + Effect.succeed({ exitCode: 128, stdout: "", stdoutTruncated: false }), + ]; + for (const failure of failures) { + const git = yield* makeGit; + const check = yield* makeIgnoreCheckQueue(git.run); + const names = Array.from({ length: 16 }, (_, index) => `folder-${index}`); + const fibers = yield* Effect.forEach(names, (name) => + check(folder(name)).pipe(Effect.forkChild({ startImmediately: true })), + ); + const [first] = yield* git.nextFour; + yield* git.answer(first!); + const sharedCall = yield* git.next; + expect(sharedCall.paths).toEqual(names.slice(4).flatMap((name) => folder(name).paths)); + yield* git.answer(sharedCall, failure); + yield* git.answerAll; + + expect(yield* Fiber.joinAll(fibers)).toEqual( + names.map((name) => expect.objectContaining(labelsOf(name))), + ); + // Every folder of the failed call got its own call, as it would without sharing. + expect(git.calls.slice(5).map((call) => call.paths)).toEqual( + names.slice(4).map((name) => folder(name).paths), + ); + expect(git.mostRunning()).toBe(IGNORE_CHECKS_PER_WORKSPACE); + } + }), + ); + + it.effect("ends waiting listings when the service closes before their call can start", () => + Effect.gen(function* () { + const git = yield* makeGit; + const scope = yield* Scope.make(); + const check = yield* makeIgnoreCheckQueue(git.run).pipe(Scope.provide(scope)); + const running = yield* Effect.forEach(["a", "b", "c", "d"], (name) => + check(folder(name)).pipe(Effect.forkChild({ startImmediately: true })), + ); + yield* git.nextFour; + // Six waiting checks, more than the four slots an unstarted call could leak. + const waiting = yield* Effect.forEach(["e", "f", "g", "h", "i", "j"], (name) => + check(folder(name, false)).pipe(Effect.forkChild({ startImmediately: true })), + ); + yield* Scope.close(scope, Exit.void); + yield* git.answerAll; + + expect(yield* Fiber.joinAll(running)).toEqual( + ["a", "b", "c", "d"].map((name) => expect.objectContaining(labelsOf(name))), + ); + for (const fiber of waiting) { + expect(Exit.isFailure(yield* Effect.exit(Fiber.join(fiber)))).toBe(true); + } + expect(git.calls.length).toBe(IGNORE_CHECKS_PER_WORKSPACE); + }), + ); + + it.effect("answers identical unshareable checks with one call and keeps others apart", () => + Effect.gen(function* () { + const git = yield* makeGit; + const check = yield* makeIgnoreCheckQueue(git.run); + yield* Effect.forEach(["a", "b", "c", "d"], (name) => + check(folder(name)).pipe(Effect.forkChild({ startImmediately: true })), + ); + const [first] = yield* git.nextFour; + const probe = { workspace: "/repo", gitCwd: "/repo", paths: ["sub/.git"], shareable: false }; + const probes = yield* Effect.forEach([1, 2, 3], () => + check(probe).pipe(Effect.forkChild({ startImmediately: true })), + ); + const linked = yield* Effect.forEach(["link-1", "link-2"], (name) => + check(folder(name, false)).pipe(Effect.forkChild({ startImmediately: true })), + ); + const refused = { exitCode: 128, stdout: "", stdoutTruncated: false }; + yield* git.answer(first!); + const probeCall = yield* git.next; + expect(probeCall.paths).toEqual(["sub/.git"]); + yield* git.answer(probeCall, Effect.succeed(refused)); + yield* git.answerAll; + + expect(yield* Fiber.joinAll(probes)).toEqual([refused, refused, refused]); + expect(yield* Fiber.joinAll(linked)).toEqual( + ["link-1", "link-2"].map((name) => expect.objectContaining(labelsOf(name))), + ); + expect(git.calls.map((call) => call.paths)).toEqual([ + ...["a", "b", "c", "d"].map((name) => folder(name).paths), + ["sub/.git"], + folder("link-1").paths, + folder("link-2").paths, + ]); + }), + ); +}); diff --git a/apps/server/src/workspace/workspaceIgnoreChecks.ts b/apps/server/src/workspace/workspaceIgnoreChecks.ts new file mode 100644 index 000000000000..2dc03cb7cec5 --- /dev/null +++ b/apps/server/src/workspace/workspaceIgnoreChecks.ts @@ -0,0 +1,214 @@ +import * as Deferred from "effect/Deferred"; +import * as Effect from "effect/Effect"; +import * as Exit from "effect/Exit"; +import * as Fiber from "effect/Fiber"; + +/** Paths in one `git check-ignore` call, which bounds its stdin and stdout. */ +export const IGNORE_CHECK_CHUNK_SIZE = 1000; + +/** Ignore checks of one workspace running at once; checks beyond this wait and share a call. */ +export const IGNORE_CHECKS_PER_WORKSPACE = 4; + +export interface IgnoreCheckOutput { + readonly exitCode: number; + readonly stdout: string; +} + +export interface IgnoreCheckRequest { + readonly workspace: string; + readonly gitCwd: string; + readonly paths: ReadonlyArray; + /** False when Git's answer can't be split per folder, as for paths beyond a symlink. */ + readonly shareable: boolean; +} + +interface Call { + waiting: number; + fiber: Fiber.Fiber | undefined; +} + +interface Check { + readonly gitCwd: string; + readonly paths: ReadonlyArray; + readonly key: string; + readonly done: Deferred.Deferred; + shareable: boolean; + abandoned: boolean; + call: Call | undefined; +} + +interface Workspace { + running: number; + queued: Array; +} + +/** + * Runs `git check-ignore -z --stdin` for listings, at most four calls per workspace. Checks that + * arrive while four run wait, and the next call answers all of them for one Git root; identical + * unshareable checks share one call. Nothing is reused: a call starts after all its checks arrived. + */ +export const makeIgnoreCheckQueue = ( + run: ( + gitCwd: string, + paths: ReadonlyArray, + ) => Effect.Effect, +) => + Effect.map(Effect.scope, (scope) => { + const workspaces = new Map(); + + const answer = Effect.fn("WorkspaceEntries.answerIgnoreChecks")(function* ( + workspace: Workspace, + checks: Array, + ) { + const shared = checks.length > 1 && checks[0]!.shareable; + const result = yield* run( + checks[0]!.gitCwd, + shared ? checks.flatMap((check) => check.paths) : checks[0]!.paths, + ).pipe(Effect.result); + if (!shared) { + const output = result._tag === "Success" ? result.success : undefined; + for (const check of checks) yield* Deferred.succeed(check.done, output); + return; + } + if ( + result._tag === "Success" && + (result.success.exitCode === 0 || result.success.exitCode === 1) && + !result.success.stdoutTruncated + ) { + // Git echoes each ignored path as it was given, so each folder takes back its own. + const ignored = new Set(result.success.stdout.split("\0")); + for (const check of checks) { + const own = check.paths.filter((checkPath) => ignored.has(checkPath)); + yield* Deferred.succeed(check.done, { + exitCode: own.length > 0 ? 0 : 1, + stdout: own.map((ignoredPath) => `${ignoredPath}\0`).join(""), + }); + } + return; + } + // One path Git refuses (inside a submodule, outside the repository) fails the whole call, + // and a timeout may be the batch's fault. Each folder is asked again alone, within the limit. + const retries = checks.filter((check) => !check.abandoned); + for (const check of retries) { + check.call = undefined; + check.shareable = false; + } + workspace.queued.unshift(...retries); + }); + + const release = (key: string, workspace: Workspace) => + Effect.suspend(() => { + workspace.running -= 1; + if (workspace.running === 0 && workspace.queued.length === 0) workspaces.delete(key); + return start(key, workspace); + }); + + const start = (key: string, workspace: Workspace): Effect.Effect => + Effect.suspend(() => { + const calls: Array> = []; + while (workspace.running < IGNORE_CHECKS_PER_WORKSPACE && workspace.queued.length > 0) { + const head = workspace.queued[0]!; + const checks: Array = []; + let pathCount = 0; + workspace.queued = workspace.queued.filter((check) => { + const joins = head.shareable + ? check.shareable && + check.gitCwd === head.gitCwd && + (checks.length === 0 || pathCount + check.paths.length <= IGNORE_CHECK_CHUNK_SIZE) + : !check.shareable && check.key === head.key; + if (!joins) return true; + checks.push(check); + pathCount += check.paths.length; + return false; + }); + workspace.running += 1; + calls.push(checks); + } + return Effect.forEach( + calls, + (checks) => { + const call: Call = { waiting: checks.length, fiber: undefined }; + for (const check of checks) check.call = call; + const finish = (exit: Exit.Exit) => { + const outcome = Exit.isSuccess(exit) + ? Exit.succeed(undefined) + : Exit.failCause(exit.cause); + return Effect.forEach( + checks, + (check) => (check.call === call ? Deferred.done(check.done, outcome) : Effect.void), + { discard: true }, + ).pipe(Effect.andThen(release(key, workspace))); + }; + let started = false; + // Started at once, so the finalizer is in place before anything can interrupt the call. + return Effect.suspend(() => { + started = true; + return answer(workspace, checks); + }).pipe( + Effect.onExit(finish), + Effect.forkIn(scope, { startImmediately: true }), + Effect.flatMap((fiber) => { + call.fiber = fiber; + // A closed scope returns a fiber that never ran, so its finalizer never will. + return started + ? Effect.void + : Effect.exit(Fiber.join(fiber)).pipe(Effect.flatMap(finish)); + }), + ); + }, + { discard: true }, + ); + }); + + const abandon = (workspace: Workspace, check: Check) => + Effect.suspend(() => { + check.abandoned = true; + const call = check.call; + if (call === undefined) { + workspace.queued = workspace.queued.filter((queued) => queued !== check); + return Effect.void; + } + call.waiting -= 1; + // Stop Git once no listing waits for its answer. + return call.waiting === 0 && call.fiber !== undefined + ? Fiber.interrupt(call.fiber) + : Effect.void; + }); + + return (request: IgnoreCheckRequest): Effect.Effect => + Effect.uninterruptibleMask((restore) => + Effect.suspend(() => { + let workspace = workspaces.get(request.workspace); + if (workspace === undefined) { + workspace = { running: 0, queued: [] }; + workspaces.set(request.workspace, workspace); + } + // A free slot means nothing waits, so this call has no one to share with: it runs here, + // and interrupting this listing stops its Git process. + if (workspace.running < IGNORE_CHECKS_PER_WORKSPACE) { + workspace.running += 1; + return restore( + run(request.gitCwd, request.paths).pipe(Effect.orElseSucceed(() => undefined)), + ).pipe(Effect.ensuring(release(request.workspace, workspace))); + } + const check: Check = { + gitCwd: request.gitCwd, + paths: request.paths, + key: `${request.gitCwd}\0${request.paths.join("\0")}`, + done: Deferred.makeUnsafe(), + shareable: request.shareable, + abandoned: false, + call: undefined, + }; + workspace.queued.push(check); + const own = workspace; + return start(request.workspace, workspace).pipe( + Effect.andThen( + restore(Deferred.await(check.done)).pipe( + Effect.onInterrupt(() => abandon(own, check)), + ), + ), + ); + }), + ); + }); From 96c639e4e15606929120953c98b9716fdccc852c Mon Sep 17 00:00:00 2001 From: Pyseph <45090858+Pyseph@users.noreply.github.com> Date: Thu, 5 Nov 2026 02:36:25 +0100 Subject: [PATCH 2/3] perf(web): keep 64 Files folder reads in flight With four reads in flight the server never holds more than four folders, so it can't share their Git ignore checks. Keeping 64 in flight lets Expand all use the shared calls: 138.4 s -> 24.1 s for this repository on Windows at the service boundary (medians, n=7). --- .../files/useDirectoryEntries.browser.test.tsx | 16 ++++++++-------- .../src/components/files/useDirectoryEntries.ts | 6 +++++- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/apps/web/src/components/files/useDirectoryEntries.browser.test.tsx b/apps/web/src/components/files/useDirectoryEntries.browser.test.tsx index 046563bcb22f..983d14c8127b 100644 --- a/apps/web/src/components/files/useDirectoryEntries.browser.test.tsx +++ b/apps/web/src/components/files/useDirectoryEntries.browser.test.tsx @@ -37,7 +37,7 @@ vi.mock("~/state/projects", () => ({ }, })); -import { useDirectoryEntries } from "./useDirectoryEntries"; +import { FOLDER_READS_IN_FLIGHT, useDirectoryEntries } from "./useDirectoryEntries"; function setup() { mocks.reconnect = false; @@ -144,8 +144,8 @@ it("shares the follow-up promise and does not resolve callers before the final l expect(completed).toBe(true); }); -it("retains the four-query concurrency bound across directories and refreshes", async () => { - const folders = Array.from({ length: 8 }, (_, index) => ({ +it("retains the query concurrency bound across directories and refreshes", async () => { + const folders = Array.from({ length: FOLDER_READS_IN_FLIGHT + 4 }, (_, index) => ({ path: `folder-${index}`, kind: "directory" as const, })); @@ -163,17 +163,17 @@ it("retains the four-query concurrency bound across directories and refreshes", const refreshStart = state.calls.length; state.resetPeakQueries(); await act(async () => state.view.refresh()); - expect(state.peakQueries).toBe(4); - expect(state.activeQueries).toBe(4); + expect(state.peakQueries).toBe(FOLDER_READS_IN_FLIGHT); + expect(state.activeQueries).toBe(FOLDER_READS_IN_FLIGHT); await act(async () => { for (let index = 0; index < 20; index++) state.view.refresh(); }); for (let index = refreshStart; index < state.calls.length; index++) { await state.settle(index, state.calls[index]!.path === "" ? folders : []); - expect(state.activeQueries).toBeLessThanOrEqual(4); + expect(state.activeQueries).toBeLessThanOrEqual(FOLDER_READS_IN_FLIGHT); } - expect(state.peakQueries).toBe(4); - expect(new Set(state.calls.map((call) => call.path)).size).toBe(9); + expect(state.peakQueries).toBe(FOLDER_READS_IN_FLIGHT); + expect(new Set(state.calls.map((call) => call.path)).size).toBe(folders.length + 1); for (const path of ["", ...folders.map((folder) => folder.path)]) { expect(state.calls.filter((call) => call.path === path).length).toBeLessThanOrEqual(3); } diff --git a/apps/web/src/components/files/useDirectoryEntries.ts b/apps/web/src/components/files/useDirectoryEntries.ts index 32aa2fe46675..8a4baab5f232 100644 --- a/apps/web/src/components/files/useDirectoryEntries.ts +++ b/apps/web/src/components/files/useDirectoryEntries.ts @@ -8,6 +8,9 @@ import { environmentCatalog } from "~/connection/catalog"; import { appAtomRegistry } from "~/rpc/atomRegistry"; import { projectEnvironment } from "~/state/projects"; +/** Folder reads in flight; the server shares one Git ignore check among reads beyond its four. */ +export const FOLDER_READS_IN_FLIGHT = 64; + export function useDirectoryEntries(environmentId: EnvironmentId, cwd: string) { const [directories, setDirectories] = useState(new Map()); const [errors, setErrors] = useState(new Map()); @@ -28,7 +31,8 @@ export function useDirectoryEntries(environmentId: EnvironmentId, cwd: string) { requested.current.add(directoryPath); const atom = projectEnvironment.listEntries({ environmentId, input: { cwd, directoryPath } }); const request = (async () => { - if (running.current >= 4) await new Promise((resolve) => waiting.current.push(resolve)); + if (running.current >= FOLDER_READS_IN_FLIGHT) + await new Promise((resolve) => waiting.current.push(resolve)); else running.current++; const result = active.current ? await executeAtomQuery(appAtomRegistry, atom, { From 3ed7095f086cd94b3dc0b489177aa842a478f698 Mon Sep 17 00:00:00 2001 From: Pyseph <45090858+Pyseph@users.noreply.github.com> Date: Thu, 5 Nov 2026 02:53:13 +0100 Subject: [PATCH 3/3] test(web): pin the 64 folder reads in the test The test imported the production constant, so lowering the limit moved the workload and the expected value together and the test still passed. --- .../files/useDirectoryEntries.browser.test.tsx | 14 ++++++++------ .../src/components/files/useDirectoryEntries.ts | 2 +- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/apps/web/src/components/files/useDirectoryEntries.browser.test.tsx b/apps/web/src/components/files/useDirectoryEntries.browser.test.tsx index 983d14c8127b..5224d5de78f3 100644 --- a/apps/web/src/components/files/useDirectoryEntries.browser.test.tsx +++ b/apps/web/src/components/files/useDirectoryEntries.browser.test.tsx @@ -37,7 +37,7 @@ vi.mock("~/state/projects", () => ({ }, })); -import { FOLDER_READS_IN_FLIGHT, useDirectoryEntries } from "./useDirectoryEntries"; +import { useDirectoryEntries } from "./useDirectoryEntries"; function setup() { mocks.reconnect = false; @@ -145,7 +145,9 @@ it("shares the follow-up promise and does not resolve callers before the final l }); it("retains the query concurrency bound across directories and refreshes", async () => { - const folders = Array.from({ length: FOLDER_READS_IN_FLIGHT + 4 }, (_, index) => ({ + // Owned by the test, so lowering the production limit fails it instead of moving the workload too. + const expectedFolderReads = 64; + const folders = Array.from({ length: expectedFolderReads + 4 }, (_, index) => ({ path: `folder-${index}`, kind: "directory" as const, })); @@ -163,16 +165,16 @@ it("retains the query concurrency bound across directories and refreshes", async const refreshStart = state.calls.length; state.resetPeakQueries(); await act(async () => state.view.refresh()); - expect(state.peakQueries).toBe(FOLDER_READS_IN_FLIGHT); - expect(state.activeQueries).toBe(FOLDER_READS_IN_FLIGHT); + expect(state.peakQueries).toBe(expectedFolderReads); + expect(state.activeQueries).toBe(expectedFolderReads); await act(async () => { for (let index = 0; index < 20; index++) state.view.refresh(); }); for (let index = refreshStart; index < state.calls.length; index++) { await state.settle(index, state.calls[index]!.path === "" ? folders : []); - expect(state.activeQueries).toBeLessThanOrEqual(FOLDER_READS_IN_FLIGHT); + expect(state.activeQueries).toBeLessThanOrEqual(expectedFolderReads); } - expect(state.peakQueries).toBe(FOLDER_READS_IN_FLIGHT); + expect(state.peakQueries).toBe(expectedFolderReads); expect(new Set(state.calls.map((call) => call.path)).size).toBe(folders.length + 1); for (const path of ["", ...folders.map((folder) => folder.path)]) { expect(state.calls.filter((call) => call.path === path).length).toBeLessThanOrEqual(3); diff --git a/apps/web/src/components/files/useDirectoryEntries.ts b/apps/web/src/components/files/useDirectoryEntries.ts index 8a4baab5f232..6db3aa964a45 100644 --- a/apps/web/src/components/files/useDirectoryEntries.ts +++ b/apps/web/src/components/files/useDirectoryEntries.ts @@ -9,7 +9,7 @@ import { appAtomRegistry } from "~/rpc/atomRegistry"; import { projectEnvironment } from "~/state/projects"; /** Folder reads in flight; the server shares one Git ignore check among reads beyond its four. */ -export const FOLDER_READS_IN_FLIGHT = 64; +const FOLDER_READS_IN_FLIGHT = 64; export function useDirectoryEntries(environmentId: EnvironmentId, cwd: string) { const [directories, setDirectories] = useState(new Map());