diff --git a/apps/server/src/workspace/WorkspaceEntries.test.ts b/apps/server/src/workspace/WorkspaceEntries.test.ts index eedff4fc79c9..9dd76906fa94 100644 --- a/apps/server/src/workspace/WorkspaceEntries.test.ts +++ b/apps/server/src/workspace/WorkspaceEntries.test.ts @@ -2,7 +2,9 @@ import * as NodeFSP from "node:fs/promises"; import * as NodeServices from "@effect/platform-node/NodeServices"; import { FileFinder } from "@ff-labs/fff-node"; +import type { ProjectEntry } from "@t3tools/contracts"; import { it, afterEach, describe, expect } from "@effect/vitest"; +import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; @@ -15,6 +17,7 @@ import * as HostProcess from "@t3tools/shared/HostProcess"; import * as VcsProcess from "../vcs/VcsProcess.ts"; import * as WorkspaceEntries from "./WorkspaceEntries.ts"; import * as WorkspacePaths from "./WorkspacePaths.ts"; +import * as WorkspaceSearchIndex from "./WorkspaceSearchIndex.ts"; vi.mock("node:fs/promises", async (importOriginal) => { const actual = await importOriginal(); @@ -83,6 +86,58 @@ const searchWorkspaceEntries = (input: { return yield* workspaceEntries.search(input); }); +/** + * Lists folders at once, as "Expand all" does with a wide client. Git calls are held until four + * have started, so later folders find four checks running; records each call's cwd and stdin. + */ +const listAtOnce = (cwd: string, folders: ReadonlyArray) => + Effect.gen(function* () { + const fourStarted = yield* Deferred.make(); + const calls: Array<{ readonly cwd: string; readonly stdin: string }> = []; + let running = 0; + let mostRunning = 0; + const vcsProcess = Layer.effect( + VcsProcess.VcsProcess, + Effect.map(VcsProcess.VcsProcess, (inner) => + VcsProcess.VcsProcess.of({ + run: (input) => + input.args.includes("check-ignore") + ? Effect.suspend(() => { + calls.push({ cwd: input.cwd, stdin: input.stdin ?? "" }); + mostRunning = Math.max(mostRunning, ++running); + if (calls.length === 4) Deferred.doneUnsafe(fourStarted, Effect.void); + return Deferred.await(fourStarted).pipe( + Effect.andThen(inner.run(input)), + Effect.ensuring(Effect.sync(() => running--)), + ); + }) + : inner.run(input), + }), + ), + ).pipe(Layer.provide(VcsProcess.layer)); + const listings = yield* Effect.forEach( + folders, + (directoryPath) => + Effect.flatMap(WorkspaceEntries.WorkspaceEntries, (workspaceEntries) => + workspaceEntries.list({ cwd, directoryPath }), + ).pipe( + Effect.map(({ entries }) => + entries.toSorted((left, right) => left.path.localeCompare(right.path)), + ), + ), + { concurrency: "unbounded" }, + ).pipe( + Effect.provide( + Layer.effect(WorkspaceEntries.WorkspaceEntries, WorkspaceEntries.make).pipe( + Layer.provide(WorkspaceSearchIndex.WorkspaceSearchIndexMap.layer), + Layer.provide(vcsProcess), + Layer.provide(WorkspacePaths.layer), + ), + ), + ); + return { listings, calls, mostRunning }; + }); + const appendSeparator = (input: string) => Effect.map(HostProcess.Platform, (platform) => input.endsWith("/") || input.endsWith("\\") @@ -328,6 +383,141 @@ it.layer(layerTest, { excludeTestServices: true })("WorkspaceEntries", (it) => { }), ); + it.effect("labels folders listed beyond four at once with shared Git calls", () => + Effect.gen(function* () { + const cwd = yield* makeTempDir({ git: true }); + yield* writeTextFile(cwd, ".gitignore", "*.log\n"); + const folders = Array.from({ length: 12 }, (_, index) => `folder-${index}`); + for (const folder of folders) { + yield* writeTextFile(cwd, `${folder}/keep.txt`); + yield* writeTextFile(cwd, `${folder}/drop.log`); + } + + const { listings, calls, mostRunning } = yield* listAtOnce(cwd, folders); + expect(listings).toEqual( + folders.map((folder) => [ + { path: `${folder}/drop.log`, kind: "file", ignored: true }, + { path: `${folder}/keep.txt`, kind: "file" }, + ]), + ); + expect(mostRunning).toBe(4); + expect(calls.length).toBeLessThan(folders.length); + }), + ); + + it.effect("keeps the labels of folders checked with one that Git refuses", () => + Effect.gen(function* () { + const cwd = yield* makeTempDir({ git: true }); + yield* writeTextFile(cwd, ".gitignore", "*.log\n"); + const gitlink = yield* git(cwd, ["hash-object", ".gitignore"]); + // A submodule that was never checked out: Git refuses every path inside it, and with it + // the call it shares with the ordinary folders listed beside it. + const folders = Array.from({ length: 40 }, (_, index) => + index === 20 ? "ghost" : `folder-${index}`, + ); + for (const folder of folders) yield* writeTextFile(cwd, `${folder}/drop.log`); + yield* git(cwd, ["update-index", "--add", "--cacheinfo", `160000,${gitlink},ghost`]); + + const { listings, calls, mostRunning } = yield* listAtOnce(cwd, folders); + expect(listings).toEqual( + folders.map((folder) => [ + folder === "ghost" + ? { path: `${folder}/drop.log`, kind: "file" } + : { path: `${folder}/drop.log`, kind: "file", ignored: true }, + ]), + ); + expect( + calls.some(({ stdin }) => stdin.includes("ghost/") && stdin.includes("folder-")), + ).toBe(true); + // Asking the folders of the refused call again stays within the workspace's four calls. + expect(mostRunning).toBe(4); + }), + ); + + it.effect("keeps the Git calls of a folder that is not a repository within four", () => + Effect.gen(function* () { + const cwd = yield* makeTempDir(); + const folders = Array.from({ length: 24 }, (_, index) => `folder-${index}`); + for (const folder of folders) yield* writeTextFile(cwd, `${folder}/drop.log`); + + const { listings, mostRunning } = yield* listAtOnce(cwd, folders); + expect(listings).toEqual( + folders.map((folder) => [{ path: `${folder}/drop.log`, kind: "file" }]), + ); + expect(mostRunning).toBe(4); + }), + ); + + it.effect("checks a submodule's folders apart from the outer repository's", () => + Effect.gen(function* () { + const cwd = yield* makeTempDir({ git: true }); + yield* writeTextFile(cwd, ".gitignore", "*.log\n"); + yield* git(cwd, ["init", "sub"]); + yield* writeTextFile(cwd, "sub/.gitignore", "*.tmp\n"); + const gitlink = yield* git(cwd, ["hash-object", ".gitignore"]); + yield* git(cwd, ["update-index", "--add", "--cacheinfo", `160000,${gitlink},sub`]); + const outer = Array.from({ length: 6 }, (_, index) => `outer-${index}`); + const inner = Array.from({ length: 6 }, (_, index) => `sub/inner-${index}`); + const folders = outer.flatMap((folder, index) => [folder, inner[index]!]); + for (const folder of folders) { + yield* writeTextFile(cwd, `${folder}/drop.log`); + yield* writeTextFile(cwd, `${folder}/drop.tmp`); + } + + const { listings, calls, mostRunning } = yield* listAtOnce(cwd, folders); + expect(listings).toEqual( + folders.map((folder) => + folder.startsWith("sub/") + ? [ + { path: `${folder}/drop.log`, kind: "file" }, + { path: `${folder}/drop.tmp`, kind: "file", ignored: true }, + ] + : [ + { path: `${folder}/drop.log`, kind: "file", ignored: true }, + { path: `${folder}/drop.tmp`, kind: "file" }, + ], + ), + ); + // Calls never mix the two roots, and listings waiting to ask about `sub/.git` share one. + const path = yield* Path.Path; + const mixed = calls.filter(({ cwd: callCwd, stdin }) => { + const paths = stdin.split("\0").filter(Boolean); + return path.basename(callCwd) === "sub" + ? paths.some((checkPath) => !checkPath.startsWith("inner-")) + : paths.some((checkPath) => checkPath.startsWith("inner-")); + }); + expect(mixed).toEqual([]); + expect(calls.filter(({ stdin }) => stdin === "sub/.git\0").length).toBeLessThan( + inner.length, + ); + expect(mostRunning).toBe(4); + }), + ); + + it.effect("applies ignore rules edited between listings", () => + Effect.gen(function* () { + const cwd = yield* makeTempDir({ git: true }); + yield* writeTextFile(cwd, ".gitignore", "*.log\n"); + const folders = Array.from({ length: 8 }, (_, index) => `folder-${index}`); + for (const folder of folders) { + yield* writeTextFile(cwd, `${folder}/keep.txt`); + yield* writeTextFile(cwd, `${folder}/drop.log`); + } + yield* writeTextFile(cwd, "folder-7/.gitignore", "!drop.log\n"); + const labels = (listings: ReadonlyArray>) => + listings.flat().flatMap((entry) => (entry.ignored === true ? [entry.path] : [])); + + expect(labels((yield* listAtOnce(cwd, folders)).listings)).toEqual( + folders.slice(0, 7).map((folder) => `${folder}/drop.log`), + ); + yield* writeTextFile(cwd, ".gitignore", "*.txt\n"); + yield* writeTextFile(cwd, "folder-7/.gitignore", ""); + expect(labels((yield* listAtOnce(cwd, folders)).listings)).toEqual( + folders.map((folder) => `${folder}/keep.txt`), + ); + }), + ); + it.effect("lists a host folder and its parent, each from its own root", () => Effect.gen(function* () { const parent = yield* makeTempDir(); diff --git a/apps/server/src/workspace/WorkspaceEntries.ts b/apps/server/src/workspace/WorkspaceEntries.ts index d388170f93f5..8913a6d5595f 100644 --- a/apps/server/src/workspace/WorkspaceEntries.ts +++ b/apps/server/src/workspace/WorkspaceEntries.ts @@ -29,6 +29,7 @@ import { expandHomePath } from "@t3tools/provider-core/server/pathExpansion"; import * as VcsProcess from "../vcs/VcsProcess.ts"; import * as WorkspacePaths from "./WorkspacePaths.ts"; import * as WorkspaceSearchIndex from "./WorkspaceSearchIndex.ts"; +import { IGNORE_CHECK_CHUNK_SIZE, makeIgnoreCheckQueue } from "./workspaceIgnoreChecks.ts"; export class WorkspaceEntriesWindowsPathUnsupportedError extends Schema.TaggedError()( "WorkspaceEntriesWindowsPathUnsupportedError", @@ -114,9 +115,6 @@ export class WorkspaceEntries extends Context.Service< /** Ignored files offered for copying; more than this is a build folder, not config. */ const IGNORED_FILES_LIMIT = 200; -/** Paths sent to one `git check-ignore` call when labelling a folder's children. */ -const IGNORE_CHECK_CHUNK_SIZE = 1000; - /** Children classified at once when listing a folder; each symlink costs a realpath and a stat. */ const SYMLINK_RESOLVE_BATCH_SIZE = 64; @@ -155,6 +153,23 @@ export const make = Effect.gen(function* () { const workspaceSearchIndexes = yield* WorkspaceSearchIndex.WorkspaceSearchIndexMap; const vcsProcess = yield* VcsProcess.VcsProcess; + // stdin avoids argument limits; ignore classification is optional without git/repositories. + const runCheckIgnore = (gitCwd: string, paths: ReadonlyArray) => + vcsProcess.run({ + operation: "WorkspaceEntries.list", + command: "git", + args: ["-c", "core.fsmonitor=false", "check-ignore", "-z", "--stdin"], + cwd: gitCwd, + stdin: `${paths.join("\0")}\0`, + allowNonZeroExit: true, + timeoutMs: 10_000, + maxOutputBytes: 16 * 1024 * 1024, + }); + + // A Git process per folder costs far more than classifying its paths, so listings share calls + // once their workspace runs four. + const checkIgnore = yield* makeIgnoreCheckQueue(runCheckIgnore); + const normalizeWorkspaceRoot = Effect.fn("WorkspaceEntries.normalizeWorkspaceRoot")(function* ( cwd: string, ): Effect.fn.Return { @@ -407,23 +422,16 @@ export const make = Effect.gen(function* () { }); let gitRoot = ""; let gitPrefix = ""; - // stdin avoids argument limits; ignore classification is optional without git/repositories. - const checkIgnore = (paths: ReadonlyArray) => - vcsProcess - .run({ - operation: "WorkspaceEntries.list", - command: "git", - args: ["-c", "core.fsmonitor=false", "check-ignore", "-z", "--stdin"], - cwd: path.join(normalizedCwd, gitRoot), - stdin: `${paths.map((entryPath) => entryPath.slice(gitPrefix.length)).join("\0")}\0`, - allowNonZeroExit: true, - timeoutMs: 10_000, - maxOutputBytes: 16 * 1024 * 1024, - }) - .pipe(Effect.orElseSucceed(() => undefined)); + const checkPaths = (paths: ReadonlyArray, shareable: boolean) => + checkIgnore({ + workspace: normalizedCwd, + gitCwd: path.join(normalizedCwd, gitRoot), + paths: paths.map((entryPath) => entryPath.slice(gitPrefix.length)), + shareable, + }); // Git refuses paths inside a submodule (128) but still answers for nested clones. for (const nestedGitRoot of nestedGitRoots) { - if ((yield* checkIgnore([`${nestedGitRoot}/.git`]))?.exitCode !== 128) continue; + if ((yield* checkPaths([`${nestedGitRoot}/.git`], false))?.exitCode !== 128) continue; gitRoot = nestedGitRoot; gitPrefix = `${gitRoot}/`; } @@ -435,7 +443,7 @@ export const make = Effect.gen(function* () { directoryGitPath !== "" && !throughSymlink && !/^:|[*?[\\]/.test(directoryGitPath) && - (yield* checkIgnore([target.relativePath]))?.exitCode === 0 + (yield* checkPaths([target.relativePath], false))?.exitCode === 0 ) { return { entries: entries.map((entry) => ({ ...entry, ignored: true })), @@ -444,8 +452,10 @@ export const make = Effect.gen(function* () { } const ignored = new Set(); for (let offset = 0; offset < entries.length; offset += IGNORE_CHECK_CHUNK_SIZE) { - const result = yield* checkIgnore( + const result = yield* checkPaths( entries.slice(offset, offset + IGNORE_CHECK_CHUNK_SIZE).map((entry) => entry.path), + // Git refuses every path beyond a symlink, so such folders never share a call. + !throughSymlink, ); if (!result || (result.exitCode !== 0 && result.exitCode !== 1)) break; for (const ignoredPath of result.stdout.split("\0")) ignored.add(gitPrefix + ignoredPath); diff --git a/apps/server/src/workspace/workspaceIgnoreChecks.test.ts b/apps/server/src/workspace/workspaceIgnoreChecks.test.ts new file mode 100644 index 000000000000..8cac963529b3 --- /dev/null +++ b/apps/server/src/workspace/workspaceIgnoreChecks.test.ts @@ -0,0 +1,242 @@ +import { it, describe, expect } from "@effect/vitest"; +import * as Cause from "effect/Cause"; +import * as Deferred from "effect/Deferred"; +import * as Effect from "effect/Effect"; +import * as Exit from "effect/Exit"; +import * as Fiber from "effect/Fiber"; +import * as Queue from "effect/Queue"; +import * as Scope from "effect/Scope"; + +import { + IGNORE_CHECKS_PER_WORKSPACE, + type IgnoreCheckOutput, + makeIgnoreCheckQueue, +} from "./workspaceIgnoreChecks.ts"; + +type GitOutput = IgnoreCheckOutput & { readonly stdoutTruncated: boolean }; + +interface GitCall { + readonly paths: ReadonlyArray; + readonly outcome: Deferred.Deferred>; + interrupted: boolean; +} + +/** Git that ignores `*.log` and answers each call only when the test says so. */ +const makeGit = Effect.gen(function* () { + const started = yield* Queue.unbounded(); + const calls: Array = []; + let running = 0; + let mostRunning = 0; + const run = (_gitCwd: string, paths: ReadonlyArray) => + Effect.suspend(() => { + const call: GitCall = { paths, outcome: Deferred.makeUnsafe(), interrupted: false }; + calls.push(call); + mostRunning = Math.max(mostRunning, ++running); + Queue.offerUnsafe(started, call); + return Deferred.await(call.outcome).pipe( + Effect.flatten, + Effect.onInterrupt(() => + Effect.sync(() => { + call.interrupted = true; + }), + ), + Effect.ensuring(Effect.sync(() => running--)), + ); + }); + const answer = (call: GitCall, outcome?: Effect.Effect) => { + const ignored = call.paths.filter((checkPath) => checkPath.endsWith(".log")); + return Deferred.succeed( + call.outcome, + outcome ?? + Effect.succeed({ + exitCode: ignored.length > 0 ? 0 : 1, + stdout: ignored.map((ignoredPath) => `${ignoredPath}\0`).join(""), + stdoutTruncated: false, + }), + ); + }; + /** Waits for the next call to start. */ + const next = Queue.take(started); + /** Answers every call, started or still to come. */ + const answerAll = Effect.forEach(calls, (call) => answer(call), { discard: true }).pipe( + Effect.andThen( + Queue.take(started).pipe( + Effect.flatMap((call) => answer(call)), + Effect.forever, + Effect.forkScoped, + ), + ), + ); + const nextFour = Effect.forEach([1, 2, 3, 4], () => next); + return { run, calls, next, nextFour, answer, answerAll, mostRunning: () => mostRunning }; +}); + +const folder = (name: string, shareable = true) => ({ + workspace: "/repo", + gitCwd: "/repo", + paths: [`${name}/drop.log`, `${name}/keep.txt`], + shareable, +}); + +const labelsOf = (name: string): IgnoreCheckOutput => ({ + exitCode: 0, + stdout: `${name}/drop.log\0`, +}); + +describe("makeIgnoreCheckQueue", () => { + it.effect("frees the slots of calls whose listings all left, before or while they ran", () => + Effect.gen(function* () { + const git = yield* makeGit; + const check = yield* makeIgnoreCheckQueue(git.run); + const running = yield* Effect.forEach(["a", "b", "c", "d"], (name) => + check(folder(name)).pipe(Effect.forkChild({ startImmediately: true })), + ); + yield* git.nextFour; + const waiting = yield* Effect.forEach(["w1", "w2", "w3", "w4", "w5", "w6"], (name) => + check(folder(name)).pipe(Effect.forkChild({ startImmediately: true })), + ); + expect(git.calls.length).toBe(IGNORE_CHECKS_PER_WORKSPACE); + + // Waiting listings leave before any call starts for them; two running ones leave mid-call. + yield* Fiber.interruptAll(waiting); + yield* Fiber.interrupt(running[0]!); + yield* Fiber.interrupt(running[1]!); + expect(git.calls.map((call) => call.interrupted)).toEqual([true, true, false, false]); + + yield* git.answerAll; + const later = yield* Effect.forEach(["e", "f", "g", "h", "i", "j"], (name) => + check(folder(name)).pipe(Effect.forkChild({ startImmediately: true })), + ); + expect(yield* Fiber.joinAll([running[2]!, running[3]!, ...later])).toEqual( + ["c", "d", "e", "f", "g", "h", "i", "j"].map((name) => + expect.objectContaining(labelsOf(name)), + ), + ); + expect(git.calls.some((call) => call.paths.some((path) => path.startsWith("w")))).toBe(false); + expect(git.mostRunning()).toBe(IGNORE_CHECKS_PER_WORKSPACE); + }), + ); + + it.effect("fails the listings of a call that dies, and keeps serving the workspace", () => + Effect.gen(function* () { + const git = yield* makeGit; + const check = yield* makeIgnoreCheckQueue(git.run); + const running = yield* Effect.forEach(["a", "b", "c", "d"], (name) => + check(folder(name)).pipe(Effect.forkChild({ startImmediately: true })), + ); + const [first] = yield* git.nextFour; + const shared = yield* Effect.forEach(["e", "f", "g"], (name) => + check(folder(name)).pipe(Effect.forkChild({ startImmediately: true })), + ); + yield* git.answer(first!); + const sharedCall = yield* git.next; + expect(sharedCall.paths).toEqual(["e", "f", "g"].flatMap((name) => folder(name).paths)); + yield* git.answer(sharedCall, Effect.die("git crashed")); + + for (const fiber of shared) { + const exit = yield* Effect.exit(Fiber.join(fiber)); + expect(Exit.isFailure(exit) && Cause.hasDies(exit.cause)).toBe(true); + } + yield* git.answerAll; + expect(yield* Fiber.joinAll(running.slice(1))).toEqual( + ["b", "c", "d"].map((name) => expect.objectContaining(labelsOf(name))), + ); + expect(yield* check(folder("later"))).toEqual(expect.objectContaining(labelsOf("later"))); + }), + ); + + it.effect( + "asks each folder of a timed-out or refused shared call again alone, four at once", + () => + Effect.gen(function* () { + const failures: ReadonlyArray> = [ + Effect.fail("timed out"), + Effect.succeed({ exitCode: 128, stdout: "", stdoutTruncated: false }), + ]; + for (const failure of failures) { + const git = yield* makeGit; + const check = yield* makeIgnoreCheckQueue(git.run); + const names = Array.from({ length: 16 }, (_, index) => `folder-${index}`); + const fibers = yield* Effect.forEach(names, (name) => + check(folder(name)).pipe(Effect.forkChild({ startImmediately: true })), + ); + const [first] = yield* git.nextFour; + yield* git.answer(first!); + const sharedCall = yield* git.next; + expect(sharedCall.paths).toEqual(names.slice(4).flatMap((name) => folder(name).paths)); + yield* git.answer(sharedCall, failure); + yield* git.answerAll; + + expect(yield* Fiber.joinAll(fibers)).toEqual( + names.map((name) => expect.objectContaining(labelsOf(name))), + ); + // Every folder of the failed call got its own call, as it would without sharing. + expect(git.calls.slice(5).map((call) => call.paths)).toEqual( + names.slice(4).map((name) => folder(name).paths), + ); + expect(git.mostRunning()).toBe(IGNORE_CHECKS_PER_WORKSPACE); + } + }), + ); + + it.effect("ends waiting listings when the service closes before their call can start", () => + Effect.gen(function* () { + const git = yield* makeGit; + const scope = yield* Scope.make(); + const check = yield* makeIgnoreCheckQueue(git.run).pipe(Scope.provide(scope)); + const running = yield* Effect.forEach(["a", "b", "c", "d"], (name) => + check(folder(name)).pipe(Effect.forkChild({ startImmediately: true })), + ); + yield* git.nextFour; + // Six waiting checks, more than the four slots an unstarted call could leak. + const waiting = yield* Effect.forEach(["e", "f", "g", "h", "i", "j"], (name) => + check(folder(name, false)).pipe(Effect.forkChild({ startImmediately: true })), + ); + yield* Scope.close(scope, Exit.void); + yield* git.answerAll; + + expect(yield* Fiber.joinAll(running)).toEqual( + ["a", "b", "c", "d"].map((name) => expect.objectContaining(labelsOf(name))), + ); + for (const fiber of waiting) { + expect(Exit.isFailure(yield* Effect.exit(Fiber.join(fiber)))).toBe(true); + } + expect(git.calls.length).toBe(IGNORE_CHECKS_PER_WORKSPACE); + }), + ); + + it.effect("answers identical unshareable checks with one call and keeps others apart", () => + Effect.gen(function* () { + const git = yield* makeGit; + const check = yield* makeIgnoreCheckQueue(git.run); + yield* Effect.forEach(["a", "b", "c", "d"], (name) => + check(folder(name)).pipe(Effect.forkChild({ startImmediately: true })), + ); + const [first] = yield* git.nextFour; + const probe = { workspace: "/repo", gitCwd: "/repo", paths: ["sub/.git"], shareable: false }; + const probes = yield* Effect.forEach([1, 2, 3], () => + check(probe).pipe(Effect.forkChild({ startImmediately: true })), + ); + const linked = yield* Effect.forEach(["link-1", "link-2"], (name) => + check(folder(name, false)).pipe(Effect.forkChild({ startImmediately: true })), + ); + const refused = { exitCode: 128, stdout: "", stdoutTruncated: false }; + yield* git.answer(first!); + const probeCall = yield* git.next; + expect(probeCall.paths).toEqual(["sub/.git"]); + yield* git.answer(probeCall, Effect.succeed(refused)); + yield* git.answerAll; + + expect(yield* Fiber.joinAll(probes)).toEqual([refused, refused, refused]); + expect(yield* Fiber.joinAll(linked)).toEqual( + ["link-1", "link-2"].map((name) => expect.objectContaining(labelsOf(name))), + ); + expect(git.calls.map((call) => call.paths)).toEqual([ + ...["a", "b", "c", "d"].map((name) => folder(name).paths), + ["sub/.git"], + folder("link-1").paths, + folder("link-2").paths, + ]); + }), + ); +}); diff --git a/apps/server/src/workspace/workspaceIgnoreChecks.ts b/apps/server/src/workspace/workspaceIgnoreChecks.ts new file mode 100644 index 000000000000..2dc03cb7cec5 --- /dev/null +++ b/apps/server/src/workspace/workspaceIgnoreChecks.ts @@ -0,0 +1,214 @@ +import * as Deferred from "effect/Deferred"; +import * as Effect from "effect/Effect"; +import * as Exit from "effect/Exit"; +import * as Fiber from "effect/Fiber"; + +/** Paths in one `git check-ignore` call, which bounds its stdin and stdout. */ +export const IGNORE_CHECK_CHUNK_SIZE = 1000; + +/** Ignore checks of one workspace running at once; checks beyond this wait and share a call. */ +export const IGNORE_CHECKS_PER_WORKSPACE = 4; + +export interface IgnoreCheckOutput { + readonly exitCode: number; + readonly stdout: string; +} + +export interface IgnoreCheckRequest { + readonly workspace: string; + readonly gitCwd: string; + readonly paths: ReadonlyArray; + /** False when Git's answer can't be split per folder, as for paths beyond a symlink. */ + readonly shareable: boolean; +} + +interface Call { + waiting: number; + fiber: Fiber.Fiber | undefined; +} + +interface Check { + readonly gitCwd: string; + readonly paths: ReadonlyArray; + readonly key: string; + readonly done: Deferred.Deferred; + shareable: boolean; + abandoned: boolean; + call: Call | undefined; +} + +interface Workspace { + running: number; + queued: Array; +} + +/** + * Runs `git check-ignore -z --stdin` for listings, at most four calls per workspace. Checks that + * arrive while four run wait, and the next call answers all of them for one Git root; identical + * unshareable checks share one call. Nothing is reused: a call starts after all its checks arrived. + */ +export const makeIgnoreCheckQueue = ( + run: ( + gitCwd: string, + paths: ReadonlyArray, + ) => Effect.Effect, +) => + Effect.map(Effect.scope, (scope) => { + const workspaces = new Map(); + + const answer = Effect.fn("WorkspaceEntries.answerIgnoreChecks")(function* ( + workspace: Workspace, + checks: Array, + ) { + const shared = checks.length > 1 && checks[0]!.shareable; + const result = yield* run( + checks[0]!.gitCwd, + shared ? checks.flatMap((check) => check.paths) : checks[0]!.paths, + ).pipe(Effect.result); + if (!shared) { + const output = result._tag === "Success" ? result.success : undefined; + for (const check of checks) yield* Deferred.succeed(check.done, output); + return; + } + if ( + result._tag === "Success" && + (result.success.exitCode === 0 || result.success.exitCode === 1) && + !result.success.stdoutTruncated + ) { + // Git echoes each ignored path as it was given, so each folder takes back its own. + const ignored = new Set(result.success.stdout.split("\0")); + for (const check of checks) { + const own = check.paths.filter((checkPath) => ignored.has(checkPath)); + yield* Deferred.succeed(check.done, { + exitCode: own.length > 0 ? 0 : 1, + stdout: own.map((ignoredPath) => `${ignoredPath}\0`).join(""), + }); + } + return; + } + // One path Git refuses (inside a submodule, outside the repository) fails the whole call, + // and a timeout may be the batch's fault. Each folder is asked again alone, within the limit. + const retries = checks.filter((check) => !check.abandoned); + for (const check of retries) { + check.call = undefined; + check.shareable = false; + } + workspace.queued.unshift(...retries); + }); + + const release = (key: string, workspace: Workspace) => + Effect.suspend(() => { + workspace.running -= 1; + if (workspace.running === 0 && workspace.queued.length === 0) workspaces.delete(key); + return start(key, workspace); + }); + + const start = (key: string, workspace: Workspace): Effect.Effect => + Effect.suspend(() => { + const calls: Array> = []; + while (workspace.running < IGNORE_CHECKS_PER_WORKSPACE && workspace.queued.length > 0) { + const head = workspace.queued[0]!; + const checks: Array = []; + let pathCount = 0; + workspace.queued = workspace.queued.filter((check) => { + const joins = head.shareable + ? check.shareable && + check.gitCwd === head.gitCwd && + (checks.length === 0 || pathCount + check.paths.length <= IGNORE_CHECK_CHUNK_SIZE) + : !check.shareable && check.key === head.key; + if (!joins) return true; + checks.push(check); + pathCount += check.paths.length; + return false; + }); + workspace.running += 1; + calls.push(checks); + } + return Effect.forEach( + calls, + (checks) => { + const call: Call = { waiting: checks.length, fiber: undefined }; + for (const check of checks) check.call = call; + const finish = (exit: Exit.Exit) => { + const outcome = Exit.isSuccess(exit) + ? Exit.succeed(undefined) + : Exit.failCause(exit.cause); + return Effect.forEach( + checks, + (check) => (check.call === call ? Deferred.done(check.done, outcome) : Effect.void), + { discard: true }, + ).pipe(Effect.andThen(release(key, workspace))); + }; + let started = false; + // Started at once, so the finalizer is in place before anything can interrupt the call. + return Effect.suspend(() => { + started = true; + return answer(workspace, checks); + }).pipe( + Effect.onExit(finish), + Effect.forkIn(scope, { startImmediately: true }), + Effect.flatMap((fiber) => { + call.fiber = fiber; + // A closed scope returns a fiber that never ran, so its finalizer never will. + return started + ? Effect.void + : Effect.exit(Fiber.join(fiber)).pipe(Effect.flatMap(finish)); + }), + ); + }, + { discard: true }, + ); + }); + + const abandon = (workspace: Workspace, check: Check) => + Effect.suspend(() => { + check.abandoned = true; + const call = check.call; + if (call === undefined) { + workspace.queued = workspace.queued.filter((queued) => queued !== check); + return Effect.void; + } + call.waiting -= 1; + // Stop Git once no listing waits for its answer. + return call.waiting === 0 && call.fiber !== undefined + ? Fiber.interrupt(call.fiber) + : Effect.void; + }); + + return (request: IgnoreCheckRequest): Effect.Effect => + Effect.uninterruptibleMask((restore) => + Effect.suspend(() => { + let workspace = workspaces.get(request.workspace); + if (workspace === undefined) { + workspace = { running: 0, queued: [] }; + workspaces.set(request.workspace, workspace); + } + // A free slot means nothing waits, so this call has no one to share with: it runs here, + // and interrupting this listing stops its Git process. + if (workspace.running < IGNORE_CHECKS_PER_WORKSPACE) { + workspace.running += 1; + return restore( + run(request.gitCwd, request.paths).pipe(Effect.orElseSucceed(() => undefined)), + ).pipe(Effect.ensuring(release(request.workspace, workspace))); + } + const check: Check = { + gitCwd: request.gitCwd, + paths: request.paths, + key: `${request.gitCwd}\0${request.paths.join("\0")}`, + done: Deferred.makeUnsafe(), + shareable: request.shareable, + abandoned: false, + call: undefined, + }; + workspace.queued.push(check); + const own = workspace; + return start(request.workspace, workspace).pipe( + Effect.andThen( + restore(Deferred.await(check.done)).pipe( + Effect.onInterrupt(() => abandon(own, check)), + ), + ), + ); + }), + ); + }); diff --git a/apps/web/src/components/files/useDirectoryEntries.browser.test.tsx b/apps/web/src/components/files/useDirectoryEntries.browser.test.tsx index 046563bcb22f..5224d5de78f3 100644 --- a/apps/web/src/components/files/useDirectoryEntries.browser.test.tsx +++ b/apps/web/src/components/files/useDirectoryEntries.browser.test.tsx @@ -144,8 +144,10 @@ it("shares the follow-up promise and does not resolve callers before the final l expect(completed).toBe(true); }); -it("retains the four-query concurrency bound across directories and refreshes", async () => { - const folders = Array.from({ length: 8 }, (_, index) => ({ +it("retains the query concurrency bound across directories and refreshes", async () => { + // Owned by the test, so lowering the production limit fails it instead of moving the workload too. + const expectedFolderReads = 64; + const folders = Array.from({ length: expectedFolderReads + 4 }, (_, index) => ({ path: `folder-${index}`, kind: "directory" as const, })); @@ -163,17 +165,17 @@ it("retains the four-query concurrency bound across directories and refreshes", const refreshStart = state.calls.length; state.resetPeakQueries(); await act(async () => state.view.refresh()); - expect(state.peakQueries).toBe(4); - expect(state.activeQueries).toBe(4); + expect(state.peakQueries).toBe(expectedFolderReads); + expect(state.activeQueries).toBe(expectedFolderReads); await act(async () => { for (let index = 0; index < 20; index++) state.view.refresh(); }); for (let index = refreshStart; index < state.calls.length; index++) { await state.settle(index, state.calls[index]!.path === "" ? folders : []); - expect(state.activeQueries).toBeLessThanOrEqual(4); + expect(state.activeQueries).toBeLessThanOrEqual(expectedFolderReads); } - expect(state.peakQueries).toBe(4); - expect(new Set(state.calls.map((call) => call.path)).size).toBe(9); + expect(state.peakQueries).toBe(expectedFolderReads); + expect(new Set(state.calls.map((call) => call.path)).size).toBe(folders.length + 1); for (const path of ["", ...folders.map((folder) => folder.path)]) { expect(state.calls.filter((call) => call.path === path).length).toBeLessThanOrEqual(3); } diff --git a/apps/web/src/components/files/useDirectoryEntries.ts b/apps/web/src/components/files/useDirectoryEntries.ts index 32aa2fe46675..6db3aa964a45 100644 --- a/apps/web/src/components/files/useDirectoryEntries.ts +++ b/apps/web/src/components/files/useDirectoryEntries.ts @@ -8,6 +8,9 @@ import { environmentCatalog } from "~/connection/catalog"; import { appAtomRegistry } from "~/rpc/atomRegistry"; import { projectEnvironment } from "~/state/projects"; +/** Folder reads in flight; the server shares one Git ignore check among reads beyond its four. */ +const FOLDER_READS_IN_FLIGHT = 64; + export function useDirectoryEntries(environmentId: EnvironmentId, cwd: string) { const [directories, setDirectories] = useState(new Map()); const [errors, setErrors] = useState(new Map()); @@ -28,7 +31,8 @@ export function useDirectoryEntries(environmentId: EnvironmentId, cwd: string) { requested.current.add(directoryPath); const atom = projectEnvironment.listEntries({ environmentId, input: { cwd, directoryPath } }); const request = (async () => { - if (running.current >= 4) await new Promise((resolve) => waiting.current.push(resolve)); + if (running.current >= FOLDER_READS_IN_FLIGHT) + await new Promise((resolve) => waiting.current.push(resolve)); else running.current++; const result = active.current ? await executeAtomQuery(appAtomRegistry, atom, {