From 4036132b98e2447c322246f0e351ec3518419db0 Mon Sep 17 00:00:00 2001 From: moizlatif7 Date: Thu, 30 Jul 2026 17:23:41 -0400 Subject: [PATCH] fix(core): strip BOM from AI-generated content Fixes #368 by changing BOM preservation logic to only preserve BOMs from existing files, never from AI-generated content. Changes: - Modified 7 locations across 6 files - Added ANRCODE_CHANGE markers to all modified regions - Updated test expectations to reflect new behavior - Added new test case for Issue #368 Behavior: - Existing files with BOMs: BOM preserved (round-trip safe) - New files: Never get BOMs (fixes Windows script issue) - AI-generated content with BOM: BOM stripped Related to #368 --- packages/core/src/file-mutation.ts | 3 ++- packages/core/src/patch.ts | 3 ++- packages/core/src/tool/edit.ts | 3 ++- packages/core/test/file-mutation.test.ts | 24 ++++++++++++++++++++++-- packages/opencode/src/patch/index.ts | 3 ++- packages/opencode/src/tool/edit.ts | 6 ++++-- packages/opencode/src/tool/write.ts | 3 ++- 7 files changed, 36 insertions(+), 9 deletions(-) diff --git a/packages/core/src/file-mutation.ts b/packages/core/src/file-mutation.ts index 80a3a449f600..53a5c0b38dc4 100644 --- a/packages/core/src/file-mutation.ts +++ b/packages/core/src/file-mutation.ts @@ -112,9 +112,10 @@ const layer = Layer.effect( const current = yield* fs .readFile(input.target.canonical) .pipe(Effect.catchReason("PlatformError", "NotFound", () => Effect.succeed(undefined))) + // ANRCODE_CHANGE {"issue":368,"branch":"anr/368/strip-bom-ai-content","date":"2026-07-30"} yield* fs.writeWithDirs( input.target.canonical, - joinBom(next.text, Boolean(current && hasUtf8Bom(current)) || next.bom), + joinBom(next.text, Boolean(current && hasUtf8Bom(current))), ) return writeResult(input.target, current !== undefined) }), diff --git a/packages/core/src/patch.ts b/packages/core/src/patch.ts index a4370d44aac1..b00ddd8af96e 100644 --- a/packages/core/src/patch.ts +++ b/packages/core/src/patch.ts @@ -77,7 +77,8 @@ export function derive(path: string, chunks: ReadonlyArray, ori for (const [start, remove, insert] of replacements.toReversed()) updated.splice(start, remove, ...insert) if (updated.at(-1) !== "") updated.push("") const next = splitBom(updated.join("\n")) - return { content: next.text, bom: source.bom || next.bom } + // ANRCODE_CHANGE {"issue":368,"branch":"anr/368/strip-bom-ai-content","date":"2026-07-30"} + return { content: next.text, bom: source.bom } } export function joinBom(text: string, bom: boolean) { diff --git a/packages/core/src/tool/edit.ts b/packages/core/src/tool/edit.ts index f0bdb488a060..85b0d96e5f1a 100644 --- a/packages/core/src/tool/edit.ts +++ b/packages/core/src/tool/edit.ts @@ -188,11 +188,12 @@ const layer = Layer.effectDiscard( { additions: 0, deletions: 0 }, ) const next = splitBom(replaced) + // ANRCODE_CHANGE {"issue":368,"branch":"anr/368/strip-bom-ai-content","date":"2026-07-30"} const result = yield* unableToEdit( files.writeIfUnchanged({ target, expected: source.content, - content: joinBom(next.text, source.bom || next.bom), + content: joinBom(next.text, source.bom), }), ) return { diff --git a/packages/core/test/file-mutation.test.ts b/packages/core/test/file-mutation.test.ts index bcd6ce97ed88..c111c8cbd6c6 100644 --- a/packages/core/test/file-mutation.test.ts +++ b/packages/core/test/file-mutation.test.ts @@ -71,7 +71,7 @@ describe("FileMutation", () => { ), ) - it.live("preserves exactly one BOM for text writes and normalizes created text", () => + it.live("preserves BOM from existing files but strips BOM from new content", () => withTmp((directory) => Effect.gen(function* () { const preservedPath = path.join(directory, "preserved.txt") @@ -84,7 +84,27 @@ describe("FileMutation", () => { yield* files.writeTextPreservingBom({ target: created, content: "\uFEFF\uFEFF\uFEFFcreated" }) expect(yield* Effect.promise(() => fs.readFile(preservedPath, "utf8"))).toBe("\uFEFFafter") - expect(yield* Effect.promise(() => fs.readFile(created.canonical, "utf8"))).toBe("\uFEFFcreated") + expect(yield* Effect.promise(() => fs.readFile(created.canonical, "utf8"))).toBe("created") + }).pipe(provide(directory)), + ), + ) + + it.live("strips BOM from AI-generated content in new files (Issue #368)", () => + withTmp((directory) => + Effect.gen(function* () { + const files = yield* FileMutation.Service + const mutation = yield* LocationMutation.Service + const testPath = path.join(directory, "ai-generated.txt") + const target = yield* mutation.resolve({ path: "ai-generated.txt" }) + + // AI generates content with BOM + const aiGeneratedContent = "\uFEFF# Test Script\necho 'Hello'\n" + yield* files.writeTextPreservingBom({ target, content: aiGeneratedContent }) + + // File should not have BOM + const result = yield* Effect.promise(() => fs.readFile(testPath, "utf8")) + expect(result).toBe("# Test Script\necho 'Hello'\n") + expect(result.startsWith("\uFEFF")).toBe(false) }).pipe(provide(directory)), ), ) diff --git a/packages/opencode/src/patch/index.ts b/packages/opencode/src/patch/index.ts index 66797d4f89ba..7bab6a6bfd77 100644 --- a/packages/opencode/src/patch/index.ts +++ b/packages/opencode/src/patch/index.ts @@ -332,10 +332,11 @@ export function deriveNewContentsFromChunks( // Generate unified diff const unifiedDiff = generateUnifiedDiff(originalContent.text, newContent) + // ANRCODE_CHANGE {"issue":368,"branch":"anr/368/strip-bom-ai-content","date":"2026-07-30"} return { unified_diff: unifiedDiff, content: newContent, - bom: originalContent.bom || next.bom, + bom: originalContent.bom, } } diff --git a/packages/opencode/src/tool/edit.ts b/packages/opencode/src/tool/edit.ts index a92e4720c0fc..74f183472ead 100644 --- a/packages/opencode/src/tool/edit.ts +++ b/packages/opencode/src/tool/edit.ts @@ -95,7 +95,8 @@ export const EditTool = Tool.define( ) } const next = Bom.split(params.newString) - const desiredBom = next.bom + // ANRCODE_CHANGE {"issue":368,"branch":"anr/368/strip-bom-ai-content","date":"2026-07-30"} + const desiredBom = false contentOld = "" contentNew = next.text diff = trimDiff(createTwoFilesPatch(filePath, filePath, contentOld, contentNew)) @@ -131,7 +132,8 @@ export const EditTool = Tool.define( const replacement = convertToLineEnding(normalizeLineEndings(params.newString), ending) const next = Bom.split(replace(contentOld, old, replacement, params.replaceAll)) - const desiredBom = source.bom || next.bom + // ANRCODE_CHANGE {"issue":368,"branch":"anr/368/strip-bom-ai-content","date":"2026-07-30"} + const desiredBom = source.bom contentNew = next.text diff = trimDiff( diff --git a/packages/opencode/src/tool/write.ts b/packages/opencode/src/tool/write.ts index 37be6d8c47bc..bcebc7a006e3 100644 --- a/packages/opencode/src/tool/write.ts +++ b/packages/opencode/src/tool/write.ts @@ -46,7 +46,8 @@ export const WriteTool = Tool.define( const exists = yield* fs.existsSafe(filepath) const source = exists ? yield* Bom.readFile(fs, filepath) : { bom: false, text: "" } const next = Bom.split(params.content) - const desiredBom = source.bom || next.bom + // ANRCODE_CHANGE {"issue":368,"branch":"anr/368/strip-bom-ai-content","date":"2026-07-30"} + const desiredBom = source.bom const contentOld = source.text const contentNew = next.text