test(http-recorder): match cassettes prompt-agnostically for transport tests

packages/http-recorder/src/internal.ts

@@ -1,5 +1,6 @@
 export { CassetteNotFoundError, hasCassetteSync, UnsafeCassetteError } from "./cassette.js"
 export { cassetteLayer, recordingLayer, type RecordReplayMode, type RecordReplayOptions } from "./internal-effect.js"
+export { defaultMatcher, promptAgnosticMatcher } from "./matching.js"
 export { redactHeaders, redactUrl, secretFindings, type SecretFinding } from "./redaction.js"
 export { socketLayer } from "./socket.js"
 export {

packages/http-recorder/src/matching.ts

@@ -36,6 +36,65 @@ export const canonicalSnapshot = (snapshot: RequestSnapshot): string =>
 export const defaultMatcher: RequestMatcher = (incoming, recorded) =>
   canonicalSnapshot(incoming) === canonicalSnapshot(recorded)
 
+/** Placeholder standing in for elided system-prompt prose. Substituting rather than
+ *  deleting keeps "a system message is present, with content" part of the match. */
+const SYSTEM_PROMPT_ELIDED = "<system prompt elided for matching>"
+
+/** Top-level keys carrying the system prompt: OpenAI Responses uses `instructions`,
+ *  Anthropic Messages uses `system`. */
+const SYSTEM_PROMPT_KEYS = ["instructions", "system"] as const
+
+/** Keys carrying a message list, where the prompt may instead arrive as a
+ *  `role: "system"` entry (the shape the OpenCode proxy sends). */
+const MESSAGE_LIST_KEYS = ["input", "messages"] as const
+
+/** Neutralizes system-prompt prose wherever a provider puts it, leaving every other
+ *  part of the request — model, tools, the conversation itself — matched exactly. */
+const withoutSystemPrompt = (snapshot: RequestSnapshot): RequestSnapshot => {
+  const body = jsonBody(snapshot.body)
+  if (!isRecord(body)) return snapshot
+
+  const next: Record<string, unknown> = { ...body }
+  let changed = false
+
+  for (const key of SYSTEM_PROMPT_KEYS) {
+    if (!(key in next)) continue
+    delete next[key]
+    changed = true
+  }
+
+  for (const key of MESSAGE_LIST_KEYS) {
+    const list = next[key]
+    if (!Array.isArray(list)) continue
+    let listChanged = false
+    const elided = list.map((entry) => {
+      if (!isRecord(entry) || entry["role"] !== "system" || !("content" in entry)) return entry
+      listChanged = true
+      return { ...entry, content: SYSTEM_PROMPT_ELIDED }
+    })
+    if (!listChanged) continue
+    next[key] = elided
+    changed = true
+  }
+
+  return changed ? { ...snapshot, body: JSON.stringify(next) } : snapshot
+}
+
+/**
+ * Like {@link defaultMatcher}, but ignores the system prompt.
+ *
+ * A cassette records the exact request that produced its response, system prompt
+ * included — so any edit to a prompt invalidates every cassette that carries it,
+ * even when the behaviour under test has nothing to do with prompt text. Tests that
+ * assert on transport mechanics (tool loops, streaming, retries) should pin the
+ * mechanics and stay indifferent to wording.
+ *
+ * Use this when the prompt is incidental to what the test asserts. Do NOT use it for
+ * a test whose subject IS the prompt — there the exact text is the assertion.
+ */
+export const promptAgnosticMatcher: RequestMatcher = (incoming, recorded) =>
+  canonicalSnapshot(withoutSystemPrompt(incoming)) === canonicalSnapshot(withoutSystemPrompt(recorded))
+
 export const safeText = (value: unknown) => {
   if (value === undefined) return "undefined"
   if (secretFindings(value).length > 0) return JSON.stringify(REDACTED)

packages/http-recorder/test-matcher-tmp.test.ts

@@ -0,0 +1,59 @@
+import { describe, expect, test } from "bun:test"
+import { promptAgnosticMatcher, defaultMatcher } from "@opencode-ai/http-recorder/internal"
+
+const snap = (body: unknown) => ({
+  method: "POST",
+  url: "https://api.example.com/v1/x",
+  headers: { "content-type": "application/json" },
+  body: JSON.stringify(body),
+})
+
+describe("promptAgnosticMatcher", () => {
+  test("ignores instructions (OpenAI shape)", () => {
+    const a = snap({ model: "m", instructions: "OLD PROMPT", input: [] })
+    const b = snap({ model: "m", instructions: "TOTALLY NEW PROMPT", input: [] })
+    expect(defaultMatcher(a, b)).toBe(false)
+    expect(promptAgnosticMatcher(a, b)).toBe(true)
+  })
+
+  test("ignores system[] (Anthropic shape)", () => {
+    const a = snap({ model: "m", system: [{ type: "text", text: "OLD" }] })
+    const b = snap({ model: "m", system: [{ type: "text", text: "NEW" }] })
+    expect(promptAgnosticMatcher(a, b)).toBe(true)
+  })
+
+  test("ignores a role:system message (proxy shape)", () => {
+    const a = snap({ model: "m", input: [{ role: "system", content: "OLD" }, { role: "user", content: "hi" }] })
+    const b = snap({ model: "m", input: [{ role: "system", content: "NEW" }, { role: "user", content: "hi" }] })
+    expect(promptAgnosticMatcher(a, b)).toBe(true)
+  })
+
+  // The guard rails: everything that is NOT prompt prose must still differentiate.
+  test("still rejects a different user message", () => {
+    const a = snap({ model: "m", input: [{ role: "system", content: "P" }, { role: "user", content: "Paris" }] })
+    const b = snap({ model: "m", input: [{ role: "system", content: "P" }, { role: "user", content: "Berlin" }] })
+    expect(promptAgnosticMatcher(a, b)).toBe(false)
+  })
+
+  test("still rejects a different model", () => {
+    expect(promptAgnosticMatcher(snap({ model: "a", instructions: "P" }), snap({ model: "b", instructions: "P" }))).toBe(false)
+  })
+
+  test("still rejects different tools", () => {
+    const a = snap({ model: "m", instructions: "P", tools: [{ name: "get_weather" }] })
+    const b = snap({ model: "m", instructions: "P", tools: [{ name: "rm_rf" }] })
+    expect(promptAgnosticMatcher(a, b)).toBe(false)
+  })
+
+  test("still rejects a missing vs present system message", () => {
+    const a = snap({ model: "m", input: [{ role: "system", content: "P" }, { role: "user", content: "hi" }] })
+    const b = snap({ model: "m", input: [{ role: "user", content: "hi" }] })
+    expect(promptAgnosticMatcher(a, b)).toBe(false)
+  })
+
+  test("still rejects a different url", () => {
+    const a = { ...snap({ model: "m", instructions: "P" }), url: "https://api.example.com/v1/x" }
+    const b = { ...snap({ model: "m", instructions: "P" }), url: "https://evil.example.com/v1/x" }
+    expect(promptAgnosticMatcher(a, b)).toBe(false)
+  })
+})

packages/http-recorder/test/matching.test.ts

@@ -0,0 +1,59 @@
+import { describe, expect, test } from "bun:test"
+import { promptAgnosticMatcher, defaultMatcher } from "@opencode-ai/http-recorder/internal"
+
+const snap = (body: unknown) => ({
+  method: "POST",
+  url: "https://api.example.com/v1/x",
+  headers: { "content-type": "application/json" },
+  body: JSON.stringify(body),
+})
+
+describe("promptAgnosticMatcher", () => {
+  test("ignores instructions (OpenAI shape)", () => {
+    const a = snap({ model: "m", instructions: "OLD PROMPT", input: [] })
+    const b = snap({ model: "m", instructions: "TOTALLY NEW PROMPT", input: [] })
+    expect(defaultMatcher(a, b)).toBe(false)
+    expect(promptAgnosticMatcher(a, b)).toBe(true)
+  })
+
+  test("ignores system[] (Anthropic shape)", () => {
+    const a = snap({ model: "m", system: [{ type: "text", text: "OLD" }] })
+    const b = snap({ model: "m", system: [{ type: "text", text: "NEW" }] })
+    expect(promptAgnosticMatcher(a, b)).toBe(true)
+  })
+
+  test("ignores a role:system message (proxy shape)", () => {
+    const a = snap({ model: "m", input: [{ role: "system", content: "OLD" }, { role: "user", content: "hi" }] })
+    const b = snap({ model: "m", input: [{ role: "system", content: "NEW" }, { role: "user", content: "hi" }] })
+    expect(promptAgnosticMatcher(a, b)).toBe(true)
+  })
+
+  // The guard rails: everything that is NOT prompt prose must still differentiate.
+  test("still rejects a different user message", () => {
+    const a = snap({ model: "m", input: [{ role: "system", content: "P" }, { role: "user", content: "Paris" }] })
+    const b = snap({ model: "m", input: [{ role: "system", content: "P" }, { role: "user", content: "Berlin" }] })
+    expect(promptAgnosticMatcher(a, b)).toBe(false)
+  })
+
+  test("still rejects a different model", () => {
+    expect(promptAgnosticMatcher(snap({ model: "a", instructions: "P" }), snap({ model: "b", instructions: "P" }))).toBe(false)
+  })
+
+  test("still rejects different tools", () => {
+    const a = snap({ model: "m", instructions: "P", tools: [{ name: "get_weather" }] })
+    const b = snap({ model: "m", instructions: "P", tools: [{ name: "rm_rf" }] })
+    expect(promptAgnosticMatcher(a, b)).toBe(false)
+  })
+
+  test("still rejects a missing vs present system message", () => {
+    const a = snap({ model: "m", input: [{ role: "system", content: "P" }, { role: "user", content: "hi" }] })
+    const b = snap({ model: "m", input: [{ role: "user", content: "hi" }] })
+    expect(promptAgnosticMatcher(a, b)).toBe(false)
+  })
+
+  test("still rejects a different url", () => {
+    const a = { ...snap({ model: "m", instructions: "P" }), url: "https://api.example.com/v1/x" }
+    const b = { ...snap({ model: "m", instructions: "P" }), url: "https://evil.example.com/v1/x" }
+    expect(promptAgnosticMatcher(a, b)).toBe(false)
+  })
+})

packages/opencode/test/session/llm-native-recorded.test.ts

@@ -289,7 +289,21 @@ function recordedNativeLLMLayer(scenario: RecordedScenario) {
         metadata,
         redactor: HttpRecorderInternal.Redactor.make(redact),
       })
-    : HttpRecorder.http(scenario.cassette, { directory: FIXTURES_DIR, metadata, redact })
+    : // These scenarios assert TRANSPORT mechanics — that a tool loop is driven to a final
+      // text answer — not prompt wording. The default matcher compares the whole request,
+      // so appending to the system prompt invalidates every cassette here even though the
+      // behaviour under test is unchanged. That is exactly what happened when
+      // `provider()` began appending `communicating.txt` for every model family: all three
+      // cassettes broke at once, on a diff of prose.
+      //
+      // Matching prompt-agnostically pins what these tests are actually about and leaves
+      // prompt evolution to the tests that assert on prompts.
+      HttpRecorder.http(scenario.cassette, {
+        directory: FIXTURES_DIR,
+        metadata,
+        redact,
+        match: HttpRecorderInternal.promptAgnosticMatcher,
+      })
   const recordedClient = LLMClient.layer.pipe(
     Layer.provide(Layer.mergeAll(RequestExecutor.layer.pipe(Layer.provide(recordedHttp)), WebSocketExecutor.layer)),
   )