Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -4651,4 +4651,28 @@ describe("splitBufferedAssistantText", () => {
rest: "~~~\n```\n\nx\n",
});
});

it("delivers tight list items one at a time", () => {
expect(splitBufferedAssistantText("## Steps\n\n- one\n- two\n- thr")).toEqual({
ready: "## Steps\n\n- one\n- two\n",
rest: "- thr",
});
expect(splitBufferedAssistantText("1. one\n2. two\n more\n3. t")).toEqual({
ready: "1. one\n2. two\n more\n",
rest: "3. t",
});
});

it("keeps a partial list marker and list-like code buffered", () => {
expect(splitBufferedAssistantText("intro\n-")).toEqual({ ready: "", rest: "intro\n-" });
expect(splitBufferedAssistantText("intro\n1.")).toEqual({ ready: "", rest: "intro\n1." });
// `intro\n- \n` would parse as a setext heading, so a bare marker with only
// trailing whitespace is not a boundary on the partial line either.
expect(splitBufferedAssistantText("intro\n- ")).toEqual({ ready: "", rest: "intro\n- " });
expect(splitBufferedAssistantText("- one\n")).toEqual({ ready: "", rest: "- one\n" });
expect(splitBufferedAssistantText("```\n- one\n- two\n")).toEqual({
ready: "",
rest: "```\n- one\n- two\n",
});
});
});
24 changes: 18 additions & 6 deletions apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.ts
Original file line number Diff line number Diff line change
Expand Up @@ -194,24 +194,36 @@ const MARKDOWN_FENCE_PATTERN = /^( *)(`{3,}|~{3,})/;
// CommonMark blank lines hold only spaces and tabs. Other whitespace, such as
// a no-break space, is paragraph content.
const BLANK_LINE_PATTERN = /^[ \t]*$/;
// A bullet or ordered marker followed by whitespace, at any indentation so
// nested items count. The trailing space is required, so a partial `-` or
// `1.` never matches before the model finishes the marker.
const LIST_ITEM_START_PATTERN = /^[ \t]*(?:[-*+]|\d{1,9}[.)])[ \t]/;
Comment thread
juliusmarminge marked this conversation as resolved.

/**
* Splits buffered assistant text at the last blank line or closing code fence
* that is not inside an open fenced code block. `ready` is safe to deliver now
* because the markdown before it will not change shape as more text arrives.
* `rest` stays buffered until the next boundary or completion. Only fully
* terminated lines count, so a trailing partial line never leaks.
* Splits buffered assistant text at the last blank line, closing code fence,
* or list item start that is not inside an open fenced code block. `ready` is
* safe to deliver now because the markdown before it will not change shape as
* more text arrives. `rest` stays buffered until the next boundary or
* completion. Only fully terminated lines count, so a trailing partial line
* never leaks; a list item start is the one lookahead that may sit on the
* partial line, since tight lists have no blank lines between items and would
* otherwise land all at once.
*/
export function splitBufferedAssistantText(text: string): { ready: string; rest: string } {
let openFence: { marker: string; indent: number } | null = null;
let boundary = -1;
let lineStart = 0;
for (;;) {
const newline = text.indexOf("\n", lineStart);
const line = text
.slice(lineStart, newline === -1 ? text.length : newline)
.replace(/[ \t\r]+$/, "");
if (openFence === null && lineStart > 0 && LIST_ITEM_START_PATTERN.test(line)) {
boundary = lineStart;
}
if (newline === -1) {
break;
}
const line = text.slice(lineStart, newline).replace(/[ \t\r]+$/, "");
const fenceMatch = MARKDOWN_FENCE_PATTERN.exec(line);
if (fenceMatch) {
const indent = fenceMatch[1]!.length;
Expand Down
Loading