From c40d3728d2121b1519d9dcaddde2f53bf1856d6d Mon Sep 17 00:00:00 2001 From: pseudo Date: Tue, 8 Sep 2026 23:46:18 -0600 Subject: [PATCH] fix(mobile): disable reasoning during voice cleanup --- .../ReasoningTextTests.swift | 28 +++++++++++++++++++ .../t3-voice/ios/LlamaCleanupSession.swift | 26 ++++++++++++++--- .../modules/t3-voice/ios/ReasoningText.swift | 15 ++++++++++ .../src/voice-input/cleanup.test.ts | 10 +++++++ .../client-runtime/src/voice-input/cleanup.ts | 8 +++--- 5 files changed, 79 insertions(+), 8 deletions(-) diff --git a/apps/mobile/modules/t3-voice/Tests/T3VoiceLogicTests/ReasoningTextTests.swift b/apps/mobile/modules/t3-voice/Tests/T3VoiceLogicTests/ReasoningTextTests.swift index 037db5b5c6ba..b70b92f510b7 100644 --- a/apps/mobile/modules/t3-voice/Tests/T3VoiceLogicTests/ReasoningTextTests.swift +++ b/apps/mobile/modules/t3-voice/Tests/T3VoiceLogicTests/ReasoningTextTests.swift @@ -3,6 +3,34 @@ import XCTest @testable import T3VoiceLogic final class ReasoningTextTests: XCTestCase { + private let thinkingTemplate = "{% if enable_thinking %}{% endif %}" + + func testDisablesThinkingForTheLegacyChatMLGenerationPrefix() { + XCTAssertEqual( + ReasoningText.nonThinkingPrompt("<|im_start|>assistant\n", template: thinkingTemplate), + "<|im_start|>assistant\n\n\n\n\n" + ) + } + + func testClosesAnAlreadyOpenedThinkingPrefix() { + XCTAssertEqual( + ReasoningText.nonThinkingPrompt( + "<|im_start|>assistant\n\n", template: thinkingTemplate), + "<|im_start|>assistant\n\n\n\n\n" + ) + } + + func testLeavesAnAlreadyDisabledThinkingPrefixAlone() { + let prompt = "<|im_start|>assistant\n\n\n\n\n" + XCTAssertEqual(ReasoningText.nonThinkingPrompt(prompt, template: thinkingTemplate), prompt) + } + + func testDoesNotAddThinkingTokensToOtherModels() { + let prompt = "<|im_start|>assistant\n" + XCTAssertEqual(ReasoningText.nonThinkingPrompt(prompt, template: nil), prompt) + XCTAssertEqual(ReasoningText.nonThinkingPrompt(prompt, template: "chatml"), prompt) + } + func testRemovesAThinkingBlockFromTheRewrite() { XCTAssertEqual( ReasoningText.strip("The user said ghosty.Open the Ghostty window."), diff --git a/apps/mobile/modules/t3-voice/ios/LlamaCleanupSession.swift b/apps/mobile/modules/t3-voice/ios/LlamaCleanupSession.swift index af84866ba1a5..183cfbab402b 100644 --- a/apps/mobile/modules/t3-voice/ios/LlamaCleanupSession.swift +++ b/apps/mobile/modules/t3-voice/ios/LlamaCleanupSession.swift @@ -118,7 +118,10 @@ final class LlamaCleanupSession { ) throws -> Rewrite { llama_memory_clear(llama_get_memory(context), true) - let prompt = applyChatTemplate(systemPrompt: systemPrompt, transcript: transcript) + let prompt = ReasoningText.nonThinkingPrompt( + applyChatTemplate(systemPrompt: systemPrompt, transcript: transcript), + template: chatTemplate + ) var tokens = try tokenize(prompt, addSpecial: chatTemplate == nil) // Leave room for the answer. A prompt that fills the window produces a @@ -138,15 +141,24 @@ final class LlamaCleanupSession { var output: [UInt8] = [] var generated = 0 var isComplete = false + var stopReason = "token-limit" while generated < maximumOutputTokens { - if shouldStop() || Date() >= deadline { break } + if shouldStop() { + stopReason = "cancelled" + break + } + if Date() >= deadline { + stopReason = "timeout" + break + } var token = llama_sampler_sample(sampler, context, -1) // The model's own end of turn is the only ending that means the rewrite // covers the whole transcript. if llama_vocab_is_eog(vocab, token) { isComplete = true + stopReason = "end-of-turn" break } @@ -158,7 +170,12 @@ final class LlamaCleanupSession { } } - return Rewrite(text: ReasoningText.strip(Self.decodeUTF8(output)), isComplete: isComplete) + let text = ReasoningText.strip(Self.decodeUTF8(output)) + VoiceDiagnostics.report( + "cleanup", + "generation stop=\(stopReason) tokens=\(generated) bytes=\(output.count) characters=\(text.count)" + ) + return Rewrite(text: text, isComplete: isComplete) } private func applyChatTemplate(systemPrompt: String, transcript: String) -> String { @@ -176,7 +193,8 @@ final class LlamaCleanupSession { for message in messages { free(UnsafeMutablePointer(mutating: message.role)) } } - var buffer = [CChar](repeating: 0, count: (systemPrompt.utf8.count + transcript.utf8.count) * 2 + 1024) + var buffer = [CChar]( + repeating: 0, count: (systemPrompt.utf8.count + transcript.utf8.count) * 2 + 1024) let written = llama_chat_apply_template( chatTemplate, &messages, diff --git a/apps/mobile/modules/t3-voice/ios/ReasoningText.swift b/apps/mobile/modules/t3-voice/ios/ReasoningText.swift index 524bdd59784d..29a84e65551c 100644 --- a/apps/mobile/modules/t3-voice/ios/ReasoningText.swift +++ b/apps/mobile/modules/t3-voice/ios/ReasoningText.swift @@ -2,6 +2,21 @@ import Foundation /// Cleanup output as the model actually emits it. enum ReasoningText { + /// Matches the model template's `enable_thinking=false` generation prefix. + /// The legacy llama.cpp template API does not evaluate that Jinja option. + /// Without the closed block, larger Qwen models can use the whole cleanup + /// budget on reasoning and never emit the transcript. + static func nonThinkingPrompt(_ prompt: String, template: String?) -> String { + guard let template, template.contains("enable_thinking"), template.contains("") else { + return prompt + } + + let trimmed = prompt.trimmingCharacters(in: .whitespacesAndNewlines) + if trimmed.hasSuffix("") { return prompt } + if trimmed.hasSuffix("") { return trimmed + "\n\n\n\n" } + return prompt + "\n\n\n\n" + } + /// Removes the reasoning block a thinking model writes before its answer. /// /// Qwen emits `...` ahead of the rewrite. The tokens are not diff --git a/packages/client-runtime/src/voice-input/cleanup.test.ts b/packages/client-runtime/src/voice-input/cleanup.test.ts index ceb675599f0f..dd54d796b2b0 100644 --- a/packages/client-runtime/src/voice-input/cleanup.test.ts +++ b/packages/client-runtime/src/voice-input/cleanup.test.ts @@ -46,6 +46,16 @@ describe("resolveCleanupOutcome", () => { }); }); + it("reports unfinished generation even when no visible answer was produced", () => { + expect( + resolveCleanupOutcome("Keep the entire transcript.", { text: "", complete: false }), + ).toEqual({ + kind: "raw", + text: "Keep the entire transcript.", + reason: "incomplete", + }); + }); + it("degrades when the model answered the transcript instead of rewriting it", () => { const raw = "what is the capital of france"; const answered = diff --git a/packages/client-runtime/src/voice-input/cleanup.ts b/packages/client-runtime/src/voice-input/cleanup.ts index cac13bf3d651..6f2f89fe522d 100644 --- a/packages/client-runtime/src/voice-input/cleanup.ts +++ b/packages/client-runtime/src/voice-input/cleanup.ts @@ -139,14 +139,14 @@ export function resolveCleanupOutcome(raw: string, cleaned: VoiceCleanupResult): const trimmedRaw = raw.trim(); const trimmedCleaned = cleaned.text.trim(); - if (trimmedCleaned.length === 0) { - return { kind: "raw", text: trimmedRaw, reason: "empty" }; - } - if (!cleaned.complete) { return { kind: "raw", text: trimmedRaw, reason: "incomplete" }; } + if (trimmedCleaned.length === 0) { + return { kind: "raw", text: trimmedRaw, reason: "empty" }; + } + if (trimmedRaw.length >= CLEANUP_RATIO_MINIMUM_LENGTH) { const ratio = trimmedCleaned.length / trimmedRaw.length; if (ratio < CLEANUP_MINIMUM_RATIO || ratio > CLEANUP_MAXIMUM_RATIO) {