diff --git a/apps/mobile/modules/t3-native-controls/ios/T3KeyboardCommandsModule.swift b/apps/mobile/modules/t3-native-controls/ios/T3KeyboardCommandsModule.swift index 863003396db4..ea65b118e64e 100644 --- a/apps/mobile/modules/t3-native-controls/ios/T3KeyboardCommandsModule.swift +++ b/apps/mobile/modules/t3-native-controls/ios/T3KeyboardCommandsModule.swift @@ -17,6 +17,7 @@ public final class T3KeyboardCommandsModule: Module { public final class T3KeyboardCommandsView: ExpoView { let onCommand = EventDispatcher() private var enabledCommands = Set() + private var dictationHeld = false public override var canBecomeFirstResponder: Bool { true } @@ -50,7 +51,10 @@ public final class T3KeyboardCommandsView: ExpoView { public override func pressesBegan(_ presses: Set, with event: UIPressesEvent?) { if holdsToTalk, Self.isDictationChord(presses) { - onCommand(["command": "dictationHoldStart"]) + if !dictationHeld { + dictationHeld = true + onCommand(["command": "dictationHoldStart"]) + } return } @@ -58,8 +62,14 @@ public final class T3KeyboardCommandsView: ExpoView { } public override func pressesEnded(_ presses: Set, with event: UIPressesEvent?) { - if holdsToTalk, Self.isDictationChord(presses) { - onCommand(["command": "dictationHoldEnd"]) + // The modifier can be released before D. Matching the original chord at + // key-up misses that order and leaves capture running indefinitely. + if dictationHeld, presses.contains(where: { press in + guard let key = press.key else { return false } + return key.charactersIgnoringModifiers.lowercased() == "d" + || !key.modifierFlags.contains(.alternate) + }) { + endDictationHold() return } @@ -69,14 +79,20 @@ public final class T3KeyboardCommandsView: ExpoView { public override func pressesCancelled(_ presses: Set, with event: UIPressesEvent?) { // A cancelled press still has to stop the recording, or releasing the key // outside the app leaves the microphone running. - if holdsToTalk, Self.isDictationChord(presses) { - onCommand(["command": "dictationHoldEnd"]) + if dictationHeld { + endDictationHold() return } super.pressesCancelled(presses, with: event) } + private func endDictationHold() { + guard dictationHeld else { return } + dictationHeld = false + onCommand(["command": "dictationHoldEnd"]) + } + /// Option plus D, matched on the unmodified character so a layout that puts /// something else on that key still works. Command is deliberately not part /// of it: iPadOS binds Command plus Option plus D to showing the dock. @@ -91,6 +107,7 @@ public final class T3KeyboardCommandsView: ExpoView { func setEnabledCommands(_ commands: [String]) { enabledCommands = Set(commands) + if !holdsToTalk { endDictationHold() } if isFirstResponder { resignFirstResponder() } @@ -136,6 +153,7 @@ public final class T3KeyboardCommandsView: ExpoView { public override func didMoveToWindow() { super.didMoveToWindow() + if window == nil { endDictationHold() } reclaimFirstResponderIfAvailable() } diff --git a/apps/mobile/modules/t3-review-diff/ios/T3ReviewDiffView.swift b/apps/mobile/modules/t3-review-diff/ios/T3ReviewDiffView.swift index cb9d9790284d..5ebc4cd8d4aa 100644 --- a/apps/mobile/modules/t3-review-diff/ios/T3ReviewDiffView.swift +++ b/apps/mobile/modules/t3-review-diff/ios/T3ReviewDiffView.swift @@ -2682,12 +2682,16 @@ private final class SourceTextView: UITextView { backgroundColor = theme.background tintColor = .systemBlue sourceLayout.numberColor = theme.mutedText - sourceLayout.numberFont = .monospacedSystemFont(ofSize: style.lineNumberFontSize, - weight: style.lineNumberFontWeight) + sourceLayout.numberFont = .monospacedSystemFont( + ofSize: style.lineNumberFontSize, + weight: style.lineNumberFontWeight + ) let digits = (String(lines.count) as NSString).size(withAttributes: [.font: sourceLayout.numberFont]).width sourceLayout.gutterWidth = max(style.gutterWidth, digits + 24) - textContainerInset = UIEdgeInsets(top: 8, left: sourceLayout.gutterWidth, - bottom: 96 + safeAreaInsets.bottom, right: 16) + textContainerInset = UIEdgeInsets( + top: 8, left: sourceLayout.gutterWidth, + bottom: 96 + safeAreaInsets.bottom, right: 16 + ) wraps = style.contentWidth < 32_000 textContainer.widthTracksTextView = wraps textContainer.size = CGSize(width: wraps ? max(1, bounds.width - textContainerInset.left - 16) : 32_000, diff --git a/apps/mobile/modules/t3-voice/ios/BackgroundActivity.swift b/apps/mobile/modules/t3-voice/ios/BackgroundActivity.swift index 0d181d9535ac..5b6e9a019af2 100644 --- a/apps/mobile/modules/t3-voice/ios/BackgroundActivity.swift +++ b/apps/mobile/modules/t3-voice/ios/BackgroundActivity.swift @@ -9,17 +9,27 @@ import UIKit /// the promise never settles, and the composer is left in a phase it cannot /// leave. The assertion does not make the work unlimited; it makes the work /// finish or stop cleanly instead of vanishing. -enum BackgroundActivity { - static func begin(_ name: String) async -> UIBackgroundTaskIdentifier { - await MainActor.run { - UIApplication.shared.beginBackgroundTask(withName: name, expirationHandler: nil) +@MainActor +final class BackgroundActivity { + private var identifier: UIBackgroundTaskIdentifier = .invalid + + static func begin( + _ name: String, + onExpiration: @escaping @Sendable () -> Void + ) -> BackgroundActivity { + let activity = BackgroundActivity() + activity.identifier = UIApplication.shared.beginBackgroundTask(withName: name) { [weak activity] in + // iOS requires ending the assertion when time expires. Keeping it open + // until inference finishes can terminate the process and lose the draft. + onExpiration() + Task { @MainActor in activity?.end() } } + return activity } - static func end(_ identifier: UIBackgroundTaskIdentifier) async { + func end() { guard identifier != .invalid else { return } - await MainActor.run { - UIApplication.shared.endBackgroundTask(identifier) - } + UIApplication.shared.endBackgroundTask(identifier) + identifier = .invalid } } diff --git a/apps/mobile/modules/t3-voice/ios/FluidAudioEngine.swift b/apps/mobile/modules/t3-voice/ios/FluidAudioEngine.swift index 310c419f1fe3..f502a1944f7b 100644 --- a/apps/mobile/modules/t3-voice/ios/FluidAudioEngine.swift +++ b/apps/mobile/modules/t3-voice/ios/FluidAudioEngine.swift @@ -142,8 +142,15 @@ actor FluidAudioEngine { func transcribe( audioPath: String, locale: String?, - speakerFiltering: Bool + speakerFiltering: Bool, + model: (id: String, folder: URL), + diarizerFolder: URL? ) async throws -> VoiceTranscriptionOutput { + // Preparation may have finished minutes ago, before a memory warning. + try await prepare(modelId: model.id, modelFolder: model.folder) + if speakerFiltering, let diarizerFolder { + try await prepareDiarizer(modelFolder: diarizerFolder) + } guard let asrManager else { throw VoiceEngineError.modelUnavailable("No speech model is loaded.") } diff --git a/apps/mobile/modules/t3-voice/ios/T3VoiceModule.swift b/apps/mobile/modules/t3-voice/ios/T3VoiceModule.swift index 4a0ffba768ff..0a78112e12e0 100644 --- a/apps/mobile/modules/t3-voice/ios/T3VoiceModule.swift +++ b/apps/mobile/modules/t3-voice/ios/T3VoiceModule.swift @@ -102,18 +102,25 @@ public class T3VoiceModule: Module { AsyncFunction("transcribe") { (operationId: String, modelId: String, audioPath: String, locale: String?, speakerFiltering: Bool, promise: Promise) in self.run(operationId: operationId, promise: promise) { + guard let folder = try Self.resolveModelFolder(modelId) else { + throw VoiceEngineError.modelUnavailable("Model \(modelId) is not installed.") + } let before = DeviceMemory.footprint() defer { Self.reportRunCost(stage: "transcribe", modelId: modelId, before: before) } guard FluidAudioEngine.asrVersion(forModelId: modelId) != nil else { - let text = try await self.engine.transcribe(audioPath: audioPath, locale: locale) + let text = try await self.engine.transcribe( + audioPath: audioPath, locale: locale, modelId: modelId, modelFolder: folder + ) return Self.encode(VoiceTranscriptionOutput(text: text, speakerFiltering: .notRequested)) } let output = try await self.fluidAudio.transcribe( audioPath: audioPath, locale: locale, - speakerFiltering: speakerFiltering + speakerFiltering: speakerFiltering, + model: (id: modelId, folder: folder), + diarizerFolder: speakerFiltering ? try Self.resolveModelFolder(FluidAudioEngine.diarizerModelId) : nil ) return Self.encode(output) } @@ -270,10 +277,13 @@ public class T3VoiceModule: Module { // Inference outlives a home-button press often enough that this matters: // without the assertion the promise never settles and the composer is // stuck in a phase it cannot leave. - let assertion = await BackgroundActivity.begin("T3Voice.\(operationId)") - defer { Task { await BackgroundActivity.end(assertion) } } + let assertion = await BackgroundActivity.begin("T3Voice.\(operationId)") { + Task { await self.operations.cancel(operationId) } + } + defer { Task { await assertion.end() } } do { + try Task.checkCancellation() let value = try await work() try Task.checkCancellation() promise.resolve(value) diff --git a/apps/mobile/modules/t3-voice/ios/WhisperKitEngine.swift b/apps/mobile/modules/t3-voice/ios/WhisperKitEngine.swift index 5846f9dda19f..075c0098de9a 100644 --- a/apps/mobile/modules/t3-voice/ios/WhisperKitEngine.swift +++ b/apps/mobile/modules/t3-voice/ios/WhisperKitEngine.swift @@ -18,13 +18,14 @@ actor WhisperKitEngine { /// Set while a load is in flight so concurrent callers await the same work /// instead of loading a second copy of the same multi-hundred-megabyte model. private var loadTask: Task? + private var loadingModelId: String? func prepare(modelId: String, modelFolder: URL) async throws -> WhisperKit { if let whisperKit, loadedModelId == modelId { return whisperKit } - if let loadTask, loadedModelId == modelId { + if let loadTask, loadingModelId == modelId { return try await loadTask.value } @@ -50,31 +51,31 @@ actor WhisperKitEngine { return try await WhisperKit(config) } - loadedModelId = modelId + loadingModelId = modelId loadTask = task do { let loaded = try await task.value whisperKit = loaded + loadedModelId = modelId loadTask = nil + loadingModelId = nil return loaded } catch { loadTask = nil - if whisperKit == nil { - loadedModelId = nil - } + loadingModelId = nil throw error } } - func transcribe(audioPath: String, locale: String?) async throws -> String { - guard let whisperKit else { - throw VoiceEngineError.modelUnavailable("No speech model is loaded.") - } + func transcribe(audioPath: String, locale: String?, modelId: String, modelFolder: URL) async throws -> String { + // Memory pressure can evict the model while the microphone is running. + let whisperKit = try await prepare(modelId: modelId, modelFolder: modelFolder) + try Task.checkCancellation() let options = DecodingOptions( task: .transcribe, - language: locale, + language: locale.flatMap { Locale(identifier: $0).language.languageCode?.identifier }, skipSpecialTokens: true ) diff --git a/apps/mobile/src/features/voice-input/ComposerDictationControl.tsx b/apps/mobile/src/features/voice-input/ComposerDictationControl.tsx index 8ab87d4c9a6a..dc7a406b52cc 100644 --- a/apps/mobile/src/features/voice-input/ComposerDictationControl.tsx +++ b/apps/mobile/src/features/voice-input/ComposerDictationControl.tsx @@ -415,7 +415,13 @@ export function ComposerDictationStartAction(props: { const openSettings = props.state.phase === "error" && props.state.errorAction === "settings"; return ( (preferences ? resolveVoiceCleanupSettings(preferences) : null), @@ -180,10 +183,10 @@ export function useVoiceInputController(input: { const settings = cleanupSettingsRef.current; return settings ? getLocalVoiceCleanup(settings) : null; }, - persistPendingTranscript: (pending) => { + persistPendingTranscript: async (pending) => { // Stamped here, not in the controller: the timestamp exists only so a // later launch can tell this record from one a dead session left behind. - savePreferencesRef.current({ + await persistPreferencesRef.current({ voicePendingTranscript: { ...pending, capturedAt: Date.now() }, }); }, @@ -256,7 +259,9 @@ export function useVoiceInputController(input: { const sampleRecording = () => { if (controller.currentState.phase !== "recording") return; const status = recorder.getStatus(); - if (!status.isRecording) return; + const isRecording = recorder.isRecording; + void controller.handleRecordingProgress({ ...status, isRecording }); + if (!isRecording) return; const level = normalizeVoiceInputDecibels(status.metering); const history = audioLevelsRef.current; @@ -315,10 +320,14 @@ export function useVoiceInputController(input: { return true; }, [canStart, controller]); - // Releasing the keys ends the recording. Holding through a phase that is not - // recording is not an error; there is simply nothing to stop. + // Key release also queues a stop during microphone startup, so a quick hold + // cannot leave capture running after the keys are already up. const endHold = useCallback(() => { - if (controller.currentState.phase !== "recording") return false; + if ( + controller.currentState.phase !== "recording" && + controller.currentState.phase !== "preparing" + ) + return false; void controller.stop(); return true; }, [controller]); diff --git a/docs/internals/voice-input.md b/docs/internals/voice-input.md index ea3e5c9e025a..71309272a960 100644 --- a/docs/internals/voice-input.md +++ b/docs/internals/voice-input.md @@ -126,18 +126,33 @@ transcript is committed. ## Durability -**The recording audio is deleted after the cleanup stage, not after transcription.** - -**The raw transcript is written to disk before cleanup starts.** Cleanup loads a +**Failed transcription retains the recording for an explicit retry.** The microphone action retries +that file without reopening capture. Dismissing the error or leaving the composer releases it; +successful insertion deletes it. Audio is currently retained only for the mounted controller, not +across process termination. + +Expo can pause an `AVAudioRecorder` without a completion event. The metering loop checks the native +`isRecording` property and asks the controller to finish the captured file when capture stops. +Backgrounding and recorder errors also finish the captured audio, with an interruption notice. +The captured URI takes precedence over the recorder's current URI because a media-services reset +can replace the recorder with a new, empty file. Speech engines ensure their model is loaded again +when transcription begins, since a memory warning can evict it during a long recording. + +**The raw transcript's disk write is awaited before cleanup starts.** Cleanup loads a multi-hundred-megabyte model, and an allocation that gets the app jetsam-killed raises no error to catch. On the next launch the record is offered back into the draft it belongs to, matched on owner, -and discarded once the user accepts or dismisses it. +and discarded once the user accepts or dismisses it. A failed recovery write skips cleanup and +commits raw text. Normal resource release does not clear an uncommitted transcript, including when +navigation or a changed draft prevented insertion. Cleanup degrades to the raw transcript on a throw, a cancel, a rewrite the model never finished, -empty output, or an output-to-input length ratio outside a defined band. A local model given a -transcript it does not understand will answer it, translate it, or apologize; all three miss the -ratio. The timeout is enforced natively, between generated tokens, because nothing in JS can -interrupt a running model. +empty output, an output-to-input length ratio outside 0.85–1.6, or a changed final three words +(ignoring punctuation and capitalization). The ending check is deliberately conservative and can +reject a legitimate spelling correction. These checks are heuristics, not proof of semantic +equivalence. A local model can answer, translate, or summarize a transcript; length and ending +checks catch some of these failures. The timeout is enforced natively, between generated tokens, +because nothing in JS can interrupt a running model. Native background assertions end on expiration +and request cancellation of the associated operation. **A rewrite carries whether the model finished it, and an unfinished one is never committed.** Generation stops at the model's end of turn, at the output-token cap, or at the timeout. The last two diff --git a/docs/user/composer.md b/docs/user/composer.md index 48b66568a188..2e52e782aedf 100644 --- a/docs/user/composer.md +++ b/docs/user/composer.md @@ -243,10 +243,13 @@ Recording starts right away. The first dictation after opening the app may still speech model when you finish talking; if so the composer says so, and the wait is only ever once per model. With a hardware keyboard, hold `Option+D` to talk. **Settings, Voice Input** switches that to press once to start and again to finish, or turns it off. -A recording can be up to five minutes long. Canceling voice input, leaving the screen, or an audio -interruption discards the new recording and keeps your existing draft and attachments. PseudoCode -deletes the local audio file when it is done with it. It sends only the normal message text when you -submit the draft. +A recording can be up to five minutes long. If the microphone is interrupted or the app moves to +the background, PseudoCode finishes transcribing the audio captured so far and tells you that +recording stopped. If transcription fails, tap the microphone to retry the saved recording. +Dismissing that error discards the saved audio. Canceling voice input or leaving the screen during +recording discards the new recording and keeps your existing draft and attachments. PseudoCode +deletes the local audio file after successful transcription and cleanup. It sends only the normal +message text when you submit the draft. ### Choosing a speech model @@ -273,8 +276,10 @@ Losing your own words would be worse than leaving a stray voice in. ### Cleaning up transcripts Turn on **Clean up transcripts** and a language model on your device rewrites what you said as -written text: punctuation, capitalization, and obvious mishearings fixed, filler words removed. It -does not answer, summarize, or add anything. +written text: punctuation, capitalization, and obvious mishearings fixed, filler words removed. +If cleanup times out, produces an incomplete rewrite, or changes the ending, PseudoCode keeps the +original transcription and tells you. A correction to the last few words can also trigger this +conservative fallback. Three cleanup models are available, trading speed for quality. You can edit the instructions they follow and reset them to the default at any time. diff --git a/packages/client-runtime/src/voice-input/cleanup.test.ts b/packages/client-runtime/src/voice-input/cleanup.test.ts index 23bdd6bdcff7..910fb5562d9c 100644 --- a/packages/client-runtime/src/voice-input/cleanup.test.ts +++ b/packages/client-runtime/src/voice-input/cleanup.test.ts @@ -67,6 +67,25 @@ describe("resolveCleanupOutcome", () => { }); }); + it("keeps the ending of a long dictation even when the rewrite reports completion", () => { + const body = "Review the implementation and preserve each instruction. ".repeat(30); + const raw = `${body}Then run the regression tests before shipping.`; + expect(resolveCleanupOutcome(raw, finished(body))).toMatchObject({ + kind: "raw", + text: raw, + reason: "missing-ending", + }); + }); + + it("accepts punctuation and capitalization changes at the end", () => { + expect( + resolveCleanupOutcome( + "please fix the recording and run the tests", + finished("Please fix the recording, and run the tests!"), + ).kind, + ).toBe("cleaned"); + }); + it("does not apply the ratio to short transcripts that legitimately change length", () => { expect(resolveCleanupOutcome("um yeah ok", finished("Yeah, OK."))).toEqual({ kind: "cleaned", diff --git a/packages/client-runtime/src/voice-input/cleanup.ts b/packages/client-runtime/src/voice-input/cleanup.ts index abde86629054..830b945282ce 100644 --- a/packages/client-runtime/src/voice-input/cleanup.ts +++ b/packages/client-runtime/src/voice-input/cleanup.ts @@ -96,10 +96,16 @@ export const CLEANUP_TIMEOUT_MS = 30_000; const CLEANUP_RATIO_MINIMUM_LENGTH = 24; /** A cleanup pass that lands outside this band rewrote more than it should. */ -const CLEANUP_MINIMUM_RATIO = 0.6; +const CLEANUP_MINIMUM_RATIO = 0.85; const CLEANUP_MAXIMUM_RATIO = 1.6; -export type CleanupDegradeReason = "failed" | "cancelled" | "empty" | "incomplete" | "length-ratio"; +export type CleanupDegradeReason = + | "failed" + | "cancelled" + | "empty" + | "incomplete" + | "length-ratio" + | "missing-ending"; export type CleanupOutcome = | { readonly kind: "cleaned"; readonly text: string } @@ -136,6 +142,16 @@ export function resolveCleanupOutcome(raw: string, cleaned: VoiceCleanupResult): if (ratio < CLEANUP_MINIMUM_RATIO || ratio > CLEANUP_MAXIMUM_RATIO) { return { kind: "raw", text: trimmedRaw, reason: "length-ratio" }; } + + // End-of-generation only says the model stopped willingly. It can still + // omit the last instruction. Require the last three words to survive; + // punctuation and capitalization may change. A spelling correction here + // can conservatively fall back to raw text, which is preferable to loss. + const words = (text: string) => text.toLowerCase().match(/[\p{L}\p{N}]+/gu) ?? []; + const ending = words(trimmedRaw).slice(-3).join(" "); + if (ending && words(trimmedCleaned).slice(-3).join(" ") !== ending) { + return { kind: "raw", text: trimmedRaw, reason: "missing-ending" }; + } } return { kind: "cleaned", text: trimmedCleaned }; diff --git a/packages/client-runtime/src/voice-input/controller.test.ts b/packages/client-runtime/src/voice-input/controller.test.ts index 50aa67fc3319..409522a52aa6 100644 --- a/packages/client-runtime/src/voice-input/controller.test.ts +++ b/packages/client-runtime/src/voice-input/controller.test.ts @@ -526,8 +526,8 @@ describe("VoiceInputController", () => { expect(harness.controller.currentState.error).toContain("no longer available"); }); - it("discards recorder errors and audio interruptions without transcribing", async () => { - const transcribe = vi.fn(async () => "ignored"); + it("transcribes the completed file after an audio interruption before deleting it", async () => { + const transcribe = vi.fn(async () => "captured speech"); const preparationEntered = deferred(); const harness = createHarness({ getTranscriber: () => ({ @@ -547,13 +547,107 @@ describe("VoiceInputController", () => { url: "file:///voice.m4a", }); - expect(harness.commits).toEqual([]); - expect(transcribe).not.toHaveBeenCalled(); - expect(signal.aborted).toBe(true); - expect(harness.controller.currentState.error).toBe("Audio route changed"); + expect(harness.commits[0]?.text).toBe("hello captured speech"); + expect(transcribe).toHaveBeenCalledWith("file:///voice.m4a", { signal }); + expect(harness.recorder.stop).not.toHaveBeenCalled(); + expect(signal.aborted).toBe(false); + expect(harness.controller.currentState.error).toBeNull(); + expect(harness.controller.currentState.notice).toContain("Audio route changed"); expect(harness.deleted).toEqual(["file:///voice.m4a", "file:///reset-empty.m4a"]); }); + it("finishes captured speech when the app backgrounds during recording", async () => { + const transcript = Array.from({ length: 300 }, (_, index) => `word${index}`).join(" "); + const harness = createHarness({ + getTranscriber: () => ({ + prepare: async () => preparedTranscription(async () => ({ text: transcript })), + }), + }); + await harness.controller.start(); + await harness.controller.appMovedToBackground(); + + expect(harness.recorder.stop).toHaveBeenCalledOnce(); + expect(harness.commits[0]?.text).toBe(`hello ${transcript}`); + expect(harness.controller.currentState.notice).toContain("Transcribed the audio captured"); + }); + + it("finishes a microphone pause even when Expo emits no completion event", async () => { + const harness = createHarness(); + await harness.controller.start(); + await harness.controller.handleRecordingProgress({ isRecording: false }); + await harness.controller.handleRecordingProgress({ isRecording: false }); + expect(harness.commits[0]?.text).toBe("hello new text"); + expect(harness.commits).toHaveLength(1); + expect(harness.controller.currentState.phase).toBe("idle"); + }); + + it("uses the original file when a media reset replaces the native recorder", async () => { + const transcribe = vi.fn(async () => ({ + text: "original speech", + })); + const harness = createHarness({ + getTranscriber: () => ({ prepare: async () => preparedTranscription(transcribe) }), + }); + await harness.controller.start(); + harness.recorder.uri = "file:///new-empty.m4a"; + await harness.controller.handleRecorderStatus({ + isFinished: true, + hasError: true, + error: null, + url: null, + }); + expect(transcribe.mock.calls[0]?.[0]).toBe("file:///voice.m4a"); + expect(harness.commits[0]?.text).toBe("hello original speech"); + }); + + it("retries failed transcription with the saved audio instead of recording again", async () => { + const transcribe = vi + .fn() + .mockRejectedValueOnce(new Error("inference failed")) + .mockResolvedValueOnce({ text: "all the recorded words" }); + const harness = createHarness({ + getTranscriber: () => ({ prepare: async () => preparedTranscription(transcribe) }), + }); + await harness.controller.start(); + await harness.controller.stop(); + expect(harness.deleted).toEqual([]); + await harness.controller.start(); + expect(harness.recorder.record).toHaveBeenCalledOnce(); + expect(transcribe.mock.calls.map(([uri]) => uri)).toEqual([ + "file:///voice.m4a", + "file:///voice.m4a", + ]); + expect(harness.commits[0]?.text).toBe("hello all the recorded words"); + expect(harness.deleted).toContain("file:///voice.m4a"); + }); + + it("deletes retained audio when a failed dictation is explicitly dismissed", async () => { + const harness = createHarness({ + getTranscriber: () => ({ + prepare: async () => + preparedTranscription(async () => { + throw new Error("failed"); + }), + }), + }); + await harness.controller.start(); + await harness.controller.stop(); + expect(harness.deleted).toEqual([]); + harness.controller.cancel(); + expect(harness.deleted).toEqual(["file:///voice.m4a"]); + }); + + it("honors key release while microphone permission is still pending", async () => { + const permission = deferred<{ granted: boolean; canAskAgain: boolean }>(); + const harness = createHarness({ requestPermission: () => permission.promise }); + const starting = harness.controller.start(); + await harness.controller.stop(); + permission.resolve({ granted: true, canAskAgain: true }); + await starting; + expect(harness.recorder.stop).toHaveBeenCalledOnce(); + expect(harness.controller.currentState.phase).toBe("idle"); + }); + it("cancels preparation when the app reaches the background", async () => { const preparation = deferred(); const preparationEntered = deferred(); @@ -593,7 +687,9 @@ describe("VoiceInputController cleanup stage", () => { prepare: async () => preparedTranscription(transcribingText(async () => RAW)), }), getCleanup: () => cleanup, - persistPendingTranscript: (pending) => persisted.push(pending), + persistPendingTranscript: (pending) => { + persisted.push(pending); + }, clearPendingTranscript: () => { cleared += 1; }, @@ -615,14 +711,16 @@ describe("VoiceInputController cleanup stage", () => { } it("commits the cleaned transcript and reports a cleaning phase while it runs", async () => { - const harness = cleanupHarness(() => rewrote("Add a retry button to the connection settings.")); + const harness = cleanupHarness(() => + rewrote("Add a retry button to the connection settings screen."), + ); await recordAndStop(harness); expect(harness.phases).toEqual(["preparing", "recording", "transcribing", "cleaning", "idle"]); expect(harness.commits).toEqual([ { - text: "hello Add a retry button to the connection settings.", - selection: { start: 52, end: 52 }, + text: "hello Add a retry button to the connection settings screen.", + selection: { start: 59, end: 59 }, }, ]); }); @@ -635,6 +733,42 @@ describe("VoiceInputController cleanup stage", () => { expect(harness.cleared()).toBe(1); }); + it("waits for the recovery write before allocating the cleanup model", async () => { + const saved = deferred(); + const saving = deferred(); + const prepareCleanup = vi.fn(async () => ({ clean: async (text: string) => rewrote(text) })); + const harness = createHarness({ + persistPendingTranscript: () => { + saving.resolve(undefined); + return saved.promise; + }, + getCleanup: () => ({ prepare: prepareCleanup }), + }); + await harness.controller.start(); + const stopping = harness.controller.stop(); + await saving.promise; + expect(prepareCleanup).not.toHaveBeenCalled(); + saved.resolve(undefined); + await stopping; + expect(prepareCleanup).toHaveBeenCalledOnce(); + expect(harness.commits[0]?.text).toBe("hello new text"); + }); + + it("inserts raw speech without cleanup if its recovery write fails", async () => { + const prepareCleanup = vi.fn(); + const harness = createHarness({ + persistPendingTranscript: async () => { + throw new Error("disk full"); + }, + getCleanup: () => ({ prepare: prepareCleanup }), + }); + await harness.controller.start(); + await harness.controller.stop(); + expect(prepareCleanup).not.toHaveBeenCalled(); + expect(harness.commits[0]?.text).toBe("hello new text"); + expect(harness.controller.currentState.notice).toContain("recovery copy"); + }); + it("commits the raw transcript when cleanup throws", async () => { const harness = cleanupHarness(async () => { throw new Error("model unavailable"); @@ -695,7 +829,7 @@ describe("VoiceInputController cleanup stage", () => { expect(harness.controller.currentState.phase).toBe("idle"); }); - it("drops the transcript when the draft owner changes during the rewrite", async () => { + it("retains the transcript for recovery when the draft owner changes during the rewrite", async () => { const cleaning = deferred(); const cleaningEntered = deferred(); const harness = cleanupHarness((_transcript, { signal }) => { @@ -712,6 +846,8 @@ describe("VoiceInputController cleanup stage", () => { expect(harness.commits).toEqual([]); expect(harness.controller.currentState.phase).toBe("idle"); + expect(harness.persisted[0]?.text).toBe(RAW); + expect(harness.cleared()).toBe(0); }); it("does not enter the cleaning phase when cleanup is switched off", async () => { diff --git a/packages/client-runtime/src/voice-input/controller.ts b/packages/client-runtime/src/voice-input/controller.ts index 42fa05735f68..37f46f551133 100644 --- a/packages/client-runtime/src/voice-input/controller.ts +++ b/packages/client-runtime/src/voice-input/controller.ts @@ -1,6 +1,6 @@ import { replaceTextRange } from "@t3tools/shared/composerTrigger"; -import { resolveCleanupOutcome, type VoiceCleanup } from "./cleanup.ts"; +import { resolveCleanupOutcome, type CleanupOutcome, type VoiceCleanup } from "./cleanup.ts"; import type { DictationAnchor } from "./learning.ts"; import { resolveSpeakerFilteringNotice, @@ -98,7 +98,7 @@ export type VoiceInputControllerDependencies = { readonly onStateChange: (state: VoiceInputState) => void; /** Returns null when cleanup is switched off or has no model loaded. */ readonly getCleanup?: () => VoiceCleanup | null; - readonly persistPendingTranscript?: (pending: PendingVoiceTranscript) => void; + readonly persistPendingTranscript?: (pending: PendingVoiceTranscript) => void | Promise; readonly clearPendingTranscript?: () => void; /** * Called once per committed dictation with the span it wrote, so the learning @@ -236,6 +236,8 @@ export class VoiceInputController { private finishing = false; private cleanupAbortController: AbortController | null = null; private pendingTranscriptPersisted = false; + private stopRequested = false; + private retryRecording: { uri: string; ownerKey: string } | null = null; constructor(dependencies: VoiceInputControllerDependencies) { this.dependencies = dependencies; @@ -262,6 +264,7 @@ export class VoiceInputController { const operationToken = ++this.operationToken; const abortController = new AbortController(); this.transcriptionAbortController = abortController; + this.stopRequested = false; this.setState({ phase: "preparing", error: null, errorAction: null, notice: null }); try { @@ -271,6 +274,22 @@ export class VoiceInputController { return; } + if (this.retryRecording) { + if (initiatingDraft.ownerKey !== this.retryRecording.ownerKey) { + this.setError("Return to the draft that owns this recording to retry it.", "retry"); + return; + } + this.recordingUri = this.retryRecording.uri; + this.rememberRecordingUri(this.recordingUri); + this.capturedDraft = initiatingDraft; + this.transcription = runTranscriptionOperation(() => + transcriber.prepare({ signal: abortController.signal }), + ); + this.transcription.catch(() => undefined); + await this.finishRecording(true, this.recordingUri); + return; + } + const permission = await this.dependencies.requestPermission(); if (!this.isCurrent(operationToken)) return; if (!permission.granted) { @@ -308,6 +327,7 @@ export class VoiceInputController { this.capturedDraft = capturedDraft; this.dependencies.recorder.record({ forDuration: VOICE_RECORDING_LIMIT_SECONDS }); this.setState({ phase: "recording", error: null, errorAction: null, notice: null }); + if (this.stopRequested) await this.stop(); } catch { if (this.isCurrent(operationToken)) this.setError("Could not start voice recording.", "retry"); @@ -321,6 +341,10 @@ export class VoiceInputController { } stop(): Promise { + if (this.state.phase === "preparing") { + this.stopRequested = true; + return Promise.resolve(); + } if (this.state.phase !== "recording") return Promise.resolve(); return this.finishRecording(false, null); } @@ -333,6 +357,7 @@ export class VoiceInputController { if (this.state.notice) this.setState(IDLE_STATE); return; case "error": + this.discardRetryRecording(); this.setState(IDLE_STATE); return; case "preparing": @@ -362,7 +387,11 @@ export class VoiceInputController { if (this.state.phase !== "recording") return; this.rememberRecordingUri(completedUri); this.recordingUri = completedUri ?? this.recordingUri; - return this.discardRecording(message); + return this.finishRecording( + completedUri !== null, + completedUri, + `${message} Transcribed the audio captured before it stopped.`, + ); } appMovedToBackground(): Promise | void { @@ -402,6 +431,17 @@ export class VoiceInputController { } } + /** Expo can pause without emitting a completion event. Never keep showing capture then. */ + handleRecordingProgress(status: { + isRecording: boolean; + mediaServicesDidReset?: boolean; + }): Promise | void { + if (this.state.phase !== "recording") return; + if (status.mediaServicesDidReset || !status.isRecording) { + return this.interruptRecording(); + } + } + ownerChanged(): void { if (this.state.phase === "idle") return; if (this.state.phase === "cleaning") { @@ -414,6 +454,10 @@ export class VoiceInputController { } dispose(): void { + if (this.state.phase === "error") { + this.discardRetryRecording(); + return; + } if (this.state.phase === "recording") { this.discardRecording(null); return; @@ -441,8 +485,9 @@ export class VoiceInputController { private async finishRecording( alreadyStopped: boolean, completedUri: string | null, + recordingNotice: string | null = null, ): Promise { - if (this.finishing || this.state.phase !== "recording") return; + if (this.finishing) return; this.finishing = true; const operationToken = this.operationToken; this.setState({ phase: "transcribing", error: null, errorAction: null, notice: null }); @@ -450,7 +495,7 @@ export class VoiceInputController { try { if (!alreadyStopped) await this.dependencies.recorder.stop(); await this.releaseAudioSession(); - this.recordingUri = completedUri ?? this.dependencies.recorder.uri ?? this.recordingUri; + this.recordingUri = completedUri ?? this.recordingUri ?? this.dependencies.recorder.uri; this.rememberRecordingUri(this.recordingUri); if (!this.isCurrent(operationToken)) return; if ( @@ -464,6 +509,7 @@ export class VoiceInputController { } const recordingUri = this.recordingUri; + this.retryRecording = { uri: recordingUri, ownerKey: this.capturedDraft.ownerKey }; const signal = this.transcriptionAbortController.signal; // The load usually finished while the user was still talking. When it did @@ -486,29 +532,58 @@ export class VoiceInputController { transcription.transcribe(recordingUri, { signal }), ); transcript = result.text; - notice = result.notice ?? resolveSpeakerFilteringNotice(result.speakerFiltering); + notice = + [recordingNotice, result.notice ?? resolveSpeakerFilteringNotice(result.speakerFiltering)] + .filter(Boolean) + .join(" ") || null; } catch (error) { if (this.isCurrent(operationToken)) { - this.setError(transcriptionErrorMessage(error), "retry"); + this.setError( + `${transcriptionErrorMessage(error)} Retry to use the saved recording.`, + "retry", + ); } return; } if (!this.isCurrent(operationToken)) return; - const cleanup = this.dependencies.getCleanup?.() ?? null; + let cleanup = this.dependencies.getCleanup?.() ?? null; let committedTranscript = transcript; - if (cleanup && transcript.trim().length > 0) { + if (transcript.trim().length > 0) { // The store stamps the time it was written. The controller has no // clock of its own, and the stamp only matters to the code deciding // whether a record outlived the session that made it. - this.dependencies.persistPendingTranscript?.({ - ownerKey: capturedDraft.ownerKey, - revision: capturedDraft.revision, - text: transcript, - }); - this.pendingTranscriptPersisted = true; + try { + await this.dependencies.persistPendingTranscript?.({ + ownerKey: capturedDraft.ownerKey, + revision: capturedDraft.revision, + text: transcript, + }); + this.pendingTranscriptPersisted = true; + } catch { + // Do not risk a large cleanup allocation before the raw words are safe. + cleanup = null; + notice = [ + notice, + "Kept the original transcription because its recovery copy could not be saved.", + ] + .filter(Boolean) + .join(" "); + } + if (!this.isCurrent(operationToken)) return; + } + if (cleanup && transcript.trim().length > 0) { this.setState({ phase: "cleaning", error: null, errorAction: null, notice: null }); - committedTranscript = await this.runCleanup(cleanup, transcript); + const outcome = await this.runCleanup(cleanup, transcript); + committedTranscript = outcome.text; + if (outcome.kind === "raw") { + notice = [ + notice, + "Kept the original transcription because cleanup was skipped or could not preserve it.", + ] + .filter(Boolean) + .join(" "); + } if (!this.isCurrent(operationToken)) return; } @@ -531,6 +606,11 @@ export class VoiceInputController { } this.dependencies.commitDraft(result.text, result.selection); + this.retryRecording = null; + if (this.pendingTranscriptPersisted) { + this.pendingTranscriptPersisted = false; + this.dependencies.clearPendingTranscript?.(); + } this.dependencies.onDictationCommitted?.({ ownerKey: capturedDraft.ownerKey, revision: capturedDraft.revision, @@ -596,7 +676,7 @@ export class VoiceInputController { * stops between tokens, so only the side running it can end a run early; a * timer here would abandon the promise while the model kept burning battery. */ - private async runCleanup(cleanup: VoiceCleanup, transcript: string): Promise { + private async runCleanup(cleanup: VoiceCleanup, transcript: string): Promise { const abortController = new AbortController(); this.cleanupAbortController = abortController; @@ -605,9 +685,13 @@ export class VoiceInputController { const prepared = await cleanup.prepare({ signal: abortController.signal }); return prepared.clean(transcript, { signal: abortController.signal }); }); - return resolveCleanupOutcome(transcript, cleaned).text; + return resolveCleanupOutcome(transcript, cleaned); } catch { - return transcript; + return { + kind: "raw", + text: transcript, + reason: abortController.signal.aborted ? "cancelled" : "failed", + }; } finally { if (this.cleanupAbortController === abortController) this.cleanupAbortController = null; } @@ -635,6 +719,7 @@ export class VoiceInputController { this.rememberRecordingUri(this.dependencies.recorder.uri); this.recordingUri = null; for (const uri of this.ownedRecordingUris) { + if (this.state.phase === "error" && uri === this.retryRecording?.uri) continue; try { this.dependencies.deleteRecording(uri); } catch { @@ -642,6 +727,7 @@ export class VoiceInputController { } } this.ownedRecordingUris.clear(); + if (this.state.phase !== "error") this.retryRecording = null; await this.releaseAudioSession(); releaseSession(this.sessionToken); this.sessionToken = null; @@ -649,11 +735,19 @@ export class VoiceInputController { this.transcription = null; this.transcriptionAbortController = null; this.cleanupAbortController = null; - if (this.pendingTranscriptPersisted) { - // Reaching here at all means the process outlived cleanup and the user - // has been told what happened, so the crash-recovery record is spent. - this.pendingTranscriptPersisted = false; - this.dependencies.clearPendingTranscript?.(); + // An uncommitted transcript remains recoverable after a restart, including + // when navigation or a changed draft prevented insertion. + this.pendingTranscriptPersisted = false; + } + + private discardRetryRecording(): void { + const retry = this.retryRecording; + this.retryRecording = null; + if (!retry) return; + try { + this.dependencies.deleteRecording(retry.uri); + } catch { + // The OS may already have removed the cache file. } }