Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion apps/mobile/src/native/voiceTranscription.android.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -78,10 +78,11 @@ it.each([
selection: { start: 16, end: 16 },
revision: 1,
};
const prepareToRecordAsync = vi.fn(async () => {});
const controller = new VoiceInputController({
recorder: {
uri: "file:///acceptance.m4a",
prepareToRecordAsync: async () => {},
prepareToRecordAsync,
record: () => {},
stop: async () => {},
},
Expand All @@ -101,4 +102,5 @@ it.each([
expect(commit).not.toHaveBeenCalled();
expect(draft.text).toBe("Preserved draft.");
expect(controller.currentState.error).toMatch(expected);
expect(prepareToRecordAsync).toHaveBeenCalledTimes(phase === "preparation" ? 0 : 1);
});
95 changes: 94 additions & 1 deletion apps/mobile/src/native/voiceTranscription.ios.test.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
import type { TranscriptionResult } from "@react-native-ai/apple/src/NativeAppleTranscription";
import { afterEach, beforeEach, describe, expect, it, vi } from "vite-plus/test";

import { VoiceTranscriptionError } from "@t3tools/client-runtime/voice-input";
import { VoiceInputController, VoiceTranscriptionError } from "@t3tools/client-runtime/voice-input";
import { resetVoiceInputGlobalsForTests } from "../../../../packages/client-runtime/src/voice-input/controller";

const mocks = vi.hoisted(() => ({
isAvailable: vi.fn<(locale: string) => boolean>(),
Expand Down Expand Up @@ -65,8 +66,43 @@ async function transcribeOnce(signal = new AbortController().signal) {
.catch((error: unknown) => error);
}

/** Drives one on-device dictation through the controller the composer uses. */
function createDictation() {
const recorder = {
uri: "file:///voice.m4a",
prepareToRecordAsync: async () => {},
record: vi.fn(),
stop: vi.fn(async () => {}),
};
const commits: string[] = [];
const deleted: string[] = [];
const ended = Promise.withResolvers<void>();
const controller = new VoiceInputController({
recorder,
getTranscriber: () => ({
prepare: async (options) => (await getLocalVoiceTranscriber())!.prepare(options),
}),
requestPermission: async () => ({ granted: true, canAskAgain: true }),
configureRecording: async () => {},
releaseRecording: async () => {},
deleteRecording: (uri) => deleted.push(uri),
readDraft: () => ({
ownerKey: "environment:thread",
text: "",
selection: { start: 0, end: 0 },
revision: 1,
}),
commitDraft: (text) => commits.push(text),
onStateChange: (state) => {
if (state.phase === "idle" || state.phase === "error") ended.resolve();
},
});
return { controller, recorder, commits, deleted, ended: ended.promise };
}

beforeEach(() => {
vi.resetAllMocks();
resetVoiceInputGlobalsForTests();
useLanguages([], "sv-SE");
mocks.isAvailable.mockReturnValue(true);
mocks.prepare.mockResolvedValue("sv-SE");
Expand Down Expand Up @@ -147,6 +183,63 @@ describe("getLocalVoiceTranscriber", () => {
},
);

it("records while Apple prepares its speech model and transcribes once it is ready", async () => {
const nativePrepare = Promise.withResolvers<string>();
mocks.prepare.mockReturnValue(nativePrepare.promise);
const dictation = createDictation();

await dictation.controller.start();
expect(dictation.recorder.record).toHaveBeenCalledOnce();
expect(dictation.controller.currentState.phase).toBe("recording");

const stopping = dictation.controller.stop();
nativePrepare.resolve("sv-SE");
await stopping;

expect(mocks.transcribe).toHaveBeenCalledOnce();
expect(dictation.commits).toEqual(["Hej världen."]);
expect(dictation.deleted).toEqual(["file:///voice.m4a"]);
});

it.each(["cancel", "cancel after stop", "model failure", "model failure after stop"] as const)(
"discards early audio without a late commit after %s",
async (ending) => {
const nativePrepare = Promise.withResolvers<string>();
mocks.prepare.mockReturnValue(nativePrepare.promise);
const dictation = createDictation();
await dictation.controller.start();

let finishing: Promise<void> | undefined;
const cancelled = ending === "cancel" || ending === "cancel after stop";
if (ending.endsWith("after stop")) finishing = dictation.controller.stop();
if (cancelled) {
dictation.controller.cancel();
} else {
nativePrepare.reject(Object.assign(new Error("download failed"), { code: "Network" }));
}
await finishing;
await dictation.ended;

const next = createDictation();
mocks.prepare.mockResolvedValue("sv-SE");
await next.controller.start();
expect(next.controller.currentState.phase).toBe("recording");
expect(dictation.deleted).toEqual(["file:///voice.m4a"]);
expect(mocks.transcribe).not.toHaveBeenCalled();
await next.controller.stop();
expect(next.commits).toEqual(["Hej världen."]);
nativePrepare.resolve("sv-SE");

expect(dictation.commits).toEqual([]);
expect(dictation.recorder.stop).toHaveBeenCalled();
expect(dictation.controller.currentState).toMatchObject(
cancelled
? { phase: "idle" }
: { phase: "error", error: "Voice transcription could not prepare this language." },
);
},
);

it("does not start native transcription after cancellation during a file read", async () => {
const enteredRead = Promise.withResolvers<void>();
const readResult = Promise.withResolvers<ArrayBuffer>();
Expand Down
45 changes: 37 additions & 8 deletions apps/mobile/src/native/voiceTranscription.ios.ts
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,41 @@ async function prepareVoiceTranscription(
});
}

let locale = locales[0]!;
const model = prepareLocales(locales, signal).then((supported) => {
locale = supported;
});
// Recorder setup can fail before either consumer observes the model.
void model.catch(() => {});

async function ready(signal: AbortSignal) {
throwIfVoiceTranscriptionAborted(signal);
const cancelled = Promise.withResolvers<void>();
const cancel = () => cancelled.resolve();
signal.addEventListener("abort", cancel, { once: true });
try {
await Promise.race([model, cancelled.promise]);
throwIfVoiceTranscriptionAborted(signal);
return locale;
} finally {
signal.removeEventListener("abort", cancel);
}
}
return {
get locale() {
return locale;
},
start: async (_uri, { signal }) => {
await ready(signal);
return null;
},
transcribe: async (uri, options) =>
transcribeVoiceRecording(uri, await ready(options.signal), options),
};
}

async function prepareLocales(locales: readonly string[], signal: AbortSignal): Promise<string> {
throwIfVoiceTranscriptionAborted(signal);
let unsupported: VoiceTranscriptionError | undefined;
for (const locale of locales) {
try {
Expand All @@ -75,17 +110,11 @@ async function prepareVoiceTranscription(
throw unsupported!;
}

async function prepareLocale(
locale: string,
signal: AbortSignal,
): Promise<PreparedVoiceTranscription> {
async function prepareLocale(locale: string, signal: AbortSignal): Promise<string> {
try {
const supportedLocale = await AppleTranscription.prepare(locale);
throwIfVoiceTranscriptionAborted(signal);
return {
locale: supportedLocale,
transcribe: (uri, options) => transcribeVoiceRecording(uri, supportedLocale, options),
};
return supportedLocale;
} catch (error) {
throwIfVoiceTranscriptionAborted(signal);
if (getNativeErrorCode(error) === "AppleTranscriptionUnsupportedLocale") {
Expand Down
4 changes: 4 additions & 0 deletions packages/client-runtime/src/voice-input/controller.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -309,7 +309,9 @@ describe("VoiceInputController", () => {
code: "unavailable",
message: "This device requires iOS 26 or later.",
});
const configureRecording = vi.fn(async () => undefined);
const unavailable = createHarness({
configureRecording,
getTranscriber: () => ({
prepare: async () => {
throw reason;
Expand All @@ -318,6 +320,8 @@ describe("VoiceInputController", () => {
});
await unavailable.controller.start();
expect(unavailable.controller.currentState.error).toBe(reason.message);
// Environment and web transcribers gate capture on consent checked during preparation.
expect(configureRecording).not.toHaveBeenCalled();
expect(unavailable.recorder.record).not.toHaveBeenCalled();

const denied = createHarness({
Expand Down
16 changes: 7 additions & 9 deletions packages/client-runtime/src/voice-input/controller.ts
Original file line number Diff line number Diff line change
Expand Up @@ -236,7 +236,7 @@ export class VoiceInputController {
private operationToken = 0;
private sessionToken: VoiceInputSession | null = null;
private transcription: PreparedVoiceTranscription | null = null;
private liveTranscription: Promise<{ text: string } | { error: unknown }> | null = null;
private liveTranscription: Promise<{ text: string | null } | { error: unknown }> | null = null;
private transcriptionAbortController: AbortController | null = null;
private postProcessingAbortController: AbortController | null = null;
private skipPostProcessingRequested = false;
Expand Down Expand Up @@ -504,17 +504,15 @@ export class VoiceInputController {
const capturedDraft = this.capturedDraft;
let transcript: string;
try {
if (this.liveTranscription) {
const result = await this.liveTranscription;
if ("error" in result) throw result.error;
transcript = result.text;
} else {
transcript = await runTranscriptionOperation(() =>
const live = this.liveTranscription ? await this.liveTranscription : null;
if (live && "error" in live) throw live.error;
transcript =
live?.text ??
(await runTranscriptionOperation(() =>
transcription.streaming
? transcription.streaming.finish({ signal })
: transcription.transcribe(recordingUri!, { signal }),
);
}
));
} catch (error) {
if (this.isCurrent(operationToken)) {
if (
Expand Down
7 changes: 6 additions & 1 deletion packages/client-runtime/src/voice-input/transcription.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,12 @@ export type VoiceStreamingSession = {

export type PreparedVoiceTranscription = {
readonly locale: string;
readonly start?: (uri: string, options: VoiceTranscriptionOptions) => Promise<string>;
/**
* Runs alongside capture once recording starts. Resolves with the transcript when it
* transcribed live, or null when `transcribe` still runs after the recording stops.
* A rejection stops the recording.
*/
readonly start?: (uri: string, options: VoiceTranscriptionOptions) => Promise<string | null>;
readonly transcribe: (uri: string, options: VoiceTranscriptionOptions) => Promise<string>;
/** Present when audio streams during capture; `finish` then replaces `transcribe`. */
readonly streaming?: VoiceStreamingSession;
Expand Down
Loading