Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,8 @@ import { VoiceSettingsPanel } from "./VoiceSettingsPanel";

vi.mock(import("../../speech/useEnvironmentSpeechInput"), { spy: true });

it("preserves the model test transcript when its model is filtered out", async () => {
it("keeps setup available with a non-English locale and preserves a filtered test transcript", async () => {
using _languages = vi.spyOn(navigator, "languages", "get").mockReturnValue(["fr-FR"]);
const target = new PrimaryConnectionTarget({
environmentId: EnvironmentId.make("voice-filter"),
label: "Test environment",
Expand All @@ -40,12 +41,12 @@ it("preserves the model test transcript when its model is filtered out", async (
supportsStreaming: false,
supportsTranslation: false,
supportsLanguageDetection: false,
active: true,
state: "installed",
active: false,
state: "downloadable",
};
const status = {
supported: true,
state: "ready",
state: "missing-model",
modelId: model.id,
model: model.name,
size: model.size,
Expand Down Expand Up @@ -103,6 +104,7 @@ it("preserves the model test transcript when its model is filtered out", async (
model: null,
downloading: false,
error: null,
show: async () => {},
setOpen: () => {},
download: async () => {},
cancelDownload: async () => {},
Expand All @@ -128,6 +130,13 @@ it("preserves the model test transcript when its model is filtered out", async (
<VoiceSettingsPanel />
</SettingsScopeProvider>,
);
await expect
.element(screen.getByRole("button", { name: "Set up voice input", exact: true }))
.toBeVisible();
await expect.element(screen.getByText("Bilingual", { exact: true })).toBeVisible();
await userEvent.click(
screen.getByRole("button", { name: "Set up voice input", exact: true }),
);
const testModel = screen.getByRole("button", { name: "Test model", exact: true });
await expect.element(testModel).toBeVisible();
await userEvent.click(testModel);
Expand All @@ -137,13 +146,22 @@ it("preserves the model test transcript when its model is filtered out", async (
const search = screen.getByRole("textbox", { name: "Search transcription models" });
await userEvent.fill(search, "no matching model");
await expect.element(screen.getByText("No models match.")).toBeVisible();
await expect
.element(screen.getByRole("button", { name: "Set up voice input", exact: true }))
.toBeVisible();
await userEvent.clear(search);
await userEvent.fill(screen.getByRole("textbox", { name: "Search languages" }), "fr");
const french = screen.getByRole("button", { name: "French", exact: true });
await userEvent.click(french);
await expect.element(screen.getByText("Bilingual", { exact: true })).toBeVisible();
await expect.element(screen.getByText("B+", { exact: true })).toBeVisible();
expect(screen.container.querySelector("#transcription-test")).toHaveAttribute("hidden");
await expect
.element(
screen.container.querySelector<HTMLParagraphElement>(
'[aria-label="Test transcript"]',
),
)
.not.toBeVisible();
await userEvent.click(french);
await expect.element(result).toBeVisible();
await vi.waitFor(() =>
Expand Down
53 changes: 46 additions & 7 deletions apps/web/src/components/settings/VoiceSettingsPanel.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,9 @@ import {
} from "lucide-react";
import { useEffect, useRef, useState, type ReactNode } from "react";

import { useAtomValue } from "@effect/atom-react";
import { shortcutLabelForCommand } from "../../keybindings";
import { primaryServerKeybindingsAtom } from "../../state/server";
import { useStableCallback } from "~/hooks/useStableCallback";
import { listBrowserMicrophones, type BrowserMicrophone } from "../../speech/browserMicrophones";
import {
Expand Down Expand Up @@ -126,6 +129,7 @@ type ModelCardProps = {
readonly onSelect: () => void;
readonly onCancel: () => void;
readonly onDelete: () => void;
readonly hidden?: boolean;
readonly testControls?: ReactNode;
readonly children?: ReactNode;
};
Expand All @@ -147,7 +151,7 @@ function ModelCard(props: ModelCardProps) {
const downloading = model.state === "downloading" || model.state === "verifying";
const progress = model.downloaded === undefined ? 0 : (model.downloaded / model.size) * 100;
return (
<div className="flex flex-wrap items-start gap-3 px-3 py-2.5 sm:px-4">
<div hidden={props.hidden} className="flex flex-wrap items-start gap-3 px-3 py-2.5 sm:px-4">
<div className="min-w-0 flex-1">
<div className="flex flex-wrap items-center gap-1.5">
<span className="text-sm font-medium leading-5">{model.name}</span>
Expand Down Expand Up @@ -737,7 +741,9 @@ function SpeechModelBrowser(props: {
(!query || (model.name + " " + model.description).toLocaleLowerCase().includes(query)),
)
.toSorted((left, right) => modelSortOrder(left) - modelSortOrder(right));
const activeModel = models.find((model) => model.active);
const activeModel = models.find(
(model) => model.active || (status?.supported && model.id === status.modelId),
);
const renderedModels =
activeModel && !sortedModels.includes(activeModel)
? [activeModel, ...sortedModels]
Expand Down Expand Up @@ -856,11 +862,15 @@ function SpeechModelBrowser(props: {
runtime.runPromise(removeEnvironmentSpeechModel(prepared, model.id)),
),
};
return model.active ? (
return model === activeModel ? (
<TranscriptionTest
key={model.id}
card={card}
hidden={!sortedModels.includes(model)}
onSetup={() => {
setSearch("");
setSelection([]);
}}
environmentId={props.testEnvironmentId}
projectId={props.projectId}
projectName={props.projectName}
Expand All @@ -881,6 +891,7 @@ function SpeechModelBrowser(props: {
function TranscriptionTest(props: {
readonly card: ModelCardProps;
readonly hidden: boolean;
readonly onSetup: () => void;
readonly environmentId: EnvironmentId;
readonly projectId: ProjectId | null;
readonly projectName: string | null;
Expand All @@ -907,9 +918,24 @@ function TranscriptionTest(props: {
commitDraft: (text) => setTranscript(text),
});
return (
<SettingsSearchTarget {...searchableSetting("transcription-test")} hidden={props.hidden}>
<SettingsSearchTarget {...searchableSetting("transcription-test")}>
{speech.available && speech.state.phase === "idle" ? (
<div className="flex justify-end px-3 py-2 sm:px-4">
<Button
size="sm"
onClick={() => {
props.onSetup();
setShowTranscript(true);
void speech.setup.show();
}}
>
Set up voice input
</Button>
</div>
) : null}
<ModelCard
{...props.card}
hidden={props.hidden}
testControls={
<div className="flex min-w-0 max-w-full items-center gap-1">
<ComposerSpeechRecordingPill
Expand Down Expand Up @@ -1001,6 +1027,10 @@ function TranscriptionTest(props: {
}

export function VoiceSettingsPanel() {
const keybindings = useAtomValue(primaryServerKeybindingsAtom);
const dictationShortcut = shortcutLabelForCommand(keybindings, "composer.dictation", {
context: { terminalFocus: false, terminalOpen: false },
});
const cleanupEnabled = useCleanupGrant() !== null;
const settings = useClientSettings();
const settingsScope = useOptionalSettingsScope();
Expand Down Expand Up @@ -1113,7 +1143,11 @@ export function VoiceSettingsPanel() {
<SettingsRow
{...searchableSetting("dictation-shortcut-mode")}
title="Dictation shortcut"
description="How the dictation shortcut starts and stops recording."
description={
dictationShortcut
? `Press ${dictationShortcut} to dictate in the focused editor. Choose how it starts and stops recording.`
: "No dictation shortcut is assigned. Set one in Keybindings settings."
}
control={
<Select
value={settings.voiceShortcutMode}
Expand Down Expand Up @@ -1143,8 +1177,8 @@ export function VoiceSettingsPanel() {
title="Send microphone audio to"
description={
<>
Microphone audio is sent to this environment to transcribe, then discarded. Choose Off
to turn voice input off on this device. This choice is saved on this device only.
Audio is sent here to transcribe, then discarded. Choose Off to disable voice input.
Saved on this device only.
</>
}
control={
Expand Down Expand Up @@ -1182,6 +1216,11 @@ export function VoiceSettingsPanel() {
</Select>
}
>
{!destination.environmentId ? (
<p className="mt-2 text-xs text-muted-foreground">
Choose an environment to enable voice input, then select Set up voice input below.
</p>
) : null}
{destination.environmentId && chosenUnsupported ? (
<p className="mt-2 text-xs text-destructive">
{chosenEnvironment.label} can’t transcribe audio. Pick an environment that can to use
Expand Down
34 changes: 32 additions & 2 deletions apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -39,10 +39,16 @@ const mocks = vi.hoisted(() => ({
recordingOptions: [] as SpeechTranscriptionOptions[],
projectWords: {} as Record<string, SpeechCustomWords>,
transcripts: [] as Array<string | Error>,
navigate: vi.fn(),
committed: vi.fn(),
draft: "",
}));

vi.mock("@tanstack/react-router", async (importOriginal) => ({
...(await importOriginal<typeof import("@tanstack/react-router")>()),
useNavigate: () => mocks.navigate,
}));

vi.mock("../state/session", () => ({
useEnvironmentScope: (id: string) => mocks.canDictate.has(id),
usePreparedConnection: (environmentId: string | null) => {
Expand Down Expand Up @@ -166,6 +172,7 @@ const THREAD_ENV = "thread-env";

function setup(options: { readonly destination?: string | null } = {}) {
mocks.settings = {};
mocks.navigate.mockReset();
setDestination(options.destination ?? null);
chooseTranscriptCleanup(false);
mocks.primaryEnvironmentId = "primary-env";
Expand Down Expand Up @@ -238,7 +245,11 @@ it.each([
await act(() => mocks.voice.start());
expectNoAudioSent();
expect(mocks.committed).not.toHaveBeenCalled();
expect(mocks.voice.available).toBe(false);
expect(mocks.voice.available).toBe(true);
expect(mocks.navigate).toHaveBeenCalledWith({
to: "/settings/voice",
hash: "voice-transcription-destination",
});
expect(mocks.voice.transcriptionEnvironmentId).toBeNull();
});

Expand Down Expand Up @@ -309,6 +320,25 @@ it("retries a failed transcription on the same chosen environment", async () =>
expect(mocks.committed).toHaveBeenCalledExactlyOnceWith("second try", expect.anything());
});

it.each(["missing-model", "ready"] as const)(
"opens setup explicitly with a %s model without recording",
async (modelState) => {
setup({ destination: "env-a" });
mocks.modelState = modelState;
await using _screen = await render(<Probe />);
await act(() => mocks.voice.setup.show());
expect(mocks.voice.setup).toMatchObject({
open: true,
step: modelState === "missing-model" ? 0 : 1,
});
expect(mocks.microphoneRequests).toEqual([]);
expect(mocks.transcribeCalls).toEqual([]);
await act(() => mocks.voice.setup.startRecording());
expect(mocks.voice.setup.open).toBe(modelState === "missing-model");
expect(mocks.microphoneRequests).toEqual(modelState === "ready" ? ["env-a"] : []);
},
);

it("reopens setup after reconnecting during a download and ignores its stale completion", async () => {
setup({ destination: "env-a" });
mocks.modelState = "missing-model";
Expand Down Expand Up @@ -370,7 +400,7 @@ it("revokes across windows even before the storage event and survives stale pref
storage.removeItem(key);
expect(canSend()).toBe(false);
writeBrowserClientSettings({ ...DEFAULT_CLIENT_SETTINGS, wordWrap: false });
await vi.waitFor(() => expect(mocks.voice.available).toBe(false));
await vi.waitFor(() => expect(mocks.voice.transcriptionEnvironmentId).toBeNull());
expect(mocks.voice.state.phase).toBe("idle");
await act(() => mocks.voice.stop());
expect(mocks.transcribeCalls).toEqual([]);
Expand Down
25 changes: 19 additions & 6 deletions apps/web/src/speech/useEnvironmentSpeechInput.ts
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ import { resolveProjectSettings } from "@t3tools/shared/projectSettings";
import { resolveSpeechVocabulary } from "@t3tools/shared/speech";
import * as Option from "effect/Option";
import { useEffect, useRef, useState } from "react";
import { useNavigate } from "@tanstack/react-router";

import { useStableCallback } from "~/hooks/useStableCallback";
import { toastManager } from "../components/ui/toast";
Expand Down Expand Up @@ -119,6 +120,7 @@ export function useVoiceTranscriptionDestination() {
}

export function useEnvironmentSpeechInput(input: HookInput) {
const navigate = useNavigate();
const destination = useVoiceTranscriptionDestination();
const prepared = destination.prepared;
const grantId = destination.grantId;
Expand Down Expand Up @@ -423,7 +425,12 @@ export function useEnvironmentSpeechInput(input: HookInput) {
if (controller.currentState.phase === "recording") void controller.stop();
});

const start = useStableCallback(async () => {
const start = useStableCallback(async (setupOnly = false) => {
if (latestRef.current.input.disabled) return;
if (!grantId) {
await navigate({ to: "/settings/voice", hash: "voice-transcription-destination" });
return;
}
const request = ++startRequestRef.current;
const controller = controllerRef.current;
const owner = latestRef.current.input.ownerKey;
Expand Down Expand Up @@ -458,8 +465,12 @@ export function useEnvironmentSpeechInput(input: HookInput) {
});
return;
}
if (latestStatus.state === "missing-model") {
setSetup({ ...INITIAL_SETUP, open: true });
if (setupOnly || latestStatus.state === "missing-model") {
setSetup({
...INITIAL_SETUP,
open: true,
step: latestStatus.state === "missing-model" ? 0 : 1,
});
return;
}
setLevel(0);
Expand Down Expand Up @@ -513,9 +524,10 @@ export function useEnvironmentSpeechInput(input: HookInput) {

return {
available:
prepared !== null &&
status?.prepared === prepared &&
status.value?.supported !== false &&
(!grantId ||
(prepared !== null &&
status?.prepared === prepared &&
status.value?.supported !== false)) &&
typeof navigator !== "undefined" &&
Boolean(navigator.mediaDevices?.getUserMedia) &&
typeof AudioWorkletNode !== "undefined",
Expand All @@ -524,6 +536,7 @@ export function useEnvironmentSpeechInput(input: HookInput) {
transcriptionEnvironmentLabel: destination.label,
setup: {
...setup,
show: () => start(true),
setOpen: (open: boolean) => setSetup((current) => ({ ...current, open })),
download: downloadSetupModel,
cancelDownload: cancelSetupDownload,
Expand Down
8 changes: 6 additions & 2 deletions docs/user/voice-input.md
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@ to a named environment on that phone, open **Settings → Voice → Transcribe o
On web and desktop, transcription runs on one of your T3 Code environments.
Nothing is sent until you choose one: open **Settings → Voice** and set
**Send microphone audio to** the environment that should transcribe for you.
The microphone and dictation shortcut open these settings when no destination
has been chosen.
Microphone audio recorded on this device is sent to that environment over your
authenticated connection, transcribed there, and discarded. The choice is saved
on this device only, so every computer and browser you use asks for it once.
Expand All @@ -25,8 +27,10 @@ substitutes another environment.

## Set up a model

The first time you dictate, T3 Code offers to download the recommended English
model to the chosen environment. In **Settings → Voice → Transcription**, select
Select **Set up voice input** in Voice settings to open
the setup wizard. It offers a model download if needed, then lets you choose a
microphone and start dictating. The first time you dictate with a missing model,
the same wizard opens. In **Settings → Voice → Transcription**, select
the languages you dictate in to compare models that support all of them.
Accuracy shows how often the model got words or characters wrong in each
language; speed is the time to transcribe 30 seconds on a reference GPU. Choose
Expand Down
Loading