feat(voice): add custom OpenAI-compatible cloud speech provider

apps/mobile/src/features/settings/CloudVoiceSection.tsx

@@ -276,6 +276,7 @@ function CloudChoices({ environment, active, onClose }: Props & { onClose: () =>
       return (
         <ListSection footer={`Audio is sent through ${environment.label} using its API key.`}>
           {CloudSpeechProvider.literals.map((provider, index) => {
+            if (provider === "custom") return null;
             const label = CLOUD_SPEECH_LABELS[provider];
             const routed = provider === "openrouter";
             const key = saved[provider];

apps/mobile/src/features/voice-input/voiceDestination.ts

@@ -52,6 +52,8 @@ function cloudVoiceDestination(
 ): CloudDestination | null {
   if (provider === "openrouter")
     return model ? { kind: "cloud", environmentId, provider, model } : null;
+  // Custom endpoints are configured from web and desktop only.
+  if (provider === "custom") return null;
   return { kind: "cloud", environmentId, provider };
 }
 

apps/server/src/serverSettings.test.ts

@@ -1811,6 +1811,7 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("server settings"
           openai: SECRET_REDACTED,
           xai: SECRET_REDACTED,
           openrouter: "",
+          custom: "",
         });
         assert.equal(legacy.sourceControlHosts.bitbucket?.accessToken, SECRET_REDACTED);
         assert.equal(legacy.usageLimitSources[hub]?.managementKey, SECRET_REDACTED);
@@ -1841,6 +1842,7 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("server settings"
           openai: usageReference,
           xai: "literal-fixture",
           openrouter: "",
+          custom: "",
         });
         assert.deepEqual(echoed.easDeviceHosts[0]?.accessToken, easReference);
         assert.deepEqual(echoed.simlockDeviceHosts[0]?.agentToken, simlockReference);
@@ -1926,7 +1928,12 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("server settings"
         "ghe.acme.test": work,
       });
       assert.equal(removed.usageLimitSources[hub]?.managementKey, "");
-      assert.deepEqual(removed.cloudSpeechCredentials, { openai: "", xai: work, openrouter: "" });
+      assert.deepEqual(removed.cloudSpeechCredentials, {
+        openai: "",
+        xai: work,
+        openrouter: "",
+        custom: "",
+      });
       assert.equal(removed.easDeviceHosts[0]?.accessToken, "");
       assert.deepEqual(Object.keys(removed.secretProviders), ["work"]);
     }).pipe(Effect.provide(layerServerSettingsWithSecrets())),
@@ -1939,6 +1946,7 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("server settings"
         "cloudSpeechCredentials.openai",
         "cloudSpeechCredentials.xai",
         "cloudSpeechCredentials.openrouter",
+        "cloudSpeechCredentials.custom",
         "easDeviceHosts.[].accessToken",
         "simlockDeviceHosts.[].agentToken",
         "usageLimitSources.*.managementKey",

apps/server/src/sourceControl/sourceControlHostSecrets.test.ts

@@ -176,7 +176,12 @@ it.effect("reads a collected reference again once it is invalidated", () =>
       resolver,
       Effect.succeed({
         sourceControlHosts: hosts,
-        cloudSpeechCredentials: { openai: source("voice#key"), xai: "", openrouter: "" },
+        cloudSpeechCredentials: {
+          openai: source("voice#key"),
+          xai: "",
+          openrouter: "",
+          custom: "",
+        },
         usageLimitSources,
         textGenerators: {},
         easDeviceHosts,

apps/server/src/speech/CloudSpeech.test.ts

@@ -49,7 +49,13 @@ it.effect.each(["openai", "xai"] as const)("sends only to %s using its own saved
     yield* service.setKey(provider === "openai" ? "xai" : "openai", Redacted.make("other"));
     expect((yield* Effect.flip(transcribe()))._tag).toBe("CloudSpeechError");
     yield* service.setKey(provider, Redacted.make("chosen"));
-    expect(yield* service.configured).toEqual({ openai: true, xai: true, openrouter: false });
+    expect(yield* service.configured).toEqual({
+      openai: true,
+      xai: true,
+      openrouter: false,
+      custom: false,
+      customEndpoint: "",
+    });
     yield* Effect.flip(transcribe(new Uint8Array(3)));
     expect(upstream).not.toHaveBeenCalled();
     expect(yield* transcribe()).toBe("hello");
@@ -198,3 +204,85 @@ it.effect("removing a key while a catalog read is pending prevents the upload",
     expect(exit._tag).toBe("Failure");
   }),
 );
+
+it.effect("sends to a configured custom OpenAI-compatible endpoint as multipart", () =>
+  Effect.gen(function* () {
+    const upstream = vi.fn<typeof fetch>(async () => Response.json({ text: "hello" }));
+    const stack = (baseUrl: string | undefined) =>
+      CloudSpeech.layer.pipe(
+        Layer.provide(FetchHttpClient.layer),
+        Layer.provide(
+          ServerSettings.ServerSettingsService.layerTest(
+            baseUrl === undefined ? {} : { cloudSpeechCustomBaseUrl: baseUrl },
+          ),
+        ),
+        Layer.provide(Layer.mock(SecretResolver.SecretResolver)({})),
+      );
+    const service = yield* CloudSpeech.CloudSpeech.pipe(
+      Effect.provide(yield* Layer.build(stack("http://localhost:8080/v1"))),
+    );
+    // Builds a separate stack for a different endpoint configuration.
+    const attempt = (baseUrl: string | undefined, model: string | undefined) =>
+      Effect.gen(function* () {
+        const context = yield* Layer.build(stack(baseUrl));
+        const scoped = yield* CloudSpeech.CloudSpeech.pipe(Effect.provide(context));
+        yield* scoped.setKey("custom", Redacted.make("custom-key"));
+        return yield* scoped
+          .transcribe(
+            "custom",
+            new Uint8Array(new Float32Array([0, 0.5]).buffer),
+            { ...DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS, speechLanguage: "en" },
+            model,
+            baseUrl,
+          )
+          .pipe(Effect.provideService(FetchHttpClient.Fetch, upstream));
+      });
+    yield* service.setKey("custom", Redacted.make("custom-key"));
+    const pcm = new Uint8Array(new Float32Array([0, 0.5]).buffer);
+    const transcribe = (model: string | undefined, endpoint = "http://localhost:8080/v1") =>
+      service
+        .transcribe(
+          "custom",
+          pcm,
+          { ...DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS, speechLanguage: "en" },
+          model,
+          endpoint,
+        )
+        .pipe(Effect.provideService(FetchHttpClient.Fetch, upstream));
+    expect(yield* Effect.flip(attempt(undefined, "whisper-1"))).toMatchObject({
+      provider: "custom",
+      reason: "model-unavailable",
+    });
+    expect(upstream).not.toHaveBeenCalled();
+    expect(yield* Effect.flip(transcribe(undefined))).toMatchObject({
+      reason: "model-unavailable",
+    });
+    expect(yield* Effect.flip(transcribe("whisper-1", "http://localhost:9090/v1"))).toMatchObject({
+      reason: "model-unavailable",
+    });
+    expect(upstream).not.toHaveBeenCalled();
+    expect(yield* transcribe("whisper-1")).toBe("hello");
+    const [url, init] = upstream.mock.calls[0]!;
+    expect(String(url)).toBe("http://localhost:8080/v1/audio/transcriptions");
+    expect(init?.redirect).toBe("error");
+    expect(new Headers(init?.headers).get("authorization")).toBe("Bearer custom-key");
+    const form = init?.body as FormData;
+    expect(form.get("model")).toBe("whisper-1");
+    expect(form.get("language")).toBe("en");
+    expect([...form.keys()].at(-1)).toBe("file");
+    expect(yield* Effect.flip(attempt("http://example.com/v1", "whisper-1"))).toMatchObject({
+      reason: "model-unavailable",
+    });
+    expect(yield* Effect.flip(attempt("ftp://localhost/v1", "whisper-1"))).toMatchObject({
+      reason: "model-unavailable",
+    });
+    expect(yield* Effect.flip(attempt(undefined, "whisper-1"))).toMatchObject({
+      reason: "model-unavailable",
+    });
+    expect(upstream).toHaveBeenCalledTimes(1);
+    yield* service.setKey("custom", null);
+    expect((yield* service.configured).custom).toBe(false);
+    expect(yield* Effect.flip(transcribe("whisper-1"))).toMatchObject({ reason: "missing-key" });
+    expect(upstream).toHaveBeenCalledTimes(1);
+  }),
+);

apps/server/src/speech/CloudSpeech.ts

@@ -101,6 +101,33 @@ function openRouterBody(model: string, audio: ArrayBuffer, language: string) {
   });
 }
 
+/** The transcriptions URL for a configured custom base URL, or null when it is unusable. */
+function customTranscriptionsUrl(baseUrl: string) {
+  let url: URL;
+  try {
+    url = new URL(baseUrl);
+  } catch {
+    return null;
+  }
+  if (url.search || url.hash || url.username || url.password) return null;
+  const hostname = url.hostname.toLowerCase();
+  const host =
+    hostname.startsWith("[") && hostname.endsWith("]") ? hostname.slice(1, -1) : hostname;
+  const local =
+    host === "localhost" || host === "::1" || host === "127.0.0.1" || host.endsWith(".localhost");
+  if (url.protocol !== "https:" && !(url.protocol === "http:" && local)) return null;
+  return `${url.origin}${url.pathname.replace(/\/+$/, "")}/audio/transcriptions`;
+}
+
+/** The generic multipart body every OpenAI-compatible transcription server accepts. */
+function customBody(model: string, audio: ArrayBuffer, language: string) {
+  const form = new FormData();
+  form.set("model", model);
+  if (language !== "auto") form.set("language", language);
+  form.set("file", new Blob([audio], { type: "audio/wav" }), "dictation.wav");
+  return HttpClientRequest.bodyFormData(form);
+}
+
 export class CloudSpeech extends Context.Service<
   CloudSpeech,
   {
@@ -116,6 +143,7 @@ export class CloudSpeech extends Context.Service<
       pcm: Uint8Array,
       options: SpeechTranscriptionOptions,
       model?: string | undefined,
+      endpoint?: string | undefined,
     ) => Effect.Effect<string, CloudSpeechError | SpeechInvalidAudioError | ServerSettingsError>;
   }
 >()("t3/speech/CloudSpeech") {}
@@ -129,11 +157,13 @@ export const layer = Layer.effect(
     const credentials = settings.getSettings.pipe(
       Effect.map((value) => value.cloudSpeechCredentials),
     );
-    const configured = credentials.pipe(
-      Effect.map(({ openai, xai, openrouter }) => ({
-        openai: Boolean(openai),
-        xai: Boolean(xai),
-        openrouter: Boolean(openrouter),
+    const configured = settings.getSettings.pipe(
+      Effect.map((value) => ({
+        openai: Boolean(value.cloudSpeechCredentials.openai),
+        xai: Boolean(value.cloudSpeechCredentials.xai),
+        openrouter: Boolean(value.cloudSpeechCredentials.openrouter),
+        custom: Boolean(value.cloudSpeechCredentials.custom),
+        customEndpoint: value.cloudSpeechCustomBaseUrl,
       })),
     );
     const execute = (request: HttpClientRequest.HttpClientRequest) =>
@@ -184,11 +214,19 @@ export const layer = Layer.effect(
         pcm: Uint8Array,
         options: SpeechTranscriptionOptions,
         model?: string,
+        endpoint?: string,
       ) {
         const source = (yield* credentials)[provider];
         if (!source) return yield* new CloudSpeechError({ provider, reason: "missing-key" });
         if (provider === "openrouter" && (!model || !isModelId(model)))
           return yield* new CloudSpeechError({ provider, reason: "model-unavailable" });
+        // A custom grant names the base URL it consented to; audio never goes anywhere else.
+        const consentedUrl = Effect.map(settings.getSettings, ({ cloudSpeechCustomBaseUrl }) =>
+          endpoint === cloudSpeechCustomBaseUrl ? customTranscriptionsUrl(endpoint) : null,
+        );
+        const customUrl = provider === "custom" ? yield* consentedUrl : null;
+        if (provider === "custom" && (!model || !customUrl))
+          return yield* new CloudSpeechError({ provider, reason: "model-unavailable" });
         const samples = yield* Effect.try({
           try: () => decodeSpeechPcm(pcm),
           catch: () =>
@@ -208,11 +246,15 @@ export const layer = Layer.effect(
         );
         if (!isApiKey(key)) return yield* unreadable();
         const audio = wave(samples);
-        const request = HttpClientRequest.post(endpoints[provider].url).pipe(
+        const request = HttpClientRequest.post(
+          provider === "custom" ? customUrl! : endpoints[provider].url,
+        ).pipe(
           HttpClientRequest.bearerToken(Redacted.make(key)),
           provider === "openrouter"
             ? openRouterBody(model!, audio, language)
-            : audioBody(provider, audio, language, options),
+            : provider === "custom"
+              ? customBody(model!, audio, language)
+              : audioBody(provider, audio, language, options),
         );
         const unavailable = () => new CloudSpeechError({ provider, reason: "unavailable" });
         // A catalog fetch or vault lookup may have outlived the credential it started with.
@@ -222,6 +264,8 @@ export const layer = Layer.effect(
             provider,
             reason: current ? "key-changed" : "missing-key",
           });
+        if (provider === "custom" && (yield* consentedUrl) !== customUrl)
+          return yield* new CloudSpeechError({ provider, reason: "model-unavailable" });
         const response = yield* execute(request).pipe(Effect.mapError(unavailable));
         if (response.status < 200 || response.status >= 300)
           return yield* new CloudSpeechError({

apps/server/src/speech/http.test.ts

@@ -221,9 +221,7 @@ effectIt.effect(
         headers: { "content-type": "application/octet-stream" },
         body: encodeSpeechPcmRequest(new Uint8Array(4), DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS),
       };
-      expect((yield* call("openrouter/transcribe?model=openrouter%2Fauto", audio)).status).toBe(
-        400,
-      );
+      expect((yield* call("openrouter/transcribe?model=", audio)).status).toBe(400);
       expect((yield* call("openrouter/transcribe?model=acme%2Ftranscribe-1", audio)).status).toBe(
         422,
       );

apps/server/src/speech/http.ts

@@ -105,6 +105,7 @@ export const speechHttpApiLayer = HttpApiBuilder.group(
               decoded.value.pcm,
               decoded.value.options,
               args.query.model,
+              args.query.endpoint,
             )
             .pipe(
               Effect.catchTags({