diff --git a/apps/server/src/provider/macosSystemProxy.test.ts b/apps/server/src/provider/macosSystemProxy.test.ts new file mode 100644 index 0000000000000..7aef7a8f75722 --- /dev/null +++ b/apps/server/src/provider/macosSystemProxy.test.ts @@ -0,0 +1,89 @@ +import { assert, describe, it } from "@effect/vitest"; + +import { parseScutilProxy } from "./macosSystemProxy.ts"; + +const dictionary = (...lines: ReadonlyArray) => + [" {", ...lines.map((line) => ` ${line}`), "}", ""].join("\n"); + +describe("parseScutilProxy", () => { + it("ignores disabled, incomplete, SOCKS and PAC proxies", () => { + assert.deepStrictEqual( + parseScutilProxy( + dictionary( + "HTTPEnable : 0", + "HTTPProxy : 10.0.0.1", + "HTTPPort : 3128", + "HTTPSEnable : 1", + "HTTPSProxy : 10.0.0.1", + "ProxyAutoConfigEnable : 1", + "ProxyAutoConfigURLString : http://wpad/proxy.pac", + "SOCKSEnable : 1", + "SOCKSPort : 1080", + "SOCKSProxy : 10.0.0.1", + ), + ), + {}, + ); + assert.deepStrictEqual(parseScutilProxy(""), {}); + }); + + it.each([ + { + name: "an IP range exception", + lines: ["ExceptionsList : {", " 0 : 192.168.0.0/16", "}"], + noProxy: "", + }, + { + name: "a wildcard IP exception", + lines: ["ExceptionsList : {", " 0 : 10.*", "}"], + noProxy: "", + }, + { name: "an IP range in NO_PROXY", lines: [], noProxy: "registry.corp,127.0.0.0/8" }, + { name: "simple hostnames excluded", lines: ["ExcludeSimpleHostnames : 1"], noProxy: "" }, + { name: "an HTTPS-only proxy", lines: ["HTTPEnable : 0"], noProxy: "" }, + { name: "different HTTP and HTTPS proxies", lines: ["HTTPPort : 8080"], noProxy: "" }, + ])("keeps the direct connection with $name, which npm cannot match", ({ lines, noProxy }) => { + const proxy = ["HTTPEnable : 1", "HTTPPort : 3128", "HTTPProxy : 10.0.0.1"]; + const httpsProxy = ["HTTPSEnable : 1", "HTTPSPort : 3128", "HTTPSProxy : 10.0.0.1"]; + assert.deepStrictEqual( + parseScutilProxy(dictionary(...proxy, ...httpsProxy, ...lines), noProxy), + {}, + ); + }); + + it("reads only the top-level settings and merges exceptions into NO_PROXY", () => { + assert.deepStrictEqual( + parseScutilProxy( + dictionary( + "ExceptionsList : {", + " 0 : *.local", + " 1 : 169.254/16", + " 2 : internal.corp", + "}", + "HTTPEnable : 1", + "HTTPPort : 8443", + "HTTPProxy : fd00::1", + "HTTPSEnable : 1", + "HTTPSPort : 8443", + "HTTPSProxy : fd00::1", + "__SCOPED__ : {", + " en0 : {", + " HTTPEnable : 1", + " HTTPPort : 9999", + " HTTPProxy : 10.9.9.9", + " }", + "}", + ), + "registry.corp, ", + ), + { + HTTP_PROXY: "http://[fd00::1]:8443", + http_proxy: "http://[fd00::1]:8443", + HTTPS_PROXY: "http://[fd00::1]:8443", + https_proxy: "http://[fd00::1]:8443", + NO_PROXY: "registry.corp,localhost,127.0.0.1,::1,[::1],.local,169.254.0.0/16,internal.corp", + no_proxy: "registry.corp,localhost,127.0.0.1,::1,[::1],.local,169.254.0.0/16,internal.corp", + }, + ); + }); +}); diff --git a/apps/server/src/provider/macosSystemProxy.ts b/apps/server/src/provider/macosSystemProxy.ts new file mode 100644 index 0000000000000..3579082b5747d --- /dev/null +++ b/apps/server/src/provider/macosSystemProxy.ts @@ -0,0 +1,115 @@ +import * as HostProcess from "@t3tools/shared/HostProcess"; +import * as Effect from "effect/Effect"; +import { ChildProcess, ChildProcessSpawner } from "effect/process"; + +const PROXY_ENV_KEYS = ["HTTP_PROXY", "HTTPS_PROXY", "ALL_PROXY"]; +// CFNetwork never sends loopback through the system proxy; env-proxy clients do unless told. +const LOOPBACK_NO_PROXY = ["localhost", "127.0.0.1", "::1", "[::1]"]; + +function proxyUrl(fields: ReadonlyMap, prefix: "HTTP" | "HTTPS"): string | null { + const host = fields.get(`${prefix}Proxy`); + const port = fields.get(`${prefix}Port`); + if (fields.get(`${prefix}Enable`) !== "1" || !host || !port || !/^\d+$/.test(port)) { + return null; + } + return `http://${host.includes(":") && !host.startsWith("[") ? `[${host}]` : host}:${port}`; +} + +// macOS writes `*.local` and `169.254/16` by default. +const LINK_LOCAL_RANGE = /^169\.254(?:\.0){0,2}\/16$/; + +function toNoProxyEntry(exception: string): string { + if (LINK_LOCAL_RANGE.test(exception)) return "169.254.0.0/16"; + return exception.startsWith("*.") ? exception.slice(1) : exception; +} + +/** + * npm matches NO_PROXY by hostname suffix and ignores IP ranges, so a bypassed + * private registry would go through the proxy. Such bypass lists keep the direct connection. + */ +function hasUnrepresentableException(exception: string): boolean { + if (LINK_LOCAL_RANGE.test(exception)) return false; + return ( + exception.includes("/") || exception.slice(exception.startsWith("*.") ? 2 : 0).includes("*") + ); +} + +/** Maps `scutil --proxy` output to proxy env vars; SOCKS and PAC are ignored. */ +export function parseScutilProxy(output: string, inheritedNoProxy = ""): Record { + const fields = new Map(); + const exceptions: Array = []; + let depth = 0; + let inExceptions = false; + for (const line of output.split("\n")) { + const text = line.trim(); + if (text === "}") { + depth -= 1; + if (depth <= 1) inExceptions = false; + continue; + } + const separator = text.indexOf(" : "); + const key = separator === -1 ? "" : text.slice(0, separator); + const value = separator === -1 ? "" : text.slice(separator + 3).trim(); + if (text.endsWith("{")) { + inExceptions = depth === 1 && key === "ExceptionsList"; + depth += 1; + } else if (depth === 1 && key) { + fields.set(key, value); + } else if (inExceptions && depth === 2 && value) { + exceptions.push(value); + } + } + + const proxy = proxyUrl(fields, "HTTP"); + const inherited = inheritedNoProxy.split(",").flatMap((entry) => entry.trim() || []); + if ( + // npm sends http requests via https_proxy and https via HTTP_PROXY, so only one shared proxy keeps macOS's routing. + !proxy || + proxy !== proxyUrl(fields, "HTTPS") || + // npm cannot match dotless hosts either, so honouring this needs a direct connection. + fields.get("ExcludeSimpleHostnames") === "1" || + [...inherited, ...exceptions].some(hasUnrepresentableException) + ) { + return {}; + } + const noProxy = [...inherited, ...LOOPBACK_NO_PROXY, ...exceptions.map(toNoProxyEntry)].join(","); + return { + HTTP_PROXY: proxy, + http_proxy: proxy, + HTTPS_PROXY: proxy, + https_proxy: proxy, + NO_PROXY: noProxy, + no_proxy: noProxy, + }; +} + +function hasProxyEnv(env: NodeJS.ProcessEnv): boolean { + return PROXY_ENV_KEYS.some((key) => Boolean(env[key] || env[key.toLowerCase()])); +} + +export const readMacosSystemProxyEnv = Effect.fn("MacosSystemProxy.readEnv")(function* ( + spawner: ChildProcessSpawner.ChildProcessSpawner["Service"], + commandEnv: NodeJS.ProcessEnv = {}, +) { + const inherited = yield* HostProcess.Environment; + // curl reads the lowercase form first. + const noProxy = + commandEnv.no_proxy || commandEnv.NO_PROXY || inherited.no_proxy || inherited.NO_PROXY || ""; + if ( + (yield* HostProcess.Platform) !== "darwin" || + hasProxyEnv(inherited) || + hasProxyEnv(commandEnv) || + noProxy.trim() === "*" + ) { + return {}; + } + const output = yield* spawner + .string( + ChildProcess.make("/usr/sbin/scutil", ["--proxy"], { stdin: "ignore", stderr: "ignore" }), + ) + .pipe( + Effect.timeout("2 seconds"), + Effect.orElseSucceed(() => ""), + ); + return parseScutilProxy(output, noProxy); +}); diff --git a/apps/server/src/provider/providerMaintenanceRunner.test.ts b/apps/server/src/provider/providerMaintenanceRunner.test.ts index adf53c5267203..fd88bb3093b0d 100644 --- a/apps/server/src/provider/providerMaintenanceRunner.test.ts +++ b/apps/server/src/provider/providerMaintenanceRunner.test.ts @@ -975,6 +975,114 @@ describe("providerMaintenanceRunner", () => { ), ); }); + + it.effect.each([ + { name: "an inherited proxy", inherited: { https_proxy: "http://proxy.corp:8080" }, env: {} }, + { name: "an updater proxy", inherited: {}, env: { HTTPS_PROXY: "http://instance:3128" } }, + { name: "a bypass-all NO_PROXY", inherited: {}, env: { no_proxy: "*" } }, + ])("does not read the macOS system proxy with $name", ({ inherited, env }) => { + const spawns: Array<{ command: string; env: NodeJS.ProcessEnv | undefined }> = []; + return Effect.gen(function* () { + const { registry } = yield* makeRegistry(baseProvider); + const runner = yield* makeTestRunner({ + ...registry, + getProviderMaintenanceCapabilitiesForInstance: (_instanceId, provider) => + Effect.succeed({ + ...lifecycleFor(provider), + update: { + command: "npm install -g @openai/codex@latest", + executable: "npm", + args: ["install", "-g", "@openai/codex@latest"], + lockKey: "npm-global", + env, + }, + }), + }); + + yield* runner.updateProvider(CODEX_DRIVER); + + assert.deepStrictEqual(spawns, [ + { command: "npm", env: Object.keys(env).length > 0 ? env : undefined }, + ]); + }).pipe( + Effect.provide( + Layer.mergeAll( + Layer.succeed(HostProcess.Platform, "darwin"), + Layer.succeed(HostProcess.Environment, inherited), + layerLatestVersionHttpClient("0.0.0"), + layerMockSpawner((command, _args, options) => { + spawns.push({ command, env: options.env }); + return { stdout: "updated" }; + }), + ), + ), + ); + }); + + it.effect("passes the macOS system proxy to the updater when the env has none", () => { + const updaterEnvs: Array = []; + return Effect.gen(function* () { + const { registry } = yield* makeRegistry(baseProvider); + const runner = yield* makeTestRunner({ + ...registry, + getProviderMaintenanceCapabilitiesForInstance: (_instanceId, provider) => + Effect.succeed({ + ...lifecycleFor(provider), + update: { + command: "npm install -g @openai/codex@latest", + executable: "npm", + args: ["install", "-g", "@openai/codex@latest"], + lockKey: "npm-global", + env: { NO_PROXY: "registry.corp" }, + }, + }), + }); + + yield* runner.updateProvider(CODEX_DRIVER); + + assert.strictEqual(updaterEnvs.length, 1); + assert.strictEqual(updaterEnvs[0]?.HTTPS_PROXY, "http://127.0.0.1:7890"); + assert.strictEqual(updaterEnvs[0]?.HTTP_PROXY, "http://127.0.0.1:7890"); + assert.strictEqual( + updaterEnvs[0]?.no_proxy, + "registry.corp,localhost,127.0.0.1,::1,[::1],.local,169.254.0.0/16", + ); + }).pipe( + Effect.provide( + Layer.mergeAll( + Layer.succeed(HostProcess.Platform, "darwin"), + Layer.succeed(HostProcess.Environment, { PATH: "/usr/bin:/bin:/usr/sbin:/sbin" }), + layerLatestVersionHttpClient("0.0.0"), + layerMockSpawner((command, _args, options) => { + if (command.split("/").pop() === "scutil") { + return { + stdout: [ + " {", + " ExceptionsList : {", + " 0 : *.local", + " 1 : 169.254/16", + " }", + " FTPPassive : 1", + " HTTPEnable : 1", + " HTTPPort : 7890", + " HTTPProxy : 127.0.0.1", + " HTTPSEnable : 1", + " HTTPSPort : 7890", + " HTTPSProxy : 127.0.0.1", + "}", + "", + ].join("\n"), + }; + } + if (command === "npm") { + updaterEnvs.push(options.env); + } + return { stdout: "updated" }; + }), + ), + ), + ); + }); }); it.effect.each([ diff --git a/apps/server/src/provider/providerMaintenanceRunner.ts b/apps/server/src/provider/providerMaintenanceRunner.ts index d3fe7fdf86739..ea66ab82d85dd 100644 --- a/apps/server/src/provider/providerMaintenanceRunner.ts +++ b/apps/server/src/provider/providerMaintenanceRunner.ts @@ -23,6 +23,7 @@ import { HttpClient } from "effect/http"; import { ChildProcess, ChildProcessSpawner } from "effect/process"; import * as NodeUtil from "node:util"; +import { readMacosSystemProxyEnv } from "./macosSystemProxy.ts"; import * as ModelManifest from "./ModelManifest.ts"; import { resolveProviderCompatibility } from "./providerCompatibility.ts"; import * as ProviderRegistry from "./ProviderRegistry.ts"; @@ -307,17 +308,22 @@ export const make = Effect.fn("ProviderMaintenanceRunner.make")(function* () { const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; const httpClient = yield* HttpClient.HttpClient; const latestVersions = yield* ProviderLatestVersions.ProviderLatestVersions; - const runMaintenanceCommand = ( - update: ProviderMaintenanceCommandAction, - onProgress: (line: string) => Effect.Effect, - ) => - runProviderMaintenanceCommandWithSpawner({ - spawner, - command: update.executable, - args: update.args, - onProgress, - ...(update.env ? { env: update.env } : {}), - }); + const runMaintenanceCommand = Effect.fn("ProviderMaintenanceRunner.runMaintenanceCommand")( + function* ( + update: ProviderMaintenanceCommandAction, + onProgress: (line: string) => Effect.Effect, + ) { + // Read per update: the user may toggle the system proxy while T3 Code runs. + const env = { ...update.env, ...(yield* readMacosSystemProxyEnv(spawner, update.env)) }; + return yield* runProviderMaintenanceCommandWithSpawner({ + spawner, + command: update.executable, + args: update.args, + onProgress, + ...(Object.keys(env).length > 0 ? { env } : {}), + }); + }, + ); const commandCoordinator = yield* makeProviderMaintenanceCommandCoordinator({ makeAlreadyRunningError: () => new ServerProviderUpdateError({