From a1770e69830ed33cf1bed5c13a92742c177de44c Mon Sep 17 00:00:00 2001 From: Timothee Guerin Date: Wed, 7 Oct 2026 22:06:49 -0400 Subject: [PATCH] fix(compiler): authenticate registry bootstrap downloads --- .../fix-authenticated-registry-bootstrap.md | 7 + eng/tsp-core/pipelines/jobs/e2e.yml | 4 + .../package-manger/npm-package-download.ts | 9 +- .../src/package-manger/npm-registry.ts | 55 ++++++- packages/compiler/test/e2e/cli/cli.e2e.ts | 50 +++++++ .../npm-package-download.test.ts | 134 ++++++++++++++++++ .../test/package-manager/npm-registry.test.ts | 24 +++- turbo.json | 1 + website/src/content/docs/docs/handbook/cli.md | 16 ++- 9 files changed, 294 insertions(+), 6 deletions(-) create mode 100644 .chronus/changes/fix-authenticated-registry-bootstrap.md diff --git a/.chronus/changes/fix-authenticated-registry-bootstrap.md b/.chronus/changes/fix-authenticated-registry-bootstrap.md new file mode 100644 index 00000000000..39a1bc239ba --- /dev/null +++ b/.chronus/changes/fix-authenticated-registry-bootstrap.md @@ -0,0 +1,7 @@ +--- +changeKind: fix +packages: + - "@typespec/compiler" +--- + +Fix package-manager downloads from authenticated registry mirrors by supporting the `TYPESPEC_NPM_REGISTRY_TOKEN` environment variable for TypeSpec's package metadata and tarball requests. diff --git a/eng/tsp-core/pipelines/jobs/e2e.yml b/eng/tsp-core/pipelines/jobs/e2e.yml index f86f55ab8f9..19ec1c21138 100644 --- a/eng/tsp-core/pipelines/jobs/e2e.yml +++ b/eng/tsp-core/pipelines/jobs/e2e.yml @@ -33,9 +33,13 @@ jobs: scriptType: bash scriptLocation: inlineScript inlineScript: pnpm run test:e2e + env: + TYPESPEC_NPM_REGISTRY_TOKEN: $(System.AccessToken) - ${{ else }}: - script: pnpm run test:e2e displayName: E2E Tests + env: + TYPESPEC_NPM_REGISTRY_TOKEN: $(System.AccessToken) - task: 1ES.PublishPipelineArtifact@1 inputs: diff --git a/packages/compiler/src/package-manger/npm-package-download.ts b/packages/compiler/src/package-manger/npm-package-download.ts index eb09ad8b2d2..0033c11e324 100644 --- a/packages/compiler/src/package-manger/npm-package-download.ts +++ b/packages/compiler/src/package-manger/npm-package-download.ts @@ -4,7 +4,12 @@ import { createHash } from "crypto"; import { Readable } from "stream"; import { extract as tarX } from "tar/extract"; import type { Hash } from "../install/spec.js"; -import { fetchPackageManifest, NpmRegistryError, type NpmPackageVersion } from "./npm-registry.js"; +import { + fetchFromNpmRegistry, + fetchPackageManifest, + NpmRegistryError, + type NpmPackageVersion, +} from "./npm-registry.js"; export async function downloadPackageVersion( packageName: string, @@ -34,7 +39,7 @@ async function downloadAndExtractTarball( ): Promise { let res: Response; try { - res = await fetch(url); + res = await fetchFromNpmRegistry(url); } catch (error) { const message = error instanceof Error ? `: ${error.message}` : ""; throw new NpmRegistryError(`Request to ${url} failed${message}`); diff --git a/packages/compiler/src/package-manger/npm-registry.ts b/packages/compiler/src/package-manger/npm-registry.ts index 80874bf2c73..206c92fa84e 100644 --- a/packages/compiler/src/package-manger/npm-registry.ts +++ b/packages/compiler/src/package-manger/npm-registry.ts @@ -91,6 +91,57 @@ export function getNpmRegistry(): string { return (process.env["TYPESPEC_NPM_REGISTRY"] ?? defaultRegistry).replace(/\/$/, ""); } +/** Only send registry credentials to URLs within the configured registry. */ +function getNpmRegistryHeaders(url: string): Record { + const token = process.env["TYPESPEC_NPM_REGISTRY_TOKEN"]; + if (!token) { + return {}; + } + + const registry = new URL(getNpmRegistry()); + const target = new URL(url); + const registryPath = registry.pathname.replace(/\/$/, ""); + if ( + target.origin !== registry.origin || + (target.pathname !== registryPath && !target.pathname.startsWith(`${registryPath}/`)) + ) { + return {}; + } + + return { Authorization: `Bearer ${token}` }; +} + +export async function fetchFromNpmRegistry( + url: string, + headers: Record = {}, +): Promise { + if (!process.env["TYPESPEC_NPM_REGISTRY_TOKEN"]) { + return fetch(url, { headers }); + } + + // Fetch preserves credentials on same-origin redirects, even outside the registry path. + let currentUrl = url; + for (let redirects = 0; ; redirects++) { + const response = await fetch(currentUrl, { + headers: { ...headers, ...getNpmRegistryHeaders(currentUrl) }, + redirect: "manual", + }); + if (![301, 302, 303, 307, 308].includes(response.status)) { + return response; + } + const location = response.headers.get("location"); + if (location === null) { + return response; + } + + await response.body?.cancel(); + if (redirects === 20) { + throw new NpmRegistryError(`Request to ${url} exceeded 20 redirects.`); + } + currentUrl = new URL(location, currentUrl).href; + } +} + export async function fetchPackageManifest( packageName: string, versionOrRange: string, @@ -101,8 +152,8 @@ export async function fetchPackageManifest( const url = `${getNpmRegistry()}/${encodedPackageName}`; let res: Response; try { - res = await fetch(url, { - headers: { Accept: "application/vnd.npm.install-v1+json" }, + res = await fetchFromNpmRegistry(url, { + Accept: "application/vnd.npm.install-v1+json", }); } catch (error) { const message = error instanceof Error ? `: ${error.message}` : ""; diff --git a/packages/compiler/test/e2e/cli/cli.e2e.ts b/packages/compiler/test/e2e/cli/cli.e2e.ts index afb62e65c9e..16e7b698e65 100644 --- a/packages/compiler/test/e2e/cli/cli.e2e.ts +++ b/packages/compiler/test/e2e/cli/cli.e2e.ts @@ -176,6 +176,56 @@ describe("install", () => { ); } }); + + it("authenticates both package manager metadata and tarball requests", async () => { + const requestUrls: string[] = []; + const server = http.createServer((req, res) => { + requestUrls.push(req.url ?? ""); + if (req.headers.authorization !== "Bearer test-token") { + res.writeHead(401); + res.end(); + } else if (req.url === "/npm") { + const manifest = { + name: "npm", + version: "99.99.98", + dist: { tarball: `http://${req.headers.host}/npm.tgz` }, + bin: { npm: "bin/npm-cli.js" }, + }; + res.writeHead(200, { "Content-Type": "application/json" }); + res.end( + JSON.stringify({ + name: "npm", + "dist-tags": { latest: manifest.version }, + versions: { [manifest.version]: manifest }, + }), + ); + } else { + res.writeHead(200); + res.end(); + } + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const { port } = server.address() as AddressInfo; + + try { + const result = await execCliFail(["install"], { + cwd: getScenarioDir("install"), + env: { + TYPESPEC_NPM_REGISTRY: `http://127.0.0.1:${port}`, + TYPESPEC_NPM_REGISTRY_TOKEN: "test-token", + }, + }); + + expect(requestUrls).toEqual(["/npm", "/npm.tgz"]); + expect(result.stdio).toContain("Failed to extract package from"); + expect(result.stdio).not.toContain("failed with status 401"); + expect(result.stdio).not.toContain("test-token"); + } finally { + await new Promise((resolve, reject) => + server.close((error) => (error ? reject(error) : resolve())), + ); + } + }); }); describe("format", () => { diff --git a/packages/compiler/test/package-manager/npm-package-download.test.ts b/packages/compiler/test/package-manager/npm-package-download.test.ts index 96b7184a230..7c239d1eca2 100644 --- a/packages/compiler/test/package-manager/npm-package-download.test.ts +++ b/packages/compiler/test/package-manager/npm-package-download.test.ts @@ -16,6 +16,140 @@ const manifest: NpmPackageVersion = { afterEach(() => { vi.unstubAllGlobals(); + vi.unstubAllEnvs(); +}); + +it("authenticates tarball downloads from the configured registry", async () => { + vi.stubEnv("TYPESPEC_NPM_REGISTRY", "https://registry.example.com/"); + vi.stubEnv("TYPESPEC_NPM_REGISTRY_TOKEN", "test-token"); + const fetch = vi.fn().mockResolvedValue({ ok: false, status: 503 }); + vi.stubGlobal("fetch", fetch); + + await expect(downloadAndExtractPackage(manifest, "/tmp/test")).rejects.toThrow( + "failed with status 503", + ); + + expect(fetch).toHaveBeenCalledWith(tarballUrl, { + headers: { Authorization: "Bearer test-token" }, + redirect: "manual", + }); +}); + +it.each([ + ["a different host", "https://other.example.com/npm.tgz"], + ["a different port", "https://registry.example.com:8443/feed/npm.tgz"], + ["a different protocol", "http://registry.example.com/feed/npm.tgz"], + ["a different feed", "https://registry.example.com/other-feed/npm.tgz"], + ["a feed with a shared path prefix", "https://registry.example.com/feed-other/npm.tgz"], +])("does not send the registry token to %s", async (_description, url) => { + vi.stubEnv("TYPESPEC_NPM_REGISTRY", "https://registry.example.com/feed/"); + vi.stubEnv("TYPESPEC_NPM_REGISTRY_TOKEN", "test-token"); + const fetch = vi.fn().mockResolvedValue({ ok: false, status: 503 }); + vi.stubGlobal("fetch", fetch); + + await expect( + downloadAndExtractPackage( + { ...manifest, dist: { ...manifest.dist, tarball: url } }, + "/tmp/test", + ), + ).rejects.toThrow("failed with status 503"); + + expect(fetch).toHaveBeenCalledWith(url, { headers: {}, redirect: "manual" }); +}); + +it("authenticates tarball downloads within the configured registry path", async () => { + vi.stubEnv("TYPESPEC_NPM_REGISTRY", "https://registry.example.com/feed"); + vi.stubEnv("TYPESPEC_NPM_REGISTRY_TOKEN", "test-token"); + const fetch = vi.fn().mockResolvedValue({ ok: false, status: 503 }); + vi.stubGlobal("fetch", fetch); + const url = "https://registry.example.com/feed/npm/-/npm.tgz"; + + await expect( + downloadAndExtractPackage( + { ...manifest, dist: { ...manifest.dist, tarball: url } }, + "/tmp/test", + ), + ).rejects.toThrow("failed with status 503"); + + expect(fetch).toHaveBeenCalledWith(url, { + headers: { Authorization: "Bearer test-token" }, + redirect: "manual", + }); +}); + +it.each([ + ["/feed/redirected/npm.tgz", "Bearer test-token"], + ["/other-feed/npm.tgz", undefined], +])("scopes registry credentials when redirected to %s", async (location, expectedAuthorization) => { + const authorizationHeaders: (string | undefined)[] = []; + const server = http.createServer((req, res) => { + authorizationHeaders.push(req.headers.authorization); + if (req.url === "/feed/npm.tgz") { + res.writeHead(302, { Location: location }); + } else { + res.writeHead(503); + } + res.end(); + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const { port } = server.address() as AddressInfo; + const registryUrl = `http://127.0.0.1:${port}/feed`; + const url = `${registryUrl}/npm.tgz`; + vi.stubEnv("TYPESPEC_NPM_REGISTRY", registryUrl); + vi.stubEnv("TYPESPEC_NPM_REGISTRY_TOKEN", "test-token"); + + try { + await expect( + downloadAndExtractPackage( + { ...manifest, dist: { ...manifest.dist, tarball: url } }, + "/tmp/test", + ), + ).rejects.toThrow("failed with status 503"); + + expect(authorizationHeaders).toEqual(["Bearer test-token", expectedAuthorization]); + } finally { + await new Promise((resolve, reject) => + server.close((error) => (error ? reject(error) : resolve())), + ); + } +}); + +it("does not forward the registry token to a different origin after a redirect", async () => { + vi.stubEnv("TYPESPEC_NPM_REGISTRY", "https://registry.example.com/"); + vi.stubEnv("TYPESPEC_NPM_REGISTRY_TOKEN", "test-token"); + const url = "https://storage.example.com/npm.tgz"; + const fetch = vi + .fn() + .mockResolvedValueOnce(new Response(null, { status: 302, headers: { Location: url } })) + .mockResolvedValueOnce(new Response(null, { status: 503 })); + vi.stubGlobal("fetch", fetch); + + await expect(downloadAndExtractPackage(manifest, "/tmp/test")).rejects.toThrow( + "failed with status 503", + ); + + expect(fetch).toHaveBeenNthCalledWith(1, tarballUrl, { + headers: { Authorization: "Bearer test-token" }, + redirect: "manual", + }); + expect(fetch).toHaveBeenNthCalledWith(2, url, { headers: {}, redirect: "manual" }); +}); + +it("reports redirect loops instead of repeatedly downloading", async () => { + vi.stubEnv("TYPESPEC_NPM_REGISTRY", "https://registry.example.com/"); + vi.stubEnv("TYPESPEC_NPM_REGISTRY_TOKEN", "test-token"); + const fetch = vi + .fn() + .mockImplementation(() => + Promise.resolve(new Response(null, { status: 302, headers: { Location: tarballUrl } })), + ); + vi.stubGlobal("fetch", fetch); + + await expect(downloadAndExtractPackage(manifest, "/tmp/test")).rejects.toThrow( + "exceeded 20 redirects", + ); + + expect(fetch).toHaveBeenCalledTimes(21); }); it("reports tarball network failures", async () => { diff --git a/packages/compiler/test/package-manager/npm-registry.test.ts b/packages/compiler/test/package-manager/npm-registry.test.ts index 25e165b8f1b..883d3e6d3bf 100644 --- a/packages/compiler/test/package-manager/npm-registry.test.ts +++ b/packages/compiler/test/package-manager/npm-registry.test.ts @@ -1,6 +1,6 @@ import * as http from "http"; import type { AddressInfo } from "net"; -import { afterEach, beforeEach, expect, it } from "vitest"; +import { afterEach, beforeEach, expect, it, vi } from "vitest"; import { fetchPackageManifest, getNpmRegistry, @@ -12,6 +12,7 @@ let server: http.Server; let registryUrl: string; let lastRequestUrl: string | undefined; let lastAcceptHeader: string | undefined; +let lastAuthorizationHeader: string | undefined; let responseStatus: number; let responseBody: NpmPackument | string; const originalTypeSpecNpmRegistry = process.env["TYPESPEC_NPM_REGISTRY"]; @@ -20,11 +21,14 @@ const originalNpmConfigRegistry = process.env["NPM_CONFIG_REGISTRY"]; beforeEach(async () => { lastRequestUrl = undefined; lastAcceptHeader = undefined; + lastAuthorizationHeader = undefined; + vi.stubEnv("TYPESPEC_NPM_REGISTRY_TOKEN", undefined); responseStatus = 200; responseBody = createPackument(); server = http.createServer((req, res) => { lastRequestUrl = req.url ?? ""; lastAcceptHeader = req.headers.accept; + lastAuthorizationHeader = req.headers.authorization; res.writeHead(responseStatus, { "Content-Type": "application/json" }); res.end(typeof responseBody === "string" ? responseBody : JSON.stringify(responseBody)); }); @@ -34,6 +38,7 @@ beforeEach(async () => { }); afterEach(async () => { + vi.unstubAllEnvs(); restoreEnvironmentVariable("TYPESPEC_NPM_REGISTRY", originalTypeSpecNpmRegistry); restoreEnvironmentVariable("NPM_CONFIG_REGISTRY", originalNpmConfigRegistry); await new Promise((resolve) => server.close(() => resolve())); @@ -54,6 +59,23 @@ it("strips trailing slash from TYPESPEC_NPM_REGISTRY", async () => { expect(lastRequestUrl).toBe("/test-pkg"); }); +it("authenticates registry metadata requests when a token is configured", async () => { + process.env["TYPESPEC_NPM_REGISTRY"] = registryUrl; + vi.stubEnv("TYPESPEC_NPM_REGISTRY_TOKEN", "test-token"); + + await fetchPackageManifest("test-pkg", "latest"); + + expect(lastAuthorizationHeader).toBe("Bearer test-token"); +}); + +it("does not authenticate registry requests when no token is configured", async () => { + process.env["TYPESPEC_NPM_REGISTRY"] = registryUrl; + + await fetchPackageManifest("test-pkg", "latest"); + + expect(lastAuthorizationHeader).toBeUndefined(); +}); + it("resolves a package version from a semver range", async () => { process.env["TYPESPEC_NPM_REGISTRY"] = registryUrl; diff --git a/turbo.json b/turbo.json index 88a6b8413ae..37d0db4b11b 100644 --- a/turbo.json +++ b/turbo.json @@ -3,6 +3,7 @@ "globalPassThroughEnv": [ "TYPESPEC_WEBSITE_BASE_PATH", "TYPESPEC_NPM_REGISTRY", + "TYPESPEC_NPM_REGISTRY_TOKEN", "TYPESPEC_SKIP_WEBSITE_BUILD", "TYPESPEC_SKIP_VS_BUILD", "TYPESPEC_VS_CI_BUILD" diff --git a/website/src/content/docs/docs/handbook/cli.md b/website/src/content/docs/docs/handbook/cli.md index a8a4a858e35..6af4397d06b 100644 --- a/website/src/content/docs/docs/handbook/cli.md +++ b/website/src/content/docs/docs/handbook/cli.md @@ -47,4 +47,18 @@ If this variable is not set, TypeSpec defaults to `https://registry.npmjs.org`. This variable does not configure the package manager invoked by `tsp init` or `tsp install`. Configure that package manager separately using its own registry and authentication settings. TypeSpec does not read those authentication settings: its package metadata request and the tarball -URL returned by the registry must be accessible to the TypeSpec process. +URL returned by the registry must be accessible to the TypeSpec process, using +`TYPESPEC_NPM_REGISTRY_TOKEN` if authentication is required. + +### `TYPESPEC_NPM_REGISTRY_TOKEN` + +Set a bearer token to authenticate TypeSpec's package metadata requests and package-manager +downloads against the registry selected by `TYPESPEC_NPM_REGISTRY`. If no registry is configured, +the token applies to the default npm registry. + +TypeSpec only sends the token to URLs with the same origin and within the configured registry's +path. It does not send the token to tarballs hosted on other origins or outside that path. + +Supply this variable through your environment or CI secret settings; do not commit tokens to +source control. This variable does not configure authentication for the invoked package manager, +which still uses its own authentication settings.