Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions .chronus/changes/fix-authenticated-registry-bootstrap.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
---
changeKind: fix
packages:
- "@typespec/compiler"
---

Fix package-manager downloads from authenticated registry mirrors by supporting the `TYPESPEC_NPM_REGISTRY_TOKEN` environment variable for TypeSpec's package metadata and tarball requests.
4 changes: 4 additions & 0 deletions eng/tsp-core/pipelines/jobs/e2e.yml
Original file line number Diff line number Diff line change
Expand Up @@ -33,9 +33,13 @@ jobs:
scriptType: bash
scriptLocation: inlineScript
inlineScript: pnpm run test:e2e
env:
TYPESPEC_NPM_REGISTRY_TOKEN: $(System.AccessToken)
- ${{ else }}:
- script: pnpm run test:e2e
displayName: E2E Tests
env:
TYPESPEC_NPM_REGISTRY_TOKEN: $(System.AccessToken)

- task: 1ES.PublishPipelineArtifact@1
inputs:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,12 @@ import { createHash } from "crypto";
import { Readable } from "stream";
import { extract as tarX } from "tar/extract";
import type { Hash } from "../install/spec.js";
import { fetchPackageManifest, NpmRegistryError, type NpmPackageVersion } from "./npm-registry.js";
import {
fetchFromNpmRegistry,
fetchPackageManifest,
NpmRegistryError,
type NpmPackageVersion,
} from "./npm-registry.js";

export async function downloadPackageVersion(
packageName: string,
Expand Down Expand Up @@ -34,7 +39,7 @@ async function downloadAndExtractTarball(
): Promise<ExtractedTarballResult> {
let res: Response;
try {
res = await fetch(url);
res = await fetchFromNpmRegistry(url);
} catch (error) {
const message = error instanceof Error ? `: ${error.message}` : "";
throw new NpmRegistryError(`Request to ${url} failed${message}`);
Expand Down
55 changes: 53 additions & 2 deletions packages/compiler/src/package-manger/npm-registry.ts
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,57 @@ export function getNpmRegistry(): string {
return (process.env["TYPESPEC_NPM_REGISTRY"] ?? defaultRegistry).replace(/\/$/, "");
}

/** Only send registry credentials to URLs within the configured registry. */
function getNpmRegistryHeaders(url: string): Record<string, string> {
const token = process.env["TYPESPEC_NPM_REGISTRY_TOKEN"];
if (!token) {
return {};
}

const registry = new URL(getNpmRegistry());
const target = new URL(url);
const registryPath = registry.pathname.replace(/\/$/, "");
if (
target.origin !== registry.origin ||
(target.pathname !== registryPath && !target.pathname.startsWith(`${registryPath}/`))
) {
return {};
}

return { Authorization: `Bearer ${token}` };
}

export async function fetchFromNpmRegistry(
url: string,
headers: Record<string, string> = {},
): Promise<Response> {
if (!process.env["TYPESPEC_NPM_REGISTRY_TOKEN"]) {
return fetch(url, { headers });
}

// Fetch preserves credentials on same-origin redirects, even outside the registry path.
let currentUrl = url;
for (let redirects = 0; ; redirects++) {
const response = await fetch(currentUrl, {
headers: { ...headers, ...getNpmRegistryHeaders(currentUrl) },
redirect: "manual",
});
if (![301, 302, 303, 307, 308].includes(response.status)) {
return response;
}
const location = response.headers.get("location");
if (location === null) {
return response;
}

await response.body?.cancel();
if (redirects === 20) {
throw new NpmRegistryError(`Request to ${url} exceeded 20 redirects.`);
}
currentUrl = new URL(location, currentUrl).href;
}
}

export async function fetchPackageManifest(
packageName: string,
versionOrRange: string,
Expand All @@ -101,8 +152,8 @@ export async function fetchPackageManifest(
const url = `${getNpmRegistry()}/${encodedPackageName}`;
let res: Response;
try {
res = await fetch(url, {
headers: { Accept: "application/vnd.npm.install-v1+json" },
res = await fetchFromNpmRegistry(url, {
Accept: "application/vnd.npm.install-v1+json",
});
} catch (error) {
const message = error instanceof Error ? `: ${error.message}` : "";
Expand Down
50 changes: 50 additions & 0 deletions packages/compiler/test/e2e/cli/cli.e2e.ts
Original file line number Diff line number Diff line change
Expand Up @@ -176,6 +176,56 @@ describe("install", () => {
);
}
});

it("authenticates both package manager metadata and tarball requests", async () => {
const requestUrls: string[] = [];
const server = http.createServer((req, res) => {
requestUrls.push(req.url ?? "");
if (req.headers.authorization !== "Bearer test-token") {
res.writeHead(401);
res.end();
} else if (req.url === "/npm") {
const manifest = {
name: "npm",
version: "99.99.98",
dist: { tarball: `http://${req.headers.host}/npm.tgz` },
bin: { npm: "bin/npm-cli.js" },
};
res.writeHead(200, { "Content-Type": "application/json" });
res.end(
JSON.stringify({
name: "npm",
"dist-tags": { latest: manifest.version },
versions: { [manifest.version]: manifest },
}),
);
} else {
res.writeHead(200);
res.end();
}
});
await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
const { port } = server.address() as AddressInfo;

try {
const result = await execCliFail(["install"], {
cwd: getScenarioDir("install"),
env: {
TYPESPEC_NPM_REGISTRY: `http://127.0.0.1:${port}`,
TYPESPEC_NPM_REGISTRY_TOKEN: "test-token",
},
});

expect(requestUrls).toEqual(["/npm", "/npm.tgz"]);
expect(result.stdio).toContain("Failed to extract package from");
expect(result.stdio).not.toContain("failed with status 401");
expect(result.stdio).not.toContain("test-token");
} finally {
await new Promise<void>((resolve, reject) =>
server.close((error) => (error ? reject(error) : resolve())),
);
}
});
});

describe("format", () => {
Expand Down
134 changes: 134 additions & 0 deletions packages/compiler/test/package-manager/npm-package-download.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,140 @@ const manifest: NpmPackageVersion = {

afterEach(() => {
vi.unstubAllGlobals();
vi.unstubAllEnvs();
});

it("authenticates tarball downloads from the configured registry", async () => {
vi.stubEnv("TYPESPEC_NPM_REGISTRY", "https://registry.example.com/");
vi.stubEnv("TYPESPEC_NPM_REGISTRY_TOKEN", "test-token");
const fetch = vi.fn().mockResolvedValue({ ok: false, status: 503 });
vi.stubGlobal("fetch", fetch);

await expect(downloadAndExtractPackage(manifest, "/tmp/test")).rejects.toThrow(
"failed with status 503",
);

expect(fetch).toHaveBeenCalledWith(tarballUrl, {
headers: { Authorization: "Bearer test-token" },
redirect: "manual",
});
});

it.each([
["a different host", "https://other.example.com/npm.tgz"],
["a different port", "https://registry.example.com:8443/feed/npm.tgz"],
["a different protocol", "http://registry.example.com/feed/npm.tgz"],
["a different feed", "https://registry.example.com/other-feed/npm.tgz"],
["a feed with a shared path prefix", "https://registry.example.com/feed-other/npm.tgz"],
])("does not send the registry token to %s", async (_description, url) => {
vi.stubEnv("TYPESPEC_NPM_REGISTRY", "https://registry.example.com/feed/");
vi.stubEnv("TYPESPEC_NPM_REGISTRY_TOKEN", "test-token");
const fetch = vi.fn().mockResolvedValue({ ok: false, status: 503 });
vi.stubGlobal("fetch", fetch);

await expect(
downloadAndExtractPackage(
{ ...manifest, dist: { ...manifest.dist, tarball: url } },
"/tmp/test",
),
).rejects.toThrow("failed with status 503");

expect(fetch).toHaveBeenCalledWith(url, { headers: {}, redirect: "manual" });
});

it("authenticates tarball downloads within the configured registry path", async () => {
vi.stubEnv("TYPESPEC_NPM_REGISTRY", "https://registry.example.com/feed");
vi.stubEnv("TYPESPEC_NPM_REGISTRY_TOKEN", "test-token");
const fetch = vi.fn().mockResolvedValue({ ok: false, status: 503 });
vi.stubGlobal("fetch", fetch);
const url = "https://registry.example.com/feed/npm/-/npm.tgz";

await expect(
downloadAndExtractPackage(
{ ...manifest, dist: { ...manifest.dist, tarball: url } },
"/tmp/test",
),
).rejects.toThrow("failed with status 503");

expect(fetch).toHaveBeenCalledWith(url, {
headers: { Authorization: "Bearer test-token" },
redirect: "manual",
});
});

it.each([
["/feed/redirected/npm.tgz", "Bearer test-token"],
["/other-feed/npm.tgz", undefined],
])("scopes registry credentials when redirected to %s", async (location, expectedAuthorization) => {
const authorizationHeaders: (string | undefined)[] = [];
const server = http.createServer((req, res) => {
authorizationHeaders.push(req.headers.authorization);
if (req.url === "/feed/npm.tgz") {
res.writeHead(302, { Location: location });
} else {
res.writeHead(503);
}
res.end();
});
await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
const { port } = server.address() as AddressInfo;
const registryUrl = `http://127.0.0.1:${port}/feed`;
const url = `${registryUrl}/npm.tgz`;
vi.stubEnv("TYPESPEC_NPM_REGISTRY", registryUrl);
vi.stubEnv("TYPESPEC_NPM_REGISTRY_TOKEN", "test-token");

try {
await expect(
downloadAndExtractPackage(
{ ...manifest, dist: { ...manifest.dist, tarball: url } },
"/tmp/test",
),
).rejects.toThrow("failed with status 503");

expect(authorizationHeaders).toEqual(["Bearer test-token", expectedAuthorization]);
} finally {
await new Promise<void>((resolve, reject) =>
server.close((error) => (error ? reject(error) : resolve())),
);
}
});

it("does not forward the registry token to a different origin after a redirect", async () => {
vi.stubEnv("TYPESPEC_NPM_REGISTRY", "https://registry.example.com/");
vi.stubEnv("TYPESPEC_NPM_REGISTRY_TOKEN", "test-token");
const url = "https://storage.example.com/npm.tgz";
const fetch = vi
.fn()
.mockResolvedValueOnce(new Response(null, { status: 302, headers: { Location: url } }))
.mockResolvedValueOnce(new Response(null, { status: 503 }));
vi.stubGlobal("fetch", fetch);

await expect(downloadAndExtractPackage(manifest, "/tmp/test")).rejects.toThrow(
"failed with status 503",
);

expect(fetch).toHaveBeenNthCalledWith(1, tarballUrl, {
headers: { Authorization: "Bearer test-token" },
redirect: "manual",
});
expect(fetch).toHaveBeenNthCalledWith(2, url, { headers: {}, redirect: "manual" });
});

it("reports redirect loops instead of repeatedly downloading", async () => {
vi.stubEnv("TYPESPEC_NPM_REGISTRY", "https://registry.example.com/");
vi.stubEnv("TYPESPEC_NPM_REGISTRY_TOKEN", "test-token");
const fetch = vi
.fn()
.mockImplementation(() =>
Promise.resolve(new Response(null, { status: 302, headers: { Location: tarballUrl } })),
);
vi.stubGlobal("fetch", fetch);

await expect(downloadAndExtractPackage(manifest, "/tmp/test")).rejects.toThrow(
"exceeded 20 redirects",
);

expect(fetch).toHaveBeenCalledTimes(21);
});

it("reports tarball network failures", async () => {
Expand Down
24 changes: 23 additions & 1 deletion packages/compiler/test/package-manager/npm-registry.test.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import * as http from "http";
import type { AddressInfo } from "net";
import { afterEach, beforeEach, expect, it } from "vitest";
import { afterEach, beforeEach, expect, it, vi } from "vitest";
import {
fetchPackageManifest,
getNpmRegistry,
Expand All @@ -12,6 +12,7 @@ let server: http.Server;
let registryUrl: string;
let lastRequestUrl: string | undefined;
let lastAcceptHeader: string | undefined;
let lastAuthorizationHeader: string | undefined;
let responseStatus: number;
let responseBody: NpmPackument | string;
const originalTypeSpecNpmRegistry = process.env["TYPESPEC_NPM_REGISTRY"];
Expand All @@ -20,11 +21,14 @@ const originalNpmConfigRegistry = process.env["NPM_CONFIG_REGISTRY"];
beforeEach(async () => {
lastRequestUrl = undefined;
lastAcceptHeader = undefined;
lastAuthorizationHeader = undefined;
vi.stubEnv("TYPESPEC_NPM_REGISTRY_TOKEN", undefined);
responseStatus = 200;
responseBody = createPackument();
server = http.createServer((req, res) => {
lastRequestUrl = req.url ?? "";
lastAcceptHeader = req.headers.accept;
lastAuthorizationHeader = req.headers.authorization;
res.writeHead(responseStatus, { "Content-Type": "application/json" });
res.end(typeof responseBody === "string" ? responseBody : JSON.stringify(responseBody));
});
Expand All @@ -34,6 +38,7 @@ beforeEach(async () => {
});

afterEach(async () => {
vi.unstubAllEnvs();
restoreEnvironmentVariable("TYPESPEC_NPM_REGISTRY", originalTypeSpecNpmRegistry);
restoreEnvironmentVariable("NPM_CONFIG_REGISTRY", originalNpmConfigRegistry);
await new Promise<void>((resolve) => server.close(() => resolve()));
Expand All @@ -54,6 +59,23 @@ it("strips trailing slash from TYPESPEC_NPM_REGISTRY", async () => {
expect(lastRequestUrl).toBe("/test-pkg");
});

it("authenticates registry metadata requests when a token is configured", async () => {
process.env["TYPESPEC_NPM_REGISTRY"] = registryUrl;
vi.stubEnv("TYPESPEC_NPM_REGISTRY_TOKEN", "test-token");

await fetchPackageManifest("test-pkg", "latest");

expect(lastAuthorizationHeader).toBe("Bearer test-token");
});

it("does not authenticate registry requests when no token is configured", async () => {
process.env["TYPESPEC_NPM_REGISTRY"] = registryUrl;

await fetchPackageManifest("test-pkg", "latest");

expect(lastAuthorizationHeader).toBeUndefined();
});

it("resolves a package version from a semver range", async () => {
process.env["TYPESPEC_NPM_REGISTRY"] = registryUrl;

Expand Down
1 change: 1 addition & 0 deletions turbo.json
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
"globalPassThroughEnv": [
"TYPESPEC_WEBSITE_BASE_PATH",
"TYPESPEC_NPM_REGISTRY",
"TYPESPEC_NPM_REGISTRY_TOKEN",
"TYPESPEC_SKIP_WEBSITE_BUILD",
"TYPESPEC_SKIP_VS_BUILD",
"TYPESPEC_VS_CI_BUILD"
Expand Down
Loading
Loading