Support registry mirrors in tsp install

.chronus/changes/fix-tsp-install-registry-bootstrap-2026-7-27-13-23-35.md

@@ -0,0 +1,7 @@
+---
+changeKind: fix
+packages:
+  - "@typespec/compiler"
+---
+
+Allow `tsp install` to download package managers from npm-compatible registry mirrors by resolving versions from package metadata instead of version-specific manifest endpoints.

eng/tsp-core/pipelines/jobs/e2e.yml

@@ -9,6 +9,7 @@ jobs:
     variables:
       TYPESPEC_VS_CI_BUILD: false # Enable official Visual Studio extension build
       TYPESPEC_SKIP_WEBSITE_BUILD: true # Disable docusaurus build
+      TYPESPEC_NPM_REGISTRY: https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-js/npm/registry/
       DISPLAY: ":99" # Set DISPLAY for Linux GUI applications
 
     pool:

eng/tsp-core/pipelines/publish.yml

@@ -31,7 +31,7 @@ extends:
           - template: /eng/tsp-core/pipelines/jobs/build-packages.yml@self
           - template: /eng/tsp-core/pipelines/jobs/build-vs.yml@self
           - template: /eng/tsp-core/pipelines/jobs/cli/build-tsp-cli-all.yml@self
-            # - template: /eng/tsp-core/pipelines/jobs/e2e.yml@self
+          - template: /eng/tsp-core/pipelines/jobs/e2e.yml@self
             parameters:
               azLogin: true
 

packages/compiler/src/install/install.ts

@@ -6,8 +6,15 @@ import { DiagnosticError } from "../core/diagnostic-error.js";
 import { createDiagnosticCollector } from "../core/diagnostics.js";
 import { getDirectoryPath, joinPaths } from "../core/path-utils.js";
 import { NoTarget, type Diagnostic, type Tracer } from "../core/types.js";
-import { downloadAndExtractPackage } from "../package-manger/npm-package-download.js";
-import { fetchPackageManifest, type NpmManifest } from "../package-manger/npm-registry.js";
+import {
+  downloadAndExtractPackage,
+  type ExtractedTarballResult,
+} from "../package-manger/npm-package-download.js";
+import {
+  fetchPackageManifest,
+  NpmRegistryError,
+  type NpmPackageVersion,
+} from "../package-manger/npm-registry.js";
 import { mkTempDir } from "../utils/fs-utils.js";
 import type { SupportedPackageManager } from "./config.js";
 import { getPackageManagerConfig, type PackageManagerConfig } from "./config.js";
@@ -90,7 +97,7 @@ async function installPackageManager(
   packageManager: SupportedPackageManager,
   spec: Descriptor,
   installDir: string,
-  manifest: NpmManifest,
+  manifest: NpmPackageVersion,
 ) {
   await rm(installDir, { recursive: true, force: true });
   const tempDir = await mkTempDir(host, pmDir, `tsp-pm-${packageManager}-${manifest.version}`);
@@ -99,7 +106,17 @@ async function installPackageManager(
     "downloading-extracting",
     `Downloading and extracting ${packageManager} at version ${manifest.version} in ${tempDir}`,
   );
-  const extractResult = await downloadAndExtractPackage(manifest, tempDir, spec.hash?.algorithm);
+  let extractResult: ExtractedTarballResult;
+  try {
+    extractResult = await downloadAndExtractPackage(manifest, tempDir, spec.hash?.algorithm);
+  } catch (error) {
+    if (error instanceof NpmRegistryError) {
+      throw new InstallDependenciesError(
+        `Failed to download package manager ${packageManager}: ${error.message}`,
+      );
+    }
+    throw error;
+  }
   if (spec.hash) {
     if (spec.hash.value !== extractResult.hash.value) {
       throw new InstallDependenciesError(
@@ -165,11 +182,27 @@ export async function installTypeSpecDependencies(
     );
     const packageManager = spec.name;
     const packageManagerConfig = getPackageManagerConfig(packageManager);
-    const manifest = await fetchPackageManifest(packageManager, spec.range);
+    let manifest: NpmPackageVersion;
+    try {
+      manifest = await fetchPackageManifest(packageManager, spec.range);
+    } catch (error) {
+      if (error instanceof NpmRegistryError) {
+        throw new InstallDependenciesError(
+          `Failed to resolve package manager ${packageManager}: ${error.message}`,
+        );
+      }
+      throw error;
+    }
     tracer.trace(
       "fetched-manifest",
       `Resolved manifest for ${packageManager} at version ${manifest.version}`,
     );
+    const bin = manifest.bin?.[packageManager];
+    if (bin === undefined) {
+      throw new InstallDependenciesError(
+        `Package manager ${packageManager} does not define a "${packageManager}" executable.`,
+      );
+    }
 
     const installDir = joinPaths(pmDir, packageManager, manifest.version);
     if ((await isDirectory(host, installDir)) && !savePackageManager) {
@@ -192,7 +225,6 @@ export async function installTypeSpecDependencies(
       }
     }
 
-    const bin = manifest.bin![packageManager];
     const binPath = joinPaths(installDir, bin);
     tracer.trace("running-binary", `Running binary ${binPath}`);
 

packages/compiler/src/package-manger/npm-package-download.ts

@@ -4,7 +4,7 @@ import { createHash } from "crypto";
 import { Readable } from "stream";
 import { extract as tarX } from "tar/extract";
 import type { Hash } from "../install/spec.js";
-import { fetchPackageManifest, type NpmManifest } from "./npm-registry.js";
+import { fetchPackageManifest, NpmRegistryError, type NpmPackageVersion } from "./npm-registry.js";
 
 export async function downloadPackageVersion(
   packageName: string,
@@ -16,7 +16,7 @@ export async function downloadPackageVersion(
 }
 
 export async function downloadAndExtractPackage(
-  manifest: NpmManifest,
+  manifest: NpmPackageVersion,
   dest: string,
   hashAlgorithm: string = "sha512",
 ): Promise<ExtractedTarballResult> {
@@ -32,7 +32,20 @@ async function downloadAndExtractTarball(
   dest: string,
   hashAlgorithm: string = "sha512",
 ): Promise<ExtractedTarballResult> {
-  const res = await fetch(url);
+  let res: Response;
+  try {
+    res = await fetch(url);
+  } catch (error) {
+    const message = error instanceof Error ? `: ${error.message}` : "";
+    throw new NpmRegistryError(`Request to ${url} failed${message}`);
+  }
+  if (!res.ok) {
+    throw new NpmRegistryError(`Request to ${url} failed with status ${res.status}.`);
+  }
+  if (res.body === null) {
+    throw new NpmRegistryError(`Request to ${url} returned an empty response.`);
+  }
+
   const tarballStream = Readable.fromWeb(res.body as any);
   const hash = tarballStream.pipe(createHash(hashAlgorithm));
   const extractor = tarX({
@@ -53,7 +66,12 @@ async function downloadAndExtractTarball(
   });
 
   tarballStream.pipe(extractor);
-  await p;
+  try {
+    await p;
+  } catch (error) {
+    const message = error instanceof Error ? `: ${error.message}` : "";
+    throw new NpmRegistryError(`Failed to extract package from ${url}${message}`);
+  }
 
   return { dest, hash: { algorithm: hashAlgorithm, value: hash.digest("hex") } };
 }

packages/compiler/src/package-manger/npm-registry.ts

@@ -1,35 +1,6 @@
 // Browser-safe helpers to access the npm registry api
 // https://github.com/npm/registry/blob/main/docs/REGISTRY-API.md#package-endpoints
-
-/** Manifest of a single package version. */
-export interface NpmManifest {
-  readonly name: string;
-  readonly version: string;
-  readonly dependencies: Record<string, string>;
-  readonly optionalDependencies: Record<string, string>;
-  readonly devDependencies: Record<string, string>;
-  readonly peerDependencies: Record<string, string>;
-  readonly bundleDependencies: false | string[];
-  readonly dist: NpmPackageDist;
-  readonly bin: Record<string, string> | null;
-  readonly _shrinkwrap: Record<string, unknown> | null;
-
-  readonly engines?: Record<string, string> | undefined;
-  readonly cpu?: string[] | undefined;
-  readonly os?: string[] | undefined;
-  readonly _id?: string | undefined;
-
-  readonly [key: string]: unknown;
-}
-
-/** Document listing a package information and all its versions. */
-export interface NpmPackument {
-  readonly name: string;
-  readonly "dist-tags": { latest: string } & Record<string, string>;
-  readonly versions: Record<string, NpmPackageVersion>;
-
-  readonly [key: string]: unknown;
-}
+import semverMaxSatisfying from "semver/ranges/max-satisfying.js";
 
 export interface NpmPackageVersion {
   readonly name: string;
@@ -38,9 +9,12 @@ export interface NpmPackageVersion {
   readonly optionalDependencies?: Record<string, string> | undefined;
   readonly devDependencies?: Record<string, string> | undefined;
   readonly peerDependencies?: Record<string, string> | undefined;
-  readonly directories: {};
+  readonly bundleDependencies?: false | string[] | undefined;
   readonly dist: NpmPackageDist;
-  readonly _hasShrinkwrap: boolean;
+  readonly bin?: Record<string, string> | null | undefined;
+  readonly _shrinkwrap?: Record<string, unknown> | null | undefined;
+  readonly directories?: {} | undefined;
+  readonly _hasShrinkwrap?: boolean | undefined;
 
   // Extra metadata which may be added by the registry:
   readonly description?: string | undefined;
@@ -61,13 +35,35 @@ export interface NpmPackageVersion {
   readonly license?: string | undefined;
   readonly homepage?: string | undefined;
   readonly bugs?: { url: string } | undefined;
+  readonly cpu?: string[] | undefined;
+  readonly os?: string[] | undefined;
   readonly _id?: string | undefined;
   readonly _nodeVersion?: string | undefined;
   readonly _npmVersion?: string | undefined;
   readonly _npmUser?: NpmHuman | undefined;
   readonly [key: string]: unknown;
 }
 
+/** Manifest of a single package version. */
+export interface NpmManifest extends NpmPackageVersion {
+  readonly dependencies: Record<string, string>;
+  readonly optionalDependencies: Record<string, string>;
+  readonly devDependencies: Record<string, string>;
+  readonly peerDependencies: Record<string, string>;
+  readonly bundleDependencies: false | string[];
+  readonly bin: Record<string, string> | null;
+  readonly _shrinkwrap: Record<string, unknown> | null;
+}
+
+/** Document listing a package information and all its versions. */
+export interface NpmPackument {
+  readonly name: string;
+  readonly "dist-tags": { latest: string } & Record<string, string>;
+  readonly versions: Record<string, NpmPackageVersion>;
+
+  readonly [key: string]: unknown;
+}
+
 export interface NpmPackageDist {
   readonly shasum: string;
   readonly tarball: string;
@@ -82,6 +78,8 @@ export interface NpmHuman {
   readonly url?: string | undefined;
 }
 
+export class NpmRegistryError extends Error {}
+
 const defaultRegistry = `https://registry.npmjs.org`;
 
 /**
@@ -95,13 +93,74 @@ export function getNpmRegistry(): string {
 
 export async function fetchPackageManifest(
   packageName: string,
-  version: string,
-): Promise<NpmManifest> {
-  const url = `${getNpmRegistry()}/${packageName}/${version}`;
-  const res = await fetch(url);
-  return await res.json();
+  versionOrRange: string,
+): Promise<NpmPackageVersion> {
+  const encodedPackageName = packageName.startsWith("@")
+    ? packageName.replace("/", "%2F")
+    : packageName;
+  const url = `${getNpmRegistry()}/${encodedPackageName}`;
+  let res: Response;
+  try {
+    res = await fetch(url, {
+      headers: { Accept: "application/vnd.npm.install-v1+json" },
+    });
+  } catch (error) {
+    const message = error instanceof Error ? `: ${error.message}` : "";
+    throw new NpmRegistryError(`Request to ${url} failed${message}`);
+  }
+  if (!res.ok) {
+    throw new NpmRegistryError(`Request to ${url} failed with status ${res.status}.`);
+  }
+
+  let response: unknown;
+  try {
+    response = await res.json();
+  } catch {
+    throw new NpmRegistryError(`Request to ${url} returned invalid JSON.`);
+  }
+  const packument = parsePackument(url, response);
+  const version =
+    packument["dist-tags"][versionOrRange] ??
+    semverMaxSatisfying(Object.keys(packument.versions), versionOrRange);
+  const manifest = version === null ? undefined : packument.versions[version];
+  if (manifest === undefined || !isNpmPackageVersion(manifest)) {
+    throw new NpmRegistryError(
+      `Package "${packageName}" does not have a version or tag matching "${versionOrRange}".`,
+    );
+  }
+  return manifest;
 }
 
-export function fetchLatestPackageManifest(packageName: string): Promise<NpmManifest> {
+export function fetchLatestPackageManifest(packageName: string): Promise<NpmPackageVersion> {
   return fetchPackageManifest(packageName, "latest");
 }
+
+function parsePackument(url: string, value: unknown): NpmPackument {
+  if (
+    !isRecord(value) ||
+    typeof value.name !== "string" ||
+    !isStringRecord(value["dist-tags"]) ||
+    !isRecord(value.versions)
+  ) {
+    throw new NpmRegistryError(`Request to ${url} returned an invalid package document.`);
+  }
+  return value as NpmPackument;
+}
+
+function isNpmPackageVersion(value: unknown): value is NpmPackageVersion {
+  return (
+    isRecord(value) &&
+    typeof value.name === "string" &&
+    typeof value.version === "string" &&
+    isRecord(value.dist) &&
+    typeof value.dist.tarball === "string"
+  );
+}
+
+function isStringRecord(value: unknown): value is Record<string, string> {
+  return isRecord(value) && Object.values(value).every((item) => typeof item === "string");
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}

packages/compiler/test/cli/init.test.ts

@@ -10,21 +10,34 @@ import { createTestFileSystem } from "../../src/testing/fs.js";
 import type { TestFileSystem } from "../../src/testing/types.js";
 import { parseYaml as coreParseYaml } from "../../src/yaml/parser.js";
 
-const fetchMock = vi.fn().mockResolvedValue({
-  json: () => Promise.resolve({ name: "mock-pkg", version: "1.0.0" }),
-});
+const fetchMock = vi.fn().mockResolvedValue(createFetchResponse());
 
 beforeEach(() => {
   vi.stubGlobal("fetch", fetchMock);
 });
 
 afterEach(() => {
   vi.unstubAllGlobals();
-  fetchMock.mockResolvedValue({
-    json: () => Promise.resolve({ name: "mock-pkg", version: "1.0.0" }),
-  });
+  fetchMock.mockResolvedValue(createFetchResponse());
 });
 
+function createFetchResponse() {
+  const manifest = {
+    name: "mock-pkg",
+    version: "1.0.0",
+    dist: { shasum: "abc", tarball: "https://example.com/mock-pkg.tgz" },
+  };
+  return {
+    ok: true,
+    json: () =>
+      Promise.resolve({
+        name: manifest.name,
+        "dist-tags": { latest: manifest.version },
+        versions: { [manifest.version]: manifest },
+      }),
+  };
+}
+
 const TEST_SCAFFOLDING = {
   foo: {
     title: "Generic REST API",