From b293f5325ed9327caa5ec9c782ef58f3b7614d14 Mon Sep 17 00:00:00 2001 From: lenmei233 <1663996167@qq.com> Date: Fri, 25 Sep 2026 01:58:04 +0800 Subject: [PATCH] fix(catalog): resolve gateway model metadata from other publishers models.dev indexes a gateway's copy of a model under the vendor that owns the weights, not under the gateway. An endpoint serving `Vendor/Model` ids such as `deepseek-ai/DeepSeek-V4-Flash-0731` therefore has no record of its own while several other publishers state the identical id. `findModel` scoped the lookup to the row's own catalog provider once that provider was known, so every such model missed and fell back to the generic 128k text-only shape, leaving its context window and tool support invisible. On ModelScope none of the 31 models its endpoint serves resolved; 14 do now. Borrowing is deliberately narrow. It runs only when the row resolved to a catalog provider that published nothing for the id, and only for a case-insensitive identical id: a record reached through an alias describes a different id and keeps its own limits. A provider sharing the row's endpoint is skipped, because it is an alias for the row and its silence answers for this deployment. Tool support must be unanimous across publishers, since a wrong `true` puts tool declarations on the wire that the endpoint may reject, while reasoning, vision and attachment are intersections and limits are medians, so a borrow can only under-claim. An unknown endpoint keeps its existing behaviour. The issue suggested widening suffix matching for ids such as `-0731`; that is not the defect. Exact matching already accepts those ids, and a global weak match would conflate distinct models (`Qwen/Qwen3.5-27B` and `Qwen/Qwen3-235B-A22B-...-2507` normalize alike). This changes metadata only: configured ids, provider identity and binding precedence are untouched. fixes #938 --- .../electron/main/models-dev-catalog.ts | 152 ++++++++++++++- apps/desktop/test/models-dev-catalog.test.mjs | 181 ++++++++++++++++++ .../13-model-catalog-and-selection.md | 29 +++ 3 files changed, 354 insertions(+), 8 deletions(-) diff --git a/apps/desktop/electron/main/models-dev-catalog.ts b/apps/desktop/electron/main/models-dev-catalog.ts index c457f5656a..df2875ebd9 100644 --- a/apps/desktop/electron/main/models-dev-catalog.ts +++ b/apps/desktop/electron/main/models-dev-catalog.ts @@ -859,6 +859,85 @@ const EMPTY_CANDIDATES: readonly IndexedModel[] = []; +/** + * Middle value of the numbers the publishers state. Even counts take the lower + * of the two middles, so a borrow never rounds a window up on its own. + */ +function medianOf(values: readonly (number | undefined)[]): number | undefined { + const present = values.filter((value): value is number => value !== undefined); + if (present.length === 0) return undefined; + const sorted = [...present].sort((left, right) => left - right); + const middle = sorted.length >> 1; + return sorted.length % 2 === 1 + ? sorted[middle] + : sorted[middle - 1]; +} + +/** + * Record to borrow for an id the row's own catalog provider does not publish. + * + * models.dev indexes a gateway's copy of a model under the vendor that owns + * the weights, so an endpoint serving `Vendor/Model` ids can have no record of + * its own while several other publishers state the identical id. Borrowing + * that record is what keeps such a model's context window, tool support and + * vision visible instead of dropping the row to the generic 128k text-only + * shape. + * + * Two rules keep the result honest about what the endpoint will accept: + * + * - Tool support is the gate. Every publisher of the id must agree, because a + * wrong `true` puts tool declarations on the wire that the endpoint may + * reject and break the turn. Publishers that split on this id are describing + * different deployments, so nothing is borrowed. + * - Every other capability is the *intersection*, so the borrow may only + * under-claim. Reasoning, image input and image/PDF attachment are reported + * only when every publisher states them, and a user who knows the endpoint + * does more can still turn them on in Advanced. Limits are the medians the + * publishers state, so neither one host's cap nor one host's round-up + * decides them. + */ +function borrowedModel(entries: readonly ModelsDevModel[]): ModelsDevModel | undefined { + if (entries.length === 0) return undefined; + const toolCall = entries[0].toolCall; + if (entries.some((model) => model.toolCall !== toolCall)) return undefined; + const every = (pick: (model: ModelsDevModel) => boolean | undefined): boolean | undefined => + entries.every((model) => pick(model) === true) ? true + : entries.every((model) => pick(model) === false) ? false + : undefined; + const base = entries[0]; + const keptModality = (modality: ModelModality): boolean => + entries.every((model) => model.modalities.input.includes(modality)); + const inputModalities = base.modalities.input.filter(keptModality); + const outputModalities = base.modalities.output.filter((modality) => + entries.every((model) => model.modalities.output.includes(modality)), + ); + const context = medianOf(entries.map((model) => model.limit.context)); + const output = medianOf(entries.map((model) => model.limit.output)); + const source = entries.find( + (model) => + (context === undefined || model.limit.context === context) && + (output === undefined || model.limit.output === output), + ) ?? base; + const reasoning = every((model) => model.reasoning) === true; + return { + ...source, + // Capabilities that must never be asserted on one publisher's word alone. + toolCall, + reasoning, + thinkingLevels: reasoning ? source.thinkingLevels : [], + structuredOutput: every((model) => model.structuredOutput), + attachment: every((model) => model.attachment), + modalities: { + input: inputModalities.length > 0 ? inputModalities : ["text"], + output: outputModalities.length > 0 ? outputModalities : ["text"], + }, + limit: { + ...(context !== undefined ? { context } : {}), + ...(output !== undefined ? { output } : {}), + }, + }; +} + export class ModelsDevCatalog { private providers = new Map(); private lookupIndex: ModelsDevLookupIndex | undefined; @@ -1030,17 +1109,74 @@ export class ModelsDevCatalog { candidates.sort((left, right) => right.score - left.score || left.model.modelId.length - right.model.modelId.length, ); - // An unknown endpoint can use a unique catalog match (including supported - // proxy aliases), but scores are not proof of provider identity. If two - // providers publish the same ID, do not borrow either one's metadata. - const result = preferredProvider || candidates.length === 1 - ? candidates[0]?.model - : undefined; + /* Within the row's own catalog provider this is an exact-provider lookup: + the provider identity is known, so its record for the id — a direct hit + or a supported alias — is authoritative. + + With no provider identity the scores prove nothing about identity, so a + catalog answer is only usable when it is unambiguous. Two providers + publishing the same id must not have one of them chosen for the other. */ + const resolved = candidates[0]?.model; + const result = preferredProvider + ? resolved + : candidates.length === 1 ? resolved : undefined; + /* The row's own catalog provider did not publish this id. Borrowing needs a + known provider identity to anchor on: the row resolved to a catalog + provider whose own records are authoritative, so anything missing from it + can be checked against the other publishers of the exact id instead of + dropping the model to the generic shape. + + This only fills a miss the lookup already had — a record the preferred + provider does publish stays authoritative. Without that anchor the scores + prove nothing about identity, and an unknown endpoint keeps the existing + behaviour: a unique unambiguous match, or nothing. */ + const borrowed = result ?? + (preferredProvider ? this.borrowedAcrossProviders(input) : undefined); // Cache the result (a miss included) so a repeated miss is also O(1) and // cannot grow the candidate index with query-dependent keys. - this.lookupMemo.set(memoKey, result); + this.lookupMemo.set(memoKey, borrowed); if (this.lookupMemo.size > LOOKUP_MEMO_LIMIT) this.lookupMemo.clear(); - return result; + return borrowed; + } + + /** + * Exact-id fallback across catalog providers, used only when the row resolved + * to a catalog provider that published nothing for the id. + * + * A gateway's copy of a model is indexed under the vendor that owns the + * weights, so an endpoint serving `Vendor/Model` ids routinely has no record + * of its own while other publishers state the identical id. Their record is + * what keeps that model's window, tool support and vision visible. + * + * Two exclusions keep this from borrowing anything identity-sensitive: + * + * - A provider sharing the row's own endpoint is an alias for the row, not an + * independent source. Its failure to publish the id is an answer about this + * deployment, so nothing is borrowed past it. + * - Only an exactly-identical id transfers. The index also reaches a record + * through aliases — a bare route leaf behind a prefix, a vendor-prefixed + * variant — and those describe a *different* id, whose limits and + * capabilities are not this model's. + */ + private borrowedAcrossProviders( + input: { vendorKey?: string; baseUrl?: string; modelId: string }, + ): ModelsDevModel | undefined { + // `findModel` already rejected an empty id; normalize here so the compare + // below is case-insensitive against the catalog's own normalization. + const requested = normalizedModelId(input.modelId); + const matches: ModelsDevModel[] = []; + const seen = new Set(); + for (const candidate of this.lookupIndex?.candidates(requested) ?? []) { + const { model, provider } = candidate; + // The row's endpoint owns its own answers, including a negative one. + if (apiMatches(input.baseUrl, provider.api)) continue; + if (normalizedModelId(model.modelId) !== requested) continue; + const key = `${provider.providerKey}\u0000${model.modelId}`; + if (seen.has(key)) continue; + seen.add(key); + matches.push(model); + } + return borrowedModel(matches); } modelsForProvider(input: { vendorKey?: string; baseUrl?: string; providerId: string }): ModelInfo[] { diff --git a/apps/desktop/test/models-dev-catalog.test.mjs b/apps/desktop/test/models-dev-catalog.test.mjs index 9922603a2e..9554e141cb 100644 --- a/apps/desktop/test/models-dev-catalog.test.mjs +++ b/apps/desktop/test/models-dev-catalog.test.mjs @@ -408,6 +408,187 @@ test("metadata lookup can share routed leaves and narrow suffix aliases without assert.equal(catalog.findModel({ modelId: "bar-thinking" })?.modelId, "bar-agent"); }); +/* + A gateway is indexed by models.dev under the vendor that owns the weights, so + an endpoint serving `Vendor/Model` ids often has no record of its own while + several other publishers state the identical id. Issue #938: those rows used + to fall back to the generic 128k text-only shape, hiding a correct context + window and tool support that the catalog does publish. +*/ +test("an id the row's own catalog provider lacks borrows a unanimous exact-id record", async (t) => { + const catalog = await loadFixtureCatalog(t, { + gateway: { api: "https://gateway.example/v1", models: { "gateway/own-model": { id: "gateway/own-model" } } }, + publisherA: { models: { "Vendor/Shared-Model-0731": { + id: "Vendor/Shared-Model-0731", tool_call: true, reasoning: true, + modalities: { input: ["text", "image"], output: ["text"] }, + attachment: true, family: "shared", + limit: { context: 262_144, output: 65_536 }, + } } }, + publisherB: { models: { "Vendor/Shared-Model-0731": { + id: "Vendor/Shared-Model-0731", tool_call: true, reasoning: true, + modalities: { input: ["text", "image"], output: ["text"] }, + attachment: true, family: "shared", + limit: { context: 1_048_576, output: 32_768 }, + } } }, + }); + const match = catalog.findModel({ + vendorKey: "gateway", + baseUrl: "https://gateway.example/v1", + modelId: "Vendor/Shared-Model-0731", + }); + assert.ok(match, "a published exact id must not drop to the generic shape"); + assert.equal(match.toolCall, true); + assert.equal(match.reasoning, true); + const info = modelInfoFromModelsDev(match, "provider-1"); + assert.equal(info.capabilities.includes("tools"), true); + assert.equal(info.capabilities.includes("vision"), true); + // Limits are the medians the publishers state; an even count takes the lower + // middle, so a borrow never rounds a window up on its own. + assert.equal(match.limit.context, 262_144); + assert.equal(match.limit.output, 32_768); +}); + +test("a borrowed record under-claims instead of asserting one publisher's extras", async (t) => { + // Both publishers agree on tool support, so the id is borrowable. They differ + // on vision, reasoning and structured output, so none of those may be + // reported: the borrow may only claim what every publisher states. + const catalog = await loadFixtureCatalog(t, { + gateway: { api: "https://gateway.example/v1", models: { "gateway/own-model": { id: "gateway/own-model" } } }, + publisherA: { models: { "Vendor/Mixed": { + id: "Vendor/Mixed", tool_call: true, reasoning: true, structured_output: true, + modalities: { input: ["text", "image"], output: ["text"] }, attachment: true, + limit: { context: 262_144 }, + } } }, + publisherB: { models: { "Vendor/Mixed": { + id: "Vendor/Mixed", tool_call: true, reasoning: false, structured_output: false, + modalities: { input: ["text"], output: ["text"] }, attachment: false, + limit: { context: 262_144 }, + } } }, + }); + const match = catalog.findModel({ + vendorKey: "gateway", + baseUrl: "https://gateway.example/v1", + modelId: "Vendor/Mixed", + }); + assert.ok(match, "unanimous tool support is enough to borrow"); + assert.equal(match.toolCall, true); + assert.equal(match.reasoning, false); + // A capability the publishers do not state unanimously is left unasserted + // (`undefined` means "no published answer"), never claimed from one source. + assert.equal(match.structuredOutput, undefined); + assert.equal(match.attachment, undefined); + assert.deepEqual(match.modalities.input, ["text"]); + const info = modelInfoFromModelsDev(match, "provider-1"); + assert.equal(info.capabilities.includes("vision"), false); + assert.equal(info.capabilities.includes("reasoning"), false); +}); + +test("publishers split on tool support are not borrowed from at all", async (t) => { + // Tool support is the gate: a wrong `true` would put tool declarations on the + // wire that the endpoint may reject, so a split here refuses the borrow. + const catalog = await loadFixtureCatalog(t, { + gateway: { api: "https://gateway.example/v1", models: {} }, + publisherA: { models: { "Vendor/Split": { id: "Vendor/Split", tool_call: true, limit: { context: 262_144 } } } }, + publisherB: { models: { "Vendor/Split": { id: "Vendor/Split", tool_call: false, limit: { context: 262_144 } } } }, + }); + assert.equal( + catalog.findModel({ + vendorKey: "gateway", + baseUrl: "https://gateway.example/v1", + modelId: "Vendor/Split", + }), + undefined, + ); +}); + +test("a borrowed record never replaces what the row's own catalog provider publishes", async (t) => { + const catalog = await loadFixtureCatalog(t, { + gateway: { api: "https://gateway.example/v1", models: { + "Vendor/Shared-Model": { id: "Vendor/Shared-Model", tool_call: false, limit: { context: 200_000 } }, + } }, + publisherA: { models: { + "Vendor/Shared-Model": { id: "Vendor/Shared-Model", tool_call: true, limit: { context: 1_048_576 } }, + } }, + }); + const match = catalog.findModel({ + vendorKey: "gateway", + baseUrl: "https://gateway.example/v1", + modelId: "Vendor/Shared-Model", + }); + assert.equal(match?.providerKey, "gateway"); + assert.equal(match.limit.context, 200_000); + assert.equal(match.toolCall, false); +}); + +test("publishers that disagree about capabilities are not borrowed from", async (t) => { + const catalog = await loadFixtureCatalog(t, { + gateway: { api: "https://gateway.example/v1", models: {} }, + publisherA: { models: { "Vendor/Disputed": { + id: "Vendor/Disputed", tool_call: true, reasoning: true, limit: { context: 262_144 }, + } } }, + publisherB: { models: { "Vendor/Disputed": { + id: "Vendor/Disputed", tool_call: false, reasoning: false, limit: { context: 262_144 }, + } } }, + }); + assert.equal( + catalog.findModel({ + vendorKey: "gateway", + baseUrl: "https://gateway.example/v1", + modelId: "Vendor/Disputed", + }), + undefined, + "a disputed capability shape must not be resolved by picking one publisher", + ); +}); + +test("borrowing stays exact-id, provider-scoped and absent for unknown ids", async (t) => { + const catalog = await loadFixtureCatalog(t, { + gateway: { api: "https://gateway.example/v1", models: {} }, + publisherA: { models: { + "Vendor/Known": { id: "Vendor/Known", limit: { context: 262_144 } }, + "Vendor/Known-2507": { id: "Vendor/Known-2507", limit: { context: 131_072 } }, + } }, + }); + const input = { vendorKey: "gateway", baseUrl: "https://gateway.example/v1" }; + // A near-miss id is not a reason to reuse a sibling's limits. + assert.equal(catalog.findModel({ ...input, modelId: "Vendor/Known-3000" }), undefined); + assert.equal(catalog.findModel({ ...input, modelId: "Other/Known" }), undefined); + assert.equal(catalog.findModel({ ...input, modelId: "unknown-model-xyz" }), undefined); + // Exact ids still resolve, including a case-only difference. + assert.equal(catalog.findModel({ ...input, modelId: "vendor/known" })?.limit.context, 262_144); + assert.equal(catalog.findModel({ ...input, modelId: "Vendor/Known" })?.limit.context, 262_144); + assert.equal(catalog.findModel({ ...input, modelId: "Vendor/Known-2507" })?.limit.context, 131_072); + // The sibling exists under a different id, so nothing is borrowed through it. + assert.equal( + catalog.findModel({ ...input, modelId: "Vendor/Known-2507" })?.modelId, + "Vendor/Known-2507", + ); +}); + +test("a provider sharing the row's endpoint owns its own negative answer", async (t) => { + // Two names for one endpoint: an id published only by the sibling is not + // borrowed, because that sibling was already consulted and does not list it. + const catalog = await loadFixtureCatalog(t, { + alpha: { api: "https://gateway.example/v1", models: { "alpha-only": { id: "alpha-only" } } }, + beta: { api: "https://gateway.example/v1", models: { "beta-only": { id: "beta-only" } } }, + }); + const input = { vendorKey: "beta", baseUrl: "https://gateway.example/v1" }; + assert.equal(catalog.findModel({ ...input, modelId: "alpha-only" }), undefined); + // A different endpoint is a genuinely independent source and still transfers. + const other = await loadFixtureCatalog(t, { + gateway: { api: "https://gateway.example/v1", models: {} }, + publisherA: { models: { "Vendor/Elsewhere": { id: "Vendor/Elsewhere", limit: { context: 262_144 } } } }, + }); + assert.equal( + other.findModel({ + vendorKey: "gateway", + baseUrl: "https://gateway.example/v1", + modelId: "Vendor/Elsewhere", + })?.limit.context, + 262_144, + ); +}); + test("models.dev records retain all published model parameters and modalities", () => { const [provider] = parseModelsDevCatalog(catalogFixture); assert.equal(provider.providerKey, "anthropic"); diff --git a/docs/spec/03-runtime/13-model-catalog-and-selection.md b/docs/spec/03-runtime/13-model-catalog-and-selection.md index afceb37aea..21347b3546 100644 --- a/docs/spec/03-runtime/13-model-catalog-and-selection.md +++ b/docs/spec/03-runtime/13-model-catalog-and-selection.md @@ -487,6 +487,35 @@ wire model ID or prove that a suffix enables reasoning. An unmatched free-form ID remains an unknown generic model with no inferred capabilities; only a published record or explicit binding settings can supply them. +#### 11.3.1 Cross-provider exact-id fallback + +models.dev indexes a gateway's copy of a model under the vendor that owns the +weights, so an endpoint serving `Vendor/Model` ids can have no record of its own +while several other publishers state the identical id. When the row resolves to +a catalog provider whose own record is missing, `findModel` consults the other +publishers of the **exact** id instead of leaving the model on the generic +128k text-only shape (issue #938). + +The borrow is bounded: + +- It runs only for a row with a known catalog provider identity, and only after + that provider's own lookup missed. A provider record, or a supported alias of + it, stays authoritative. +- A provider sharing the row's own endpoint is an alias for the row, so its + silence is an answer about this deployment and nothing is borrowed past it. +- Only a case-insensitive identical id transfers; a record reached through an + alias (a bare route leaf, a vendor-prefixed variant) is a different id and + keeps its own limits. +- Tool support must be unanimous across those publishers, because a wrong `true` + puts tool declarations on the wire that the endpoint may reject. Reasoning, + image/PDF input and attachment are the intersection, so a borrow may only + under-claim; a user who knows the endpoint does more still enables it in + Advanced. Limits are the medians the publishers state, never one host's cap. +- An id no publisher states stays an unknown generic model. + +This changes metadata only. The configured wire id, provider identity, and the +binding precedence in §11.3 are unchanged. + ## 12. Refresh strategy - manual refresh button in settings/model picker