diff --git a/README.md b/README.md index ac716486a..7b44b7259 100644 --- a/README.md +++ b/README.md @@ -91,7 +91,7 @@ const { text } = await generateText({ // Generate embeddings const { embedding } = await embed({ - model: apple.textEmbeddingModel(), + model: apple.embeddingModel(), value: 'Hello world', }) @@ -166,11 +166,11 @@ Run any GGUF model on-device using [llama.rn](https://github.com/mybigday/llama. #### Supported Features -| Feature | Method | Description | -| --------------- | ---------------------------- | --------------------------------------------- | -| Text Generation | `llama.languageModel()` | Chat, completion, streaming, reasoning models | -| Embeddings | `llama.textEmbeddingModel()` | Text embeddings for RAG and similarity search | -| Speech | `llama.speechModel()` | Text-to-speech with vocoder models | +| Feature | Method | Description | +| --------------- | ------------------------ | --------------------------------------------- | +| Text Generation | `llama.languageModel()` | Chat, completion, streaming, reasoning models | +| Embeddings | `llama.embeddingModel()` | Text embeddings for RAG and similarity search | +| Speech | `llama.speechModel()` | Text-to-speech with vocoder models | #### Installation diff --git a/packages/adk/src/ai-sdk.ts b/packages/adk/src/ai-sdk.ts index 79e7f4aba..079e1505c 100644 --- a/packages/adk/src/ai-sdk.ts +++ b/packages/adk/src/ai-sdk.ts @@ -59,6 +59,7 @@ async function checkNanoSupported(): Promise { export interface AdkProvider { (): LanguageModelV4 + readonly specificationVersion: 'v4' languageModel: () => LanguageModelV4 /** Device capability check (ML Kit `checkStatus !== 0`). Returns `false` on non-Android. */ isNanoSupported: () => Promise @@ -79,6 +80,7 @@ export function createAdkProvider( const provider = function () { return createLanguageModel() } + provider.specificationVersion = 'v4' as const provider.languageModel = createLanguageModel provider.isNanoSupported = () => checkNanoSupported() provider.isAvailable = async (modelType: AdkModelType = 'genai-nano') => { diff --git a/packages/apple-llm/ios/AppleLLM.mm b/packages/apple-llm/ios/AppleLLM.mm index 55ef1d9f9..74d34c034 100644 --- a/packages/apple-llm/ios/AppleLLM.mm +++ b/packages/apple-llm/ios/AppleLLM.mm @@ -208,8 +208,12 @@ - (void)generateStream:(nonnull NSString *)streamId messages:(nonnull NSArray *) } [self emitOnStreamUpdate:payload]; } - onComplete:^(NSString *streamId) { - [self emitOnStreamComplete:@{@"streamId": streamId}]; + onComplete:^(NSString *streamId, NSDictionary *usage) { + NSMutableDictionary *payload = [@{@"streamId": streamId} mutableCopy]; + if (usage != nil) { + payload[@"usage"] = usage; + } + [self emitOnStreamComplete:payload]; } onError:^(NSString *streamId, NSString *code, NSString *error) { NSMutableDictionary *payload = [@{@"streamId": streamId, @"error": error} mutableCopy]; diff --git a/packages/apple-llm/ios/AppleLLMImpl.swift b/packages/apple-llm/ios/AppleLLMImpl.swift index 55ebee62c..6c80e25bd 100644 --- a/packages/apple-llm/ios/AppleLLMImpl.swift +++ b/packages/apple-llm/ios/AppleLLMImpl.swift @@ -254,7 +254,7 @@ public class AppleLLMImpl: NSObject { messages: [[String: Any]], options: [String: Any], onUpdate: @escaping (String, String, [[String: Any]]?) -> Void, - onComplete: @escaping (String) -> Void, + onComplete: @escaping (String, [String: Any]?) -> Void, onError: @escaping (String, String, String) -> Void, toolInvoker: @escaping ToolInvoker ) { @@ -275,7 +275,7 @@ public class AppleLLMImpl: NSObject { let generationSchema = try self.createGenerationSchema(from: options) do { - try await self.streamResponse( + let usage = try await self.streamResponse( with: session, userPrompt: userPrompt, promptAttachments: promptAttachments, @@ -287,7 +287,7 @@ public class AppleLLMImpl: NSObject { } if !Task.isCancelled { - onComplete(streamId) + onComplete(streamId, usage) } } catch { if Task.isCancelled { @@ -525,7 +525,7 @@ public class AppleLLMImpl: NSObject { generationSchema: GenerationSchema?, generationOptions: GenerationOptions, providerOptions: [String: Any] - ) async throws -> [[String: Any]] { + ) async throws -> [String: Any] { #if compiler(>=6.4) if #available(iOS 27, *) { let contextOptions = try createContextOptions( @@ -548,7 +548,7 @@ public class AppleLLMImpl: NSObject { attachment } } - return response.toModelMessages() + return Self.generationResult(content: response.toModelMessages(), usage: Self.usageDictionary(response.usage)) } let response = try await session.respond( @@ -560,7 +560,7 @@ public class AppleLLMImpl: NSObject { attachment } } - return response.toModelMessages() + return Self.generationResult(content: response.toModelMessages(), usage: Self.usageDictionary(response.usage)) } #endif @@ -569,6 +569,8 @@ public class AppleLLMImpl: NSObject { throw AppleLLMError.unsupportedOS } + let transcriptBefore = session.transcript + let content: [[String: Any]] if let generationSchema { let response = try await session.respond( to: userPrompt, @@ -576,11 +578,71 @@ public class AppleLLMImpl: NSObject { includeSchemaInPrompt: true, options: generationOptions ) - return response.toModelMessages() + content = response.toModelMessages() + } else { + let response = try await session.respond(to: userPrompt, options: generationOptions) + content = response.toModelMessages() } - let response = try await session.respond(to: userPrompt, options: generationOptions) - return response.toModelMessages() + let usage = await countUsage( + session: session, + transcriptBefore: transcriptBefore, + userPrompt: userPrompt, + providerOptions: providerOptions + ) + return Self.generationResult(content: content, usage: usage) + } + + private static func generationResult(content: [[String: Any]], usage: [String: Any]?) -> [String: Any] { + var result: [String: Any] = ["content": content] + if let usage { + result["usage"] = usage + } + return result + } + +#if compiler(>=6.4) + @available(iOS 27, *) + private static func usageDictionary(_ usage: LanguageModelSession.Usage) -> [String: Any] { + return [ + "inputTokens": usage.input.totalTokenCount, + "cachedInputTokens": usage.input.cachedTokenCount, + "outputTokens": usage.output.totalTokenCount, + "reasoningTokens": usage.output.reasoningTokenCount, + ] + } +#endif + + /// iOS 26 responses report no usage, so the system model's tokenizer (iOS 26.4) + /// counts the transcript before and after the response. Private Cloud Compute + /// has no tokenizer, so its usage stays unknown + @available(iOS 26, *) + private func countUsage( + session: LanguageModelSession, + transcriptBefore: Transcript, + userPrompt: String, + providerOptions: [String: Any] + ) async -> [String: Any]? { +#if compiler(>=6.3) + guard #available(iOS 26.4, *), (providerOptions["model"] as? String ?? "system") == "system" else { + return nil + } + + let model = SystemLanguageModel.default + guard let historyTokens = try? await model.tokenCount(for: transcriptBefore), + let promptTokens = try? await model.tokenCount(for: userPrompt), + let totalTokens = try? await model.tokenCount(for: session.transcript) else { + return nil + } + + let inputTokens = historyTokens + promptTokens + return [ + "inputTokens": inputTokens, + "outputTokens": max(totalTokens - inputTokens, 0), + ] +#else + return nil +#endif } @available(iOS 26, *) @@ -592,7 +654,20 @@ public class AppleLLMImpl: NSObject { generationOptions: GenerationOptions, providerOptions: [String: Any], onUpdate: @escaping (String, [[String: Any]]?) -> Void - ) async throws { + ) async throws -> [String: Any]? { + // Reasoning and tool activity are only sent when they change + var previousContent: String? + var previousEntries: NSArray = [] + func update(_ content: String, _ entries: [[String: Any]]) { + let entriesChanged = !(entries as NSArray).isEqual(previousEntries) + guard entriesChanged || content != previousContent else { + return + } + previousContent = content + previousEntries = entries as NSArray + onUpdate(content, entriesChanged ? entries : nil) + } + #if compiler(>=6.4) if #available(iOS 27, *) { let contextOptions = try createContextOptions( @@ -604,6 +679,7 @@ public class AppleLLMImpl: NSObject { Self.removeTemporaryFiles(preparedAttachments.temporaryFileURLs) } + var usage: LanguageModelSession.Usage? if let generationSchema { let responseStream = session.streamResponse( schema: generationSchema, @@ -616,29 +692,31 @@ public class AppleLLMImpl: NSObject { } } for try await chunk in responseStream { - onUpdate(String(describing: chunk.content), nil) + usage = chunk.usage + update( + chunk.rawContent.jsonString, + AppleLLMTranscriptSerializer.serialize(chunk.transcriptEntries, includeResponses: false) + ) } - return - } - - let responseStream = session.streamResponse( - options: generationOptions, - contextOptions: contextOptions - ) { - userPrompt - for attachment in preparedAttachments.values { - attachment + } else { + let responseStream = session.streamResponse( + options: generationOptions, + contextOptions: contextOptions + ) { + userPrompt + for attachment in preparedAttachments.values { + attachment + } + } + for try await chunk in responseStream { + usage = chunk.usage + update( + chunk.content, + AppleLLMTranscriptSerializer.serialize(chunk.transcriptEntries, includeResponses: false) + ) } } - // Reasoning and tool activity are only sent when they change - var previousEntries: NSArray = [] - for try await chunk in responseStream { - let entries = AppleLLMTranscriptSerializer.serialize(chunk.transcriptEntries, includeResponses: false) - let entriesChanged = !(entries as NSArray).isEqual(previousEntries) - previousEntries = entries as NSArray - onUpdate(chunk.content, entriesChanged ? entries : nil) - } - return + return usage.map(Self.usageDictionary) } #endif @@ -647,6 +725,17 @@ public class AppleLLMImpl: NSObject { throw AppleLLMError.unsupportedOS } + // iOS 26 snapshots have no transcript entries, so tool activity is read from + // the session transcript, which grows while the response streams + let transcriptBefore = session.transcript + let newEntries = { + AppleLLMTranscriptSerializer.serialize( + session.transcript.dropFirst(transcriptBefore.count), + includeResponses: false + ) + } + + var content = "" if let generationSchema { let responseStream = session.streamResponse( to: userPrompt, @@ -655,15 +744,25 @@ public class AppleLLMImpl: NSObject { options: generationOptions ) for try await chunk in responseStream { - onUpdate(String(describing: chunk.content), nil) + content = chunk.rawContent.jsonString + update(content, newEntries()) + } + } else { + let responseStream = session.streamResponse(to: userPrompt, options: generationOptions) + for try await chunk in responseStream { + content = chunk.content + update(content, newEntries()) } - return } + // Entries added after the last snapshot + update(content, newEntries()) - let responseStream = session.streamResponse(to: userPrompt, options: generationOptions) - for try await chunk in responseStream { - onUpdate(chunk.content, nil) - } + return await countUsage( + session: session, + transcriptBefore: transcriptBefore, + userPrompt: userPrompt, + providerOptions: providerOptions + ) } #if compiler(>=6.4) diff --git a/packages/apple-llm/src/AppleFoundationModels.ts b/packages/apple-llm/src/AppleFoundationModels.ts deleted file mode 100644 index 74ff7873b..000000000 --- a/packages/apple-llm/src/AppleFoundationModels.ts +++ /dev/null @@ -1,34 +0,0 @@ -import type { AppleLanguageModelId } from './ai-sdk' -import NativeAppleLLM, { - type AppleGenerationOptions, - type AppleMessage, - type Spec, -} from './NativeAppleLLM' - -type AppleFoundationModelsSpec = Omit & { - getModelInfo( - locale?: string, - model?: AppleLanguageModelId - ): ReturnType -} - -const AppleFoundationModels: AppleFoundationModelsSpec = { - isAvailable: () => NativeAppleLLM.isAvailable(), - getModelInfo: (locale?: string, model?: AppleLanguageModelId) => - NativeAppleLLM.getModelInfo(locale, model), - countTokens: (text) => NativeAppleLLM.countTokens(text), - generateImages: (options) => NativeAppleLLM.generateImages(options), - generateText: (messages: AppleMessage[], options: AppleGenerationOptions) => - NativeAppleLLM.generateText(messages, options), - generateStream: ( - streamId, - messages: AppleMessage[], - options: AppleGenerationOptions - ) => NativeAppleLLM.generateStream(streamId, messages, options), - cancelStream: (streamId) => NativeAppleLLM.cancelStream(streamId), - onStreamUpdate: NativeAppleLLM.onStreamUpdate, - onStreamComplete: NativeAppleLLM.onStreamComplete, - onStreamError: NativeAppleLLM.onStreamError, -} - -export default AppleFoundationModels diff --git a/packages/apple-llm/src/NativeAppleLLM.ts b/packages/apple-llm/src/NativeAppleLLM.ts index dbd477731..5dbf6002b 100644 --- a/packages/apple-llm/src/NativeAppleLLM.ts +++ b/packages/apple-llm/src/NativeAppleLLM.ts @@ -27,8 +27,19 @@ export type StreamUpdateEvent = { entries?: CodegenTypes.UnsafeObject[] } +// Token counts from Foundation Models (iOS 27) or the system model's +// tokenizer (iOS 26.4). Left out when they are unknown +export type AppleUsage = { + inputTokens: number + cachedInputTokens?: number + outputTokens: number + reasoningTokens?: number +} + export type StreamCompleteEvent = { streamId: string + // AppleUsage + usage?: CodegenTypes.UnsafeObject } export type StreamErrorEvent = { @@ -48,8 +59,8 @@ export interface Spec extends TurboModule { generateText( messages: AppleMessage[], options: AppleGenerationOptions - ): Promise< - ( + ): Promise<{ + content: ( | { type: 'text'; text: string } | { type: 'reasoning' @@ -70,7 +81,8 @@ export interface Spec extends TurboModule { output: string } )[] - > + usage?: AppleUsage + }> generateStream( streamId: string, messages: AppleMessage[], diff --git a/packages/apple-llm/src/__tests__/ai-sdk.test.ts b/packages/apple-llm/src/__tests__/ai-sdk.test.ts index 1d6ab19ce..80cab1693 100644 --- a/packages/apple-llm/src/__tests__/ai-sdk.test.ts +++ b/packages/apple-llm/src/__tests__/ai-sdk.test.ts @@ -21,8 +21,11 @@ const generateText = mock( async ( _messages: unknown, _options: Record - ): Promise[]> => [{ type: 'text', text: 'ok' }] + ): Promise> => ({ + content: [{ type: 'text', text: 'ok' }], + }) ) +const countTokens = mock(async (text: string) => text.length) const generateStream = mock( (_streamId: string, _messages: unknown, _options: unknown) => {} ) @@ -37,6 +40,7 @@ mock.module('react-native', () => ({ ? { generateText, generateStream, + countTokens, cancelStream: () => {}, onStreamUpdate, onStreamComplete, @@ -354,22 +358,24 @@ async function readAll(stream: ReadableStream) { describe('apple generated content', () => { test('returns reasoning and tool parts with their ids', async () => { - generateText.mockImplementationOnce(async () => [ - { type: 'reasoning', id: 'r1', text: 'thinking', signature: 'c2ln' }, - { - type: 'tool-call', - toolCallId: 'call-1', - toolName: 'weather', - input: '{"city":"Paris"}', - }, - { - type: 'tool-result', - toolCallId: 'call-1', - toolName: 'weather', - output: 'sunny', - }, - { type: 'text', text: 'It is sunny' }, - ]) + generateText.mockImplementationOnce(async () => ({ + content: [ + { type: 'reasoning', id: 'r1', text: 'thinking', signature: 'c2ln' }, + { + type: 'tool-call', + toolCallId: 'call-1', + toolName: 'weather', + input: '{"city":"Paris"}', + }, + { + type: 'tool-result', + toolCallId: 'call-1', + toolName: 'weather', + output: 'sunny', + }, + { type: 'text', text: 'It is sunny' }, + ], + })) const result = await apple().doGenerate(createCallOptions()) @@ -469,3 +475,134 @@ describe('apple generated content', () => { ]) }) }) + +describe('apple usage', () => { + test('maps native token usage', async () => { + generateText.mockImplementationOnce(async () => ({ + content: [{ type: 'text', text: 'ok' }], + usage: { + inputTokens: 120, + cachedInputTokens: 20, + outputTokens: 30, + reasoningTokens: 10, + }, + })) + + const result = await apple().doGenerate(createCallOptions()) + + expect(result.usage).toEqual({ + inputTokens: { + total: 120, + noCache: 100, + cacheRead: 20, + cacheWrite: undefined, + }, + outputTokens: { total: 30, text: 20, reasoning: 10 }, + }) + }) + + test('reports unknown usage as undefined instead of zero', async () => { + const result = await apple().doGenerate(createCallOptions()) + + expect(result.usage).toEqual({ + inputTokens: { + total: undefined, + noCache: undefined, + cacheRead: undefined, + cacheWrite: undefined, + }, + outputTokens: { total: undefined, text: undefined, reasoning: undefined }, + }) + }) + + test('reports stream usage on finish', async () => { + const { stream } = await apple().doStream(createCallOptions()) + const parts = readAll(stream) + const streamId = generateStream.mock.calls.at(-1)![0] + + onStreamUpdate.emit({ streamId, content: 'ok' }) + onStreamComplete.emit({ + streamId, + usage: { inputTokens: 12, outputTokens: 3 }, + }) + + const finish = (await parts).find((part) => part.type === 'finish') + expect(finish).toMatchObject({ + usage: { + inputTokens: { total: 12, noCache: undefined, cacheRead: undefined }, + outputTokens: { total: 3, text: undefined, reasoning: undefined }, + }, + }) + }) +}) + +describe('apple structured output stream', () => { + test('sends the final json once the stream completes', async () => { + const { stream } = await apple().doStream( + createCallOptions({ + responseFormat: { + type: 'json', + schema: { type: 'object', properties: { city: { type: 'string' } } }, + }, + }) + ) + const parts = readAll(stream) + const streamId = generateStream.mock.calls.at(-1)![0] + + onStreamUpdate.emit({ streamId, content: '{"city":"Par"}' }) + onStreamUpdate.emit({ streamId, content: '{"city":"Paris"}' }) + onStreamComplete.emit({ streamId }) + + expect( + (await parts).filter( + (part) => part.type !== 'stream-start' && part.type !== 'finish' + ) + ).toEqual([ + { type: 'text-start', id: streamId }, + { type: 'text-delta', id: streamId, delta: '{"city":"Paris"}' }, + { type: 'text-end', id: streamId }, + ]) + }) +}) + +describe('apple with the ai sdk', () => { + test('streams structured output and usage through streamText', async () => { + const { jsonSchema, Output, streamText } = await import('ai') + generateStream.mockImplementationOnce((streamId) => { + setTimeout(() => { + onStreamUpdate.emit({ streamId, content: '{"city":"Par"}' }) + onStreamUpdate.emit({ streamId, content: '{"city":"Paris"}' }) + onStreamComplete.emit({ + streamId, + usage: { inputTokens: 40, outputTokens: 6 }, + }) + }) + }) + + const result = streamText({ + model: apple(), + prompt: 'Where is the Eiffel Tower?', + output: Output.object({ + schema: jsonSchema<{ city: string }>({ + type: 'object', + properties: { city: { type: 'string' } }, + required: ['city'], + }), + }), + }) + + expect(await result.output).toEqual({ city: 'Paris' }) + expect(await result.usage).toMatchObject({ + inputTokens: 40, + outputTokens: 6, + totalTokens: 46, + }) + }) +}) + +describe('apple token counting', () => { + test('counts tokens through the language model', async () => { + expect(await apple().countTokens('hello')).toBe(5) + expect(countTokens).toHaveBeenCalledWith('hello') + }) +}) diff --git a/packages/apple-llm/src/ai-sdk.ts b/packages/apple-llm/src/ai-sdk.ts index 4ba9540ef..c4f5bd6f2 100644 --- a/packages/apple-llm/src/ai-sdk.ts +++ b/packages/apple-llm/src/ai-sdk.ts @@ -12,6 +12,7 @@ import type { LanguageModelV4ProviderTool, LanguageModelV4StreamPart, LanguageModelV4ToolResultOutput, + LanguageModelV4Usage, SharedV4Warning, SpeechModelV4, SpeechModelV4CallOptions, @@ -28,7 +29,10 @@ import { import { createAppleLLMError, isAppleLLMErrorCode } from './errors' import NativeAppleEmbeddings from './NativeAppleEmbeddings' -import NativeAppleLLM, { type AppleMessage } from './NativeAppleLLM' +import NativeAppleLLM, { + type AppleMessage, + type AppleUsage, +} from './NativeAppleLLM' import NativeAppleSpeech from './NativeAppleSpeech' import NativeAppleTranscription from './NativeAppleTranscription' import NativeAppleUtils from './NativeAppleUtils' @@ -39,6 +43,12 @@ export type AppleToolDefinitionSet = Record export interface AppleLanguageModel extends LanguageModelV4 { prepare: () => Promise updateTools: (tools: AppleToolDefinitionSet) => void + /** + * Counts tokens in text with the on-device model's tokenizer (iOS 26.4+). + * Private Cloud Compute has no token counting API, so Private Cloud Compute + * models also count with the on-device tokenizer, which may not match + */ + countTokens: (text: string) => Promise } export type AppleLanguageModelId = 'system' | 'private-cloud-compute' export type AppleBuiltInTool = 'ocr' | 'barcode' @@ -445,6 +455,30 @@ function prepareToolChoice( } } +function toUsage(usage: AppleUsage | undefined): LanguageModelV4Usage { + const cachedInputTokens = usage?.cachedInputTokens + const reasoningTokens = usage?.reasoningTokens + return { + inputTokens: { + total: usage?.inputTokens, + noCache: + usage && cachedInputTokens !== undefined + ? usage.inputTokens - cachedInputTokens + : undefined, + cacheRead: cachedInputTokens, + cacheWrite: undefined, + }, + outputTokens: { + total: usage?.outputTokens, + text: + usage && reasoningTokens !== undefined + ? usage.outputTokens - reasoningTokens + : undefined, + reasoning: reasoningTokens, + }, + } +} + function appendMessageContent(content: string, addition: string) { if (!addition) { return content @@ -474,6 +508,7 @@ export function createAppleProvider({ ) { return createLanguageModel(options) } + provider.specificationVersion = 'v4' as const provider.isAvailable = () => NativeAppleLLM.isAvailable() provider.getModelInfo = ( options: { locale?: string; model?: AppleLanguageModelId } = {} @@ -483,9 +518,11 @@ export function createAppleProvider({ options.model ) as Promise provider.languageModel = createLanguageModel - provider.textEmbeddingModel = (options: AppleEmbeddingOptions = {}) => { + provider.embeddingModel = (options: AppleEmbeddingOptions = {}) => { return new AppleTextEmbeddingModel(options) } + /** @deprecated Use `embeddingModel` */ + provider.textEmbeddingModel = provider.embeddingModel provider.imageModel = (options: AppleImageModelOptions = {}) => { return new AppleImageModel(options) } @@ -731,6 +768,10 @@ class AppleLLMChatLanguageModel implements AppleLanguageModel { async prepare(): Promise {} + countTokens(text: string): Promise { + return NativeAppleLLM.countTokens(text) + } + private prepareMessages(messages: LanguageModelV4Prompt): AppleMessage[] { return messages.map((message, index): AppleMessage => { switch (message.role) { @@ -838,7 +879,7 @@ class AppleLLMChatLanguageModel implements AppleLanguageModel { }) return { - content: response.map((part) => { + content: response.content.map((part) => { switch (part.type) { case 'text': return part @@ -869,19 +910,7 @@ class AppleLLMChatLanguageModel implements AppleLanguageModel { } }), finishReason: { unified: 'stop' as const, raw: 'stop' }, - usage: { - inputTokens: { - total: 0, - noCache: undefined, - cacheRead: undefined, - cacheWrite: undefined, - }, - outputTokens: { - total: 0, - text: undefined, - reasoning: undefined, - }, - }, + usage: toUsage(response.usage), warnings, } } finally { @@ -912,10 +941,6 @@ class AppleLLMChatLanguageModel implements AppleLanguageModel { ? options.responseFormat.schema : undefined - if (schema) { - throw new Error('Streaming JSON responses is not yet supported.') - } - for (const tool of tools) { globalThis.__APPLE_LLM_TOOLS__[tool.id] = tool.execute } @@ -1023,6 +1048,14 @@ class AppleLLMChatLanguageModel implements AppleLanguageModel { } const nextRawContent = String(data.content ?? '') + + // Partial JSON snapshots are not prefixes of the final JSON, so + // structured output is sent once the stream completes + if (schema) { + previousRawContent = nextRawContent + return + } + const rawDelta = getSnapshotDelta( previousRawContent, nextRawContent @@ -1049,6 +1082,15 @@ class AppleLLMChatLanguageModel implements AppleLanguageModel { const completeListener = NativeAppleLLM.onStreamComplete((data) => { if (data.streamId === streamId) { endReasoning() + if (schema && previousRawContent) { + controller.enqueue({ type: 'text-start', id }) + controller.enqueue({ + type: 'text-delta', + delta: previousRawContent, + id, + }) + textStarted = true + } if (textStarted) { controller.enqueue({ type: 'text-end', @@ -1058,19 +1100,7 @@ class AppleLLMChatLanguageModel implements AppleLanguageModel { controller.enqueue({ type: 'finish', finishReason: { unified: 'stop' as const, raw: 'stop' }, - usage: { - inputTokens: { - total: 0, - noCache: undefined, - cacheRead: undefined, - cacheWrite: undefined, - }, - outputTokens: { - total: 0, - text: undefined, - reasoning: undefined, - }, - }, + usage: toUsage(data.usage as AppleUsage | undefined), }) cleanup() controller.close() diff --git a/packages/apple-llm/src/index.ts b/packages/apple-llm/src/index.ts index e0fbf1bb8..cc88e3eb2 100644 --- a/packages/apple-llm/src/index.ts +++ b/packages/apple-llm/src/index.ts @@ -14,10 +14,10 @@ export type { AppleToolDefinitionSet, } from './ai-sdk' export { apple, createAppleProvider } from './ai-sdk' -export { default as AppleFoundationModels } from './AppleFoundationModels' export type { AppleLLMError, AppleLLMErrorCode } from './errors' export { AppleLLMErrorCodes } from './errors' export { default as AppleEmbeddings } from './NativeAppleEmbeddings' +export { default as AppleFoundationModels } from './NativeAppleLLM' export { default as AppleSpeech, VoiceInfo } from './NativeAppleSpeech' export { default as AppleTranscription } from './NativeAppleTranscription' export { default as AppleUtils } from './NativeAppleUtils' diff --git a/packages/apple-llm/src/stream.ts b/packages/apple-llm/src/stream.ts deleted file mode 100644 index c3847aa65..000000000 --- a/packages/apple-llm/src/stream.ts +++ /dev/null @@ -1,98 +0,0 @@ -import type { LanguageModelV2StreamPart } from '@ai-sdk/provider' -import { generateId } from '@ai-sdk/provider-utils' - -import { createAppleLLMError, isAppleLLMErrorCode } from './errors' -import NativeAppleLLMSpec, { - type AppleGenerationOptions, - type AppleMessage, -} from './NativeAppleLLM' - -export function generateStream( - messages: AppleMessage[], - options: AppleGenerationOptions = {} -): ReadableStream { - if (typeof ReadableStream === 'undefined') { - throw new Error( - `ReadableStream is not available in this environment. Please load a polyfill, such as web-streams-polyfill.` - ) - } - - let streamId: string | null = null - let listeners: { remove(): void }[] = [] - - const cleanup = () => { - listeners.forEach((listener) => listener.remove()) - listeners = [] - } - - const stream = new ReadableStream({ - async start(controller) { - try { - streamId = generateId() - - controller.enqueue({ - type: 'text-start', - id: streamId, - }) - - const updateListener = NativeAppleLLMSpec.onStreamUpdate((data) => { - if (data.streamId === streamId) { - controller.enqueue({ - type: 'text-delta', - delta: data.content, - id: data.streamId, - }) - } - }) - - const completeListener = NativeAppleLLMSpec.onStreamComplete((data) => { - if (data.streamId === streamId) { - controller.enqueue({ - type: 'text-end', - id: streamId, - }) - controller.enqueue({ - type: 'finish', - finishReason: 'stop', - usage: { - inputTokens: 0, - outputTokens: 0, - totalTokens: 0, - }, - }) - cleanup() - controller.close() - } - }) - - const errorListener = NativeAppleLLMSpec.onStreamError((data) => { - if (data.streamId === streamId) { - controller.enqueue({ - type: 'error', - error: isAppleLLMErrorCode(data.code) - ? createAppleLLMError(data.error, data.code) - : new Error(data.error), - }) - cleanup() - controller.close() - } - }) - - listeners = [updateListener, completeListener, errorListener] - - NativeAppleLLMSpec.generateStream(streamId, messages, options) - } catch (error) { - cleanup() - controller.error(new Error(`Apple LLM stream failed: ${error}`)) - } - }, - cancel() { - cleanup() - if (streamId) { - NativeAppleLLMSpec.cancelStream(streamId) - } - }, - }) - - return stream -} diff --git a/packages/core-ai/src/ai-sdk.ts b/packages/core-ai/src/ai-sdk.ts index d6ffc9cf7..56610170e 100644 --- a/packages/core-ai/src/ai-sdk.ts +++ b/packages/core-ai/src/ai-sdk.ts @@ -38,14 +38,14 @@ export function createCoreAIProvider() { return provider.languageModel(config) } + provider.specificationVersion = 'v4' as const provider.getCapabilities = () => NativeCoreAI.getCapabilities() provider.languageModel = (config: CoreAIModelConfig) => { return new CoreAILanguageModel(config) } - provider.textEmbeddingModel = (config: CoreAIModelConfig) => { + provider.embeddingModel = (config: CoreAIModelConfig) => { return new CoreAITextEmbeddingModel(config) } - provider.embeddingModel = provider.textEmbeddingModel provider.imageModel = (config: CoreAIModelConfig) => { return new CoreAIImageGenerationModel(config) } @@ -564,13 +564,13 @@ function toFinishReason( function emptyUsage() { return { inputTokens: { - total: 0, + total: undefined, noCache: undefined, cacheRead: undefined, cacheWrite: undefined, }, outputTokens: { - total: 0, + total: undefined, text: undefined, reasoning: undefined, }, diff --git a/packages/llama/src/ai-sdk.ts b/packages/llama/src/ai-sdk.ts index 10a3ec1d0..3a50fedc8 100644 --- a/packages/llama/src/ai-sdk.ts +++ b/packages/llama/src/ai-sdk.ts @@ -193,6 +193,15 @@ export class LlamaLanguageModel implements LanguageModelV4 { } } + /** + * Count tokens in text with the model's tokenizer + */ + async countTokens(text: string): Promise { + const context = this.context ?? (await this.prepare()) + const { tokens } = await context.tokenize(text) + return tokens.length + } + /** * Get the underlying LlamaContext (for advanced usage) */ @@ -296,13 +305,13 @@ export class LlamaLanguageModel implements LanguageModelV4 { : convertFinishReason(response), usage: { inputTokens: { - total: response.timings?.prompt_n || 0, + total: response.timings?.prompt_n, noCache: undefined, cacheRead: undefined, cacheWrite: undefined, }, outputTokens: { - total: response.timings?.predicted_n || 0, + total: response.timings?.predicted_n, text: undefined, reasoning: undefined, }, @@ -373,13 +382,13 @@ export class LlamaLanguageModel implements LanguageModelV4 { : convertFinishReason(result), usage: { inputTokens: { - total: result.timings?.prompt_n || 0, + total: result.timings?.prompt_n, noCache: undefined, cacheRead: undefined, cacheWrite: undefined, }, outputTokens: { - total: result.timings?.predicted_n || 0, + total: result.timings?.predicted_n, text: undefined, reasoning: undefined, }, @@ -824,6 +833,8 @@ export function createLlamaProvider() { return provider.languageModel(modelPath, options) } + provider.specificationVersion = 'v4' as const + /** * Create a language model instance * @param modelPath - Path to the model file (from downloadModel() or getModelPath()) @@ -841,25 +852,31 @@ export function createLlamaProvider() { * @param modelPath - Path to the model file (from downloadModel() or getModelPath()) * @param options - Model configuration options */ - provider.textEmbeddingModel = ( + provider.embeddingModel = ( modelPath: string, options: LlamaEmbeddingOptions = {} ): LlamaEmbeddingModel => { return new LlamaEmbeddingModel(modelPath, options) } + /** @deprecated Use `embeddingModel` */ + provider.textEmbeddingModel = provider.embeddingModel + /** * Create a rerank model instance for document ranking * @param modelPath - Path to a reranker model file (from downloadModel() or getModelPath()) * @param options - Model configuration options */ - provider.rerankModel = ( + provider.rerankingModel = ( modelPath: string, options: LlamaRerankOptions = {} ): LlamaRerankModel => { return new LlamaRerankModel(modelPath, options) } + /** @deprecated Use `rerankingModel` */ + provider.rerankModel = provider.rerankingModel + /** * Create a speech model instance * @param modelPath - Path to the model file (from downloadModel() or getModelPath()) diff --git a/packages/mlc/src/ai-sdk.ts b/packages/mlc/src/ai-sdk.ts index 1eb70cc1d..26d2ef9cc 100644 --- a/packages/mlc/src/ai-sdk.ts +++ b/packages/mlc/src/ai-sdk.ts @@ -18,12 +18,21 @@ import NativeMLCEngine, { type Message, } from './NativeMLCEngine' -export const mlc = { - languageModel: (modelId: string = 'Llama-3.2-3B-Instruct') => { +export function createMlcProvider() { + const provider = function (modelId?: string) { + return provider.languageModel(modelId) + } + + provider.specificationVersion = 'v4' as const + provider.languageModel = (modelId: string = 'Llama-3.2-3B-Instruct') => { return new MlcChatLanguageModel(modelId) - }, + } + + return provider } +export const mlc = createMlcProvider() + const convertToolsToNativeFormat = ( tools: (LanguageModelV4FunctionTool | LanguageModelV4ProviderTool)[] ) => { diff --git a/packages/mlc/src/index.ts b/packages/mlc/src/index.ts index 45fa79e4d..9e96986ca 100644 --- a/packages/mlc/src/index.ts +++ b/packages/mlc/src/index.ts @@ -1,2 +1,2 @@ -export { mlc } from './ai-sdk' +export { createMlcProvider, mlc } from './ai-sdk' export { default as MLCEngine } from './NativeMLCEngine' diff --git a/packages/utils/README.md b/packages/utils/README.md index 2ac6a6051..9b73b23c0 100644 --- a/packages/utils/README.md +++ b/packages/utils/README.md @@ -14,3 +14,7 @@ const modelWithHistory = wrapLanguageModelWithHistory(model, { dropCompletedToolCalls: true, }) ``` + +`summarizeHistory` counts prompt tokens with the summarizer model's +`countTokens` method when it has one (Apple and llama.rn language models do). +Other models fall back to an estimate of four characters per token. diff --git a/packages/utils/src/__tests__/index.test.ts b/packages/utils/src/__tests__/index.test.ts new file mode 100644 index 000000000..4cfd067ef --- /dev/null +++ b/packages/utils/src/__tests__/index.test.ts @@ -0,0 +1,70 @@ +import type { + LanguageModelV4, + LanguageModelV4CallOptions, + LanguageModelV4Prompt, +} from '@ai-sdk/provider' +import { describe, expect, mock, test } from 'bun:test' + +import { applyHistoryManagement } from '../index' + +const summaryResult = { + content: [ + { type: 'text' as const, text: 'Earlier: the user asked about Paris' }, + ], + finishReason: { unified: 'stop' as const, raw: 'stop' }, + usage: { + inputTokens: { + total: undefined, + noCache: undefined, + cacheRead: undefined, + cacheWrite: undefined, + }, + outputTokens: { total: undefined, text: undefined, reasoning: undefined }, + }, + warnings: [], +} + +// A model whose countTokens reads instance state, like the llama.rn model +class TokenCountingModel implements LanguageModelV4 { + readonly specificationVersion = 'v4' + readonly provider = 'test' + readonly modelId = 'test' + readonly supportedUrls = {} + + private tokensPerCharacter = 1 + + doGenerate = mock( + async (_options: LanguageModelV4CallOptions) => summaryResult + ) + doStream = mock(async (_options: LanguageModelV4CallOptions) => { + throw new Error('not used') + }) + + async countTokens(text: string) { + return text.length * this.tokensPerCharacter + } +} + +const prompt: LanguageModelV4Prompt = [ + { role: 'user', content: [{ type: 'text', text: 'Tell me about Paris' }] }, + { + role: 'assistant', + content: [{ type: 'text', text: 'Paris is in France' }], + }, + { role: 'user', content: [{ type: 'text', text: 'And Lyon?' }] }, +] + +describe('history summarization', () => { + test('counts tokens with the model as receiver', async () => { + const model = new TokenCountingModel() + + // About 70 characters, so the length / 4 fallback (about 18) stays under + // the threshold and only the model's own count goes over it + const result = await applyHistoryManagement(prompt, { + summarizeHistory: { threshold: 40, model }, + }) + + expect(model.doGenerate).toHaveBeenCalledTimes(1) + expect(result).toHaveLength(2) + }) +}) diff --git a/packages/utils/src/index.ts b/packages/utils/src/index.ts index 77bbe30a2..0223630e4 100644 --- a/packages/utils/src/index.ts +++ b/packages/utils/src/index.ts @@ -100,11 +100,12 @@ async function countPromptTokens( model: LanguageModelV4 ): Promise { const serializedPrompt = serializePromptForSummary(prompt) - const countTokens = (model as TokenCountCapableLanguageModel).countTokens + const tokenCountModel = model as TokenCountCapableLanguageModel - if (typeof countTokens === 'function') { + // Called on the model, since providers read their own state in countTokens + if (typeof tokenCountModel.countTokens === 'function') { try { - return await countTokens(serializedPrompt) + return await tokenCountModel.countTokens(serializedPrompt) } catch { return estimateTokenCount(serializedPrompt) } diff --git a/packages/utils/tsconfig.json b/packages/utils/tsconfig.json index bf5a36ddb..54f5777b4 100644 --- a/packages/utils/tsconfig.json +++ b/packages/utils/tsconfig.json @@ -1,4 +1,5 @@ { "extends": "../../tsconfig.base.json", - "include": ["src/**/*"] + "include": ["src/**/*"], + "exclude": ["src/**/__tests__/**"] } diff --git a/skills/react-native-ai/references/apple-provider.md b/skills/react-native-ai/references/apple-provider.md index 72c6f0b12..c6e1630c7 100644 --- a/skills/react-native-ai/references/apple-provider.md +++ b/skills/react-native-ai/references/apple-provider.md @@ -57,7 +57,7 @@ the user in Settings. | Type | Method | Use Case | Documentation | | ------------- | ---------------------------- | --------------------------------------- | -------------------------------------------------------- | | Language | `apple.languageModel()` | Text generation, chat | https://www.react-native-ai.dev/docs/apple/generating | -| Embedding | `apple.textEmbeddingModel()` | RAG, similarity, prompt size estimation | https://www.react-native-ai.dev/docs/apple/embeddings | +| Embedding | `apple.embeddingModel()` | RAG, similarity, prompt size estimation | https://www.react-native-ai.dev/docs/apple/embeddings | | Transcription | `apple.transcriptionModel()` | Speech-to-text | https://www.react-native-ai.dev/docs/apple/transcription | | Speech | `apple.speechModel()` | Text-to-speech | https://www.react-native-ai.dev/docs/apple/speech | diff --git a/skills/react-native-ai/references/llama-provider.md b/skills/react-native-ai/references/llama-provider.md index 319480d48..1b6420f1f 100644 --- a/skills/react-native-ai/references/llama-provider.md +++ b/skills/react-native-ai/references/llama-provider.md @@ -89,12 +89,12 @@ const exists = await isModelDownloaded('owner/repo/model.gguf') ### 5. Model Types -| Type | Method | Notes | Documentation | -| --------- | ---------------------------- | --------------------------------------- | ------------------------------------------------------------------------------------ | -| Language | `llama.languageModel()` | Text generation, chat | https://www.react-native-ai.dev/docs/llama/generating | -| Embedding | `llama.textEmbeddingModel()` | RAG, similarity, prompt size estimation | https://www.react-native-ai.dev/docs/llama/embeddings | -| Rerank | `llama.rerankModel()` | Document ranking, RAG | https://www.react-native-ai.dev/docs/llama/reranking | -| Speech | `llama.speechModel()` | Requires `vocoderPath` in opts | https://www.react-native-ai.dev/docs/llama/model-management#creating-model-instances | +| Type | Method | Notes | Documentation | +| --------- | ------------------------ | --------------------------------------- | ------------------------------------------------------------------------------------ | +| Language | `llama.languageModel()` | Text generation, chat | https://www.react-native-ai.dev/docs/llama/generating | +| Embedding | `llama.embeddingModel()` | RAG, similarity, prompt size estimation | https://www.react-native-ai.dev/docs/llama/embeddings | +| Rerank | `llama.rerankingModel()` | Document ranking, RAG | https://www.react-native-ai.dev/docs/llama/reranking | +| Speech | `llama.speechModel()` | Requires `vocoderPath` in opts | https://www.react-native-ai.dev/docs/llama/model-management#creating-model-instances | ### 6. Lifecycle diff --git a/website/src/docs/apple/embeddings.md b/website/src/docs/apple/embeddings.md index d2d26387b..cd366e77a 100644 --- a/website/src/docs/apple/embeddings.md +++ b/website/src/docs/apple/embeddings.md @@ -29,7 +29,7 @@ import { embed } from 'ai' import { apple } from '@react-native-ai/apple' const { embedding } = await embed({ - model: apple.textEmbeddingModel(), + model: apple.embeddingModel(), value: 'Hello world', }) @@ -43,7 +43,7 @@ import { embedMany } from 'ai' import { apple } from '@react-native-ai/apple' const { embeddings } = await embedMany({ - model: apple.textEmbeddingModel(), + model: apple.embeddingModel(), values: ['Hello world', 'How are you?', 'Goodbye'], }) @@ -55,7 +55,7 @@ console.log(embeddings) The embeddings model supports multiple languages. You can specify the language using [ISO 639-1 codes](https://en.wikipedia.org/wiki/List_of_ISO_639_language_codes) or full names when creating the model: ```tsx -const model = apple.textEmbeddingModel({ language: 'fr' }) +const model = apple.embeddingModel({ language: 'fr' }) await embed({ model, @@ -73,7 +73,7 @@ For list of all supported languages, [check Apple documentation](https://develop Apple's NLContextualEmbedding requires downloading language-specific assets to the device. While the provider automatically prepares assets when needed, you can call `prepare()` ahead of time for better performance: ```tsx -const model = apple.textEmbeddingModel({ language: 'en' }) +const model = apple.embeddingModel({ language: 'en' }) // Call prepare() ahead of time to optimize first inference latency await model.prepare() diff --git a/website/src/docs/apple/generating.md b/website/src/docs/apple/generating.md index 7a99cdb12..4ac4bc919 100644 --- a/website/src/docs/apple/generating.md +++ b/website/src/docs/apple/generating.md @@ -66,7 +66,9 @@ for await (const delta of result.textStream) { } ``` -Streaming structured objects are not currently supported by this provider. +Structured output (`Output.object`) also works with `streamText`, but Apple +returns partial objects that are not prefixes of the final JSON, so the provider +sends the complete object once generation finishes instead of streaming it. ## Structured Output @@ -337,6 +339,8 @@ Important Apple-specific behavior: - Tool calls are provider-executed, so AI SDK step callbacks such as `stopWhen`, `onStepStart`, and `onStepEnd` do not run for native Apple tool execution. +- Tool calls and their results are reported with Apple's tool call IDs, in + `generateText` content and as stream parts in `streamText`. - You can update the registered tools on a model instance with `model.updateTools(...)` before generation. @@ -370,19 +374,32 @@ const info = await apple.getModelInfo() console.log(info.contextSize) ``` +Each response reports token usage in `result.usage`, which AI SDK features +such as `prepareStep` and telemetry read: + +- iOS 27 reports the usage measured by Foundation Models, for both the + on-device model and Private Cloud Compute. +- iOS 26.4 counts the transcript with the on-device model's tokenizer. +- Otherwise usage is `undefined`. + For reusable history policies such as summarization, rolling windows, and -completed tool-call pruning, use `@react-native-ai/utils`. +completed tool-call pruning, use `@react-native-ai/utils`. Its summarizer uses +`countTokens` from the model you pass it. -Use low-level token counting as an estimate, not a fit guarantee: +Count tokens before sending a request as an estimate, not a fit guarantee: ```typescript -import { AppleFoundationModels } from '@react-native-ai/apple' - -const tokenCount = await AppleFoundationModels.countTokens( +const model = apple() +const tokenCount = await model.countTokens( 'Summarize this text in three bullet points.' ) ``` +Token counting requires iOS 26.4 and an available on-device model. Private +Cloud Compute has no token counting API, so `countTokens` on a Private Cloud +Compute model still uses the on-device model's tokenizer, whose count may not +match the server model. + The full request also includes instructions, tool definitions, schema text, attachments, and generated output budget. If Apple reports a context overflow, trim or summarize and retry. @@ -464,7 +481,7 @@ import { AppleFoundationModels } from '@react-native-ai/apple' const info = await AppleFoundationModels.getModelInfo('en-US', 'system') const tokens = await AppleFoundationModels.countTokens('A short prompt') -const result = await AppleFoundationModels.generateText( +const { content, usage } = await AppleFoundationModels.generateText( [{ role: 'user', content: 'Hello' }], { temperature: 0.7, maxTokens: 100 } ) diff --git a/website/src/docs/llama/embeddings.md b/website/src/docs/llama/embeddings.md index c88ee084f..30aa4836e 100644 --- a/website/src/docs/llama/embeddings.md +++ b/website/src/docs/llama/embeddings.md @@ -17,7 +17,7 @@ import { embed } from 'ai' const modelPath = await downloadModel('owner/repo/embedding-model.gguf') // Create embedding model with the path -const model = llama.textEmbeddingModel(modelPath) +const model = llama.embeddingModel(modelPath) await model.prepare() @@ -40,7 +40,7 @@ import { embedMany } from 'ai' const modelPath = await downloadModel('owner/repo/embedding-model.gguf') -const model = llama.textEmbeddingModel(modelPath) +const model = llama.embeddingModel(modelPath) await model.prepare() const { embeddings } = await embedMany({ @@ -61,7 +61,7 @@ console.log(embeddings[0].length) // Embedding dimension (e.g., 384, 768) Configure the embedding model with specific options: ```typescript -const model = llama.textEmbeddingModel(modelPath, { +const model = llama.embeddingModel(modelPath, { normalize: -1, // Normalization mode (default: -1) contextParams: { n_ctx: 2048, // Context size (default: 2048) diff --git a/website/src/docs/llama/getting-started.mdx b/website/src/docs/llama/getting-started.mdx index fc21f7025..0c844c413 100644 --- a/website/src/docs/llama/getting-started.mdx +++ b/website/src/docs/llama/getting-started.mdx @@ -50,11 +50,11 @@ For all other installation tips and tricks, refer to the [llama.rn Expo document The Llama provider supports multiple model types: -| Model Type | Method | Use Case | -| --------------- | ---------------------------- | ----------------------------------- | -| Language Model | `llama.languageModel()` | Text generation, chat, reasoning | -| Embedding Model | `llama.textEmbeddingModel()` | Text embeddings for RAG, similarity | -| Speech Model | `llama.speechModel()` | Text-to-speech with vocoder | +| Model Type | Method | Use Case | +| --------------- | ------------------------ | ----------------------------------- | +| Language Model | `llama.languageModel()` | Text generation, chat, reasoning | +| Embedding Model | `llama.embeddingModel()` | Text embeddings for RAG, similarity | +| Speech Model | `llama.speechModel()` | Text-to-speech with vocoder | ## Basic Usage diff --git a/website/src/docs/llama/model-management.md b/website/src/docs/llama/model-management.md index c8224a9dd..91ffd5e39 100644 --- a/website/src/docs/llama/model-management.md +++ b/website/src/docs/llama/model-management.md @@ -72,7 +72,7 @@ const modelPath = await downloadModel('bartowski/Qwen_Qwen3.5-2B-GGUF/Qwen_Qwen3 const languageModel = llama.languageModel(modelPath) // Embedding model -const embeddingModel = llama.textEmbeddingModel(modelPath) +const embeddingModel = llama.embeddingModel(modelPath) // Speech model (requires vocoder) const speechModel = llama.speechModel(modelPath, { @@ -162,7 +162,7 @@ Creates a language model instance. - `n_ctx`: Context size (default: 2048, or 4096 for multimodal) - `n_gpu_layers`: Number of GPU layers (default: 99) -### `llama.textEmbeddingModel(modelPath, options?)` +### `llama.embeddingModel(modelPath, options?)` Creates an embedding model instance. diff --git a/website/src/docs/llama/reranking.md b/website/src/docs/llama/reranking.md index 9d03e95ff..a903be2e5 100644 --- a/website/src/docs/llama/reranking.md +++ b/website/src/docs/llama/reranking.md @@ -19,7 +19,7 @@ const modelPath = await downloadModel( ) // Create rerank model with the path -const model = llama.rerankModel(modelPath) +const model = llama.rerankingModel(modelPath) await model.prepare() @@ -62,7 +62,7 @@ const { ranking } = await rerank({ Configure the rerank model with specific options: ```typescript -const model = llama.rerankModel(modelPath, { +const model = llama.rerankingModel(modelPath, { normalize: 1, // Score normalization (default: from model config) contextParams: { n_ctx: 2048, // Context size (default: 2048)