From aa276ee1492bf16549d32a930c44c5122f5aff3f Mon Sep 17 00:00:00 2001 From: Mike Grabowski Date: Sat, 24 Oct 2026 23:43:17 +0400 Subject: [PATCH 1/8] refactor(apple): export the native module as AppleFoundationModels AppleFoundationModels.ts was added in #212 to reject countTokens on native builds without it. #248 removed that guard, so the wrapper only re-declared the native methods. Export the TurboModule directly, like the other native modules in the package and as 0.12.0 did. Also remove src/stream.ts, an unused stream helper still typed with LanguageModelV2StreamPart. Co-Authored-By: Claude Opus 5.5 --- .../apple-llm/src/AppleFoundationModels.ts | 34 ------- packages/apple-llm/src/index.ts | 2 +- packages/apple-llm/src/stream.ts | 98 ------------------- 3 files changed, 1 insertion(+), 133 deletions(-) delete mode 100644 packages/apple-llm/src/AppleFoundationModels.ts delete mode 100644 packages/apple-llm/src/stream.ts diff --git a/packages/apple-llm/src/AppleFoundationModels.ts b/packages/apple-llm/src/AppleFoundationModels.ts deleted file mode 100644 index 74ff7873b..000000000 --- a/packages/apple-llm/src/AppleFoundationModels.ts +++ /dev/null @@ -1,34 +0,0 @@ -import type { AppleLanguageModelId } from './ai-sdk' -import NativeAppleLLM, { - type AppleGenerationOptions, - type AppleMessage, - type Spec, -} from './NativeAppleLLM' - -type AppleFoundationModelsSpec = Omit & { - getModelInfo( - locale?: string, - model?: AppleLanguageModelId - ): ReturnType -} - -const AppleFoundationModels: AppleFoundationModelsSpec = { - isAvailable: () => NativeAppleLLM.isAvailable(), - getModelInfo: (locale?: string, model?: AppleLanguageModelId) => - NativeAppleLLM.getModelInfo(locale, model), - countTokens: (text) => NativeAppleLLM.countTokens(text), - generateImages: (options) => NativeAppleLLM.generateImages(options), - generateText: (messages: AppleMessage[], options: AppleGenerationOptions) => - NativeAppleLLM.generateText(messages, options), - generateStream: ( - streamId, - messages: AppleMessage[], - options: AppleGenerationOptions - ) => NativeAppleLLM.generateStream(streamId, messages, options), - cancelStream: (streamId) => NativeAppleLLM.cancelStream(streamId), - onStreamUpdate: NativeAppleLLM.onStreamUpdate, - onStreamComplete: NativeAppleLLM.onStreamComplete, - onStreamError: NativeAppleLLM.onStreamError, -} - -export default AppleFoundationModels diff --git a/packages/apple-llm/src/index.ts b/packages/apple-llm/src/index.ts index e0fbf1bb8..cc88e3eb2 100644 --- a/packages/apple-llm/src/index.ts +++ b/packages/apple-llm/src/index.ts @@ -14,10 +14,10 @@ export type { AppleToolDefinitionSet, } from './ai-sdk' export { apple, createAppleProvider } from './ai-sdk' -export { default as AppleFoundationModels } from './AppleFoundationModels' export type { AppleLLMError, AppleLLMErrorCode } from './errors' export { AppleLLMErrorCodes } from './errors' export { default as AppleEmbeddings } from './NativeAppleEmbeddings' +export { default as AppleFoundationModels } from './NativeAppleLLM' export { default as AppleSpeech, VoiceInfo } from './NativeAppleSpeech' export { default as AppleTranscription } from './NativeAppleTranscription' export { default as AppleUtils } from './NativeAppleUtils' diff --git a/packages/apple-llm/src/stream.ts b/packages/apple-llm/src/stream.ts deleted file mode 100644 index c3847aa65..000000000 --- a/packages/apple-llm/src/stream.ts +++ /dev/null @@ -1,98 +0,0 @@ -import type { LanguageModelV2StreamPart } from '@ai-sdk/provider' -import { generateId } from '@ai-sdk/provider-utils' - -import { createAppleLLMError, isAppleLLMErrorCode } from './errors' -import NativeAppleLLMSpec, { - type AppleGenerationOptions, - type AppleMessage, -} from './NativeAppleLLM' - -export function generateStream( - messages: AppleMessage[], - options: AppleGenerationOptions = {} -): ReadableStream { - if (typeof ReadableStream === 'undefined') { - throw new Error( - `ReadableStream is not available in this environment. Please load a polyfill, such as web-streams-polyfill.` - ) - } - - let streamId: string | null = null - let listeners: { remove(): void }[] = [] - - const cleanup = () => { - listeners.forEach((listener) => listener.remove()) - listeners = [] - } - - const stream = new ReadableStream({ - async start(controller) { - try { - streamId = generateId() - - controller.enqueue({ - type: 'text-start', - id: streamId, - }) - - const updateListener = NativeAppleLLMSpec.onStreamUpdate((data) => { - if (data.streamId === streamId) { - controller.enqueue({ - type: 'text-delta', - delta: data.content, - id: data.streamId, - }) - } - }) - - const completeListener = NativeAppleLLMSpec.onStreamComplete((data) => { - if (data.streamId === streamId) { - controller.enqueue({ - type: 'text-end', - id: streamId, - }) - controller.enqueue({ - type: 'finish', - finishReason: 'stop', - usage: { - inputTokens: 0, - outputTokens: 0, - totalTokens: 0, - }, - }) - cleanup() - controller.close() - } - }) - - const errorListener = NativeAppleLLMSpec.onStreamError((data) => { - if (data.streamId === streamId) { - controller.enqueue({ - type: 'error', - error: isAppleLLMErrorCode(data.code) - ? createAppleLLMError(data.error, data.code) - : new Error(data.error), - }) - cleanup() - controller.close() - } - }) - - listeners = [updateListener, completeListener, errorListener] - - NativeAppleLLMSpec.generateStream(streamId, messages, options) - } catch (error) { - cleanup() - controller.error(new Error(`Apple LLM stream failed: ${error}`)) - } - }, - cancel() { - cleanup() - if (streamId) { - NativeAppleLLMSpec.cancelStream(streamId) - } - }, - }) - - return stream -} From dfc20a1b6c48a92b08bff26344ae7949824dff4c Mon Sep 17 00:00:00 2001 From: Mike Grabowski Date: Sat, 24 Oct 2026 23:45:50 +0400 Subject: [PATCH 2/8] feat(apple): report token usage from foundation models Responses and streams reported 0 input and 0 output tokens, so AI SDK usage, prepareStep thresholds and telemetry always saw zero. - iOS 27 passes through the usage Foundation Models measures (input, cached input, output and reasoning tokens), for the on-device model and Private Cloud Compute - iOS 26.4 counts the transcript before and after the response with the on-device model's tokenizer - otherwise usage is undefined instead of 0 Native generateText now resolves { content, usage } and onStreamComplete carries usage, which changes the shape returned by AppleFoundationModels.generateText. Co-Authored-By: Claude Opus 5.5 --- packages/apple-llm/ios/AppleLLM.mm | 8 +- packages/apple-llm/ios/AppleLLMImpl.swift | 106 +++++++++++++++--- packages/apple-llm/src/NativeAppleLLM.ts | 18 ++- .../apple-llm/src/__tests__/ai-sdk.test.ts | 98 +++++++++++++--- packages/apple-llm/src/ai-sdk.ts | 60 +++++----- website/src/docs/apple/generating.md | 10 +- 6 files changed, 232 insertions(+), 68 deletions(-) diff --git a/packages/apple-llm/ios/AppleLLM.mm b/packages/apple-llm/ios/AppleLLM.mm index 55ef1d9f9..74d34c034 100644 --- a/packages/apple-llm/ios/AppleLLM.mm +++ b/packages/apple-llm/ios/AppleLLM.mm @@ -208,8 +208,12 @@ - (void)generateStream:(nonnull NSString *)streamId messages:(nonnull NSArray *) } [self emitOnStreamUpdate:payload]; } - onComplete:^(NSString *streamId) { - [self emitOnStreamComplete:@{@"streamId": streamId}]; + onComplete:^(NSString *streamId, NSDictionary *usage) { + NSMutableDictionary *payload = [@{@"streamId": streamId} mutableCopy]; + if (usage != nil) { + payload[@"usage"] = usage; + } + [self emitOnStreamComplete:payload]; } onError:^(NSString *streamId, NSString *code, NSString *error) { NSMutableDictionary *payload = [@{@"streamId": streamId, @"error": error} mutableCopy]; diff --git a/packages/apple-llm/ios/AppleLLMImpl.swift b/packages/apple-llm/ios/AppleLLMImpl.swift index 55ebee62c..c77ffc22a 100644 --- a/packages/apple-llm/ios/AppleLLMImpl.swift +++ b/packages/apple-llm/ios/AppleLLMImpl.swift @@ -254,7 +254,7 @@ public class AppleLLMImpl: NSObject { messages: [[String: Any]], options: [String: Any], onUpdate: @escaping (String, String, [[String: Any]]?) -> Void, - onComplete: @escaping (String) -> Void, + onComplete: @escaping (String, [String: Any]?) -> Void, onError: @escaping (String, String, String) -> Void, toolInvoker: @escaping ToolInvoker ) { @@ -275,7 +275,7 @@ public class AppleLLMImpl: NSObject { let generationSchema = try self.createGenerationSchema(from: options) do { - try await self.streamResponse( + let usage = try await self.streamResponse( with: session, userPrompt: userPrompt, promptAttachments: promptAttachments, @@ -287,7 +287,7 @@ public class AppleLLMImpl: NSObject { } if !Task.isCancelled { - onComplete(streamId) + onComplete(streamId, usage) } } catch { if Task.isCancelled { @@ -525,7 +525,7 @@ public class AppleLLMImpl: NSObject { generationSchema: GenerationSchema?, generationOptions: GenerationOptions, providerOptions: [String: Any] - ) async throws -> [[String: Any]] { + ) async throws -> [String: Any] { #if compiler(>=6.4) if #available(iOS 27, *) { let contextOptions = try createContextOptions( @@ -548,7 +548,7 @@ public class AppleLLMImpl: NSObject { attachment } } - return response.toModelMessages() + return Self.generationResult(content: response.toModelMessages(), usage: Self.usageDictionary(response.usage)) } let response = try await session.respond( @@ -560,7 +560,7 @@ public class AppleLLMImpl: NSObject { attachment } } - return response.toModelMessages() + return Self.generationResult(content: response.toModelMessages(), usage: Self.usageDictionary(response.usage)) } #endif @@ -569,6 +569,8 @@ public class AppleLLMImpl: NSObject { throw AppleLLMError.unsupportedOS } + let transcriptBefore = session.transcript + let content: [[String: Any]] if let generationSchema { let response = try await session.respond( to: userPrompt, @@ -576,11 +578,71 @@ public class AppleLLMImpl: NSObject { includeSchemaInPrompt: true, options: generationOptions ) - return response.toModelMessages() + content = response.toModelMessages() + } else { + let response = try await session.respond(to: userPrompt, options: generationOptions) + content = response.toModelMessages() + } + + let usage = await countUsage( + session: session, + transcriptBefore: transcriptBefore, + userPrompt: userPrompt, + providerOptions: providerOptions + ) + return Self.generationResult(content: content, usage: usage) + } + + private static func generationResult(content: [[String: Any]], usage: [String: Any]?) -> [String: Any] { + var result: [String: Any] = ["content": content] + if let usage { + result["usage"] = usage + } + return result + } + +#if compiler(>=6.4) + @available(iOS 27, *) + private static func usageDictionary(_ usage: LanguageModelSession.Usage) -> [String: Any] { + return [ + "inputTokens": usage.input.totalTokenCount, + "cachedInputTokens": usage.input.cachedTokenCount, + "outputTokens": usage.output.totalTokenCount, + "reasoningTokens": usage.output.reasoningTokenCount, + ] + } +#endif + + /// iOS 26 responses report no usage, so the system model's tokenizer (iOS 26.4) + /// counts the transcript before and after the response. Private Cloud Compute + /// has no tokenizer, so its usage stays unknown + @available(iOS 26, *) + private func countUsage( + session: LanguageModelSession, + transcriptBefore: Transcript, + userPrompt: String, + providerOptions: [String: Any] + ) async -> [String: Any]? { +#if compiler(>=6.3) + guard #available(iOS 26.4, *), (providerOptions["model"] as? String ?? "system") == "system" else { + return nil } - let response = try await session.respond(to: userPrompt, options: generationOptions) - return response.toModelMessages() + let model = SystemLanguageModel.default + guard let historyTokens = try? await model.tokenCount(for: transcriptBefore), + let promptTokens = try? await model.tokenCount(for: userPrompt), + let totalTokens = try? await model.tokenCount(for: session.transcript) else { + return nil + } + + let inputTokens = historyTokens + promptTokens + return [ + "inputTokens": inputTokens, + "outputTokens": max(totalTokens - inputTokens, 0), + ] +#else + return nil +#endif } @available(iOS 26, *) @@ -592,7 +654,7 @@ public class AppleLLMImpl: NSObject { generationOptions: GenerationOptions, providerOptions: [String: Any], onUpdate: @escaping (String, [[String: Any]]?) -> Void - ) async throws { + ) async throws -> [String: Any]? { #if compiler(>=6.4) if #available(iOS 27, *) { let contextOptions = try createContextOptions( @@ -604,6 +666,7 @@ public class AppleLLMImpl: NSObject { Self.removeTemporaryFiles(preparedAttachments.temporaryFileURLs) } + var usage: LanguageModelSession.Usage? if let generationSchema { let responseStream = session.streamResponse( schema: generationSchema, @@ -616,9 +679,10 @@ public class AppleLLMImpl: NSObject { } } for try await chunk in responseStream { + usage = chunk.usage onUpdate(String(describing: chunk.content), nil) } - return + return usage.map(Self.usageDictionary) } let responseStream = session.streamResponse( @@ -633,12 +697,13 @@ public class AppleLLMImpl: NSObject { // Reasoning and tool activity are only sent when they change var previousEntries: NSArray = [] for try await chunk in responseStream { + usage = chunk.usage let entries = AppleLLMTranscriptSerializer.serialize(chunk.transcriptEntries, includeResponses: false) let entriesChanged = !(entries as NSArray).isEqual(previousEntries) previousEntries = entries as NSArray onUpdate(chunk.content, entriesChanged ? entries : nil) } - return + return usage.map(Self.usageDictionary) } #endif @@ -647,6 +712,7 @@ public class AppleLLMImpl: NSObject { throw AppleLLMError.unsupportedOS } + let transcriptBefore = session.transcript if let generationSchema { let responseStream = session.streamResponse( to: userPrompt, @@ -657,13 +723,19 @@ public class AppleLLMImpl: NSObject { for try await chunk in responseStream { onUpdate(String(describing: chunk.content), nil) } - return + } else { + let responseStream = session.streamResponse(to: userPrompt, options: generationOptions) + for try await chunk in responseStream { + onUpdate(chunk.content, nil) + } } - let responseStream = session.streamResponse(to: userPrompt, options: generationOptions) - for try await chunk in responseStream { - onUpdate(chunk.content, nil) - } + return await countUsage( + session: session, + transcriptBefore: transcriptBefore, + userPrompt: userPrompt, + providerOptions: providerOptions + ) } #if compiler(>=6.4) diff --git a/packages/apple-llm/src/NativeAppleLLM.ts b/packages/apple-llm/src/NativeAppleLLM.ts index dbd477731..5dbf6002b 100644 --- a/packages/apple-llm/src/NativeAppleLLM.ts +++ b/packages/apple-llm/src/NativeAppleLLM.ts @@ -27,8 +27,19 @@ export type StreamUpdateEvent = { entries?: CodegenTypes.UnsafeObject[] } +// Token counts from Foundation Models (iOS 27) or the system model's +// tokenizer (iOS 26.4). Left out when they are unknown +export type AppleUsage = { + inputTokens: number + cachedInputTokens?: number + outputTokens: number + reasoningTokens?: number +} + export type StreamCompleteEvent = { streamId: string + // AppleUsage + usage?: CodegenTypes.UnsafeObject } export type StreamErrorEvent = { @@ -48,8 +59,8 @@ export interface Spec extends TurboModule { generateText( messages: AppleMessage[], options: AppleGenerationOptions - ): Promise< - ( + ): Promise<{ + content: ( | { type: 'text'; text: string } | { type: 'reasoning' @@ -70,7 +81,8 @@ export interface Spec extends TurboModule { output: string } )[] - > + usage?: AppleUsage + }> generateStream( streamId: string, messages: AppleMessage[], diff --git a/packages/apple-llm/src/__tests__/ai-sdk.test.ts b/packages/apple-llm/src/__tests__/ai-sdk.test.ts index 1d6ab19ce..29c8b56da 100644 --- a/packages/apple-llm/src/__tests__/ai-sdk.test.ts +++ b/packages/apple-llm/src/__tests__/ai-sdk.test.ts @@ -21,7 +21,9 @@ const generateText = mock( async ( _messages: unknown, _options: Record - ): Promise[]> => [{ type: 'text', text: 'ok' }] + ): Promise> => ({ + content: [{ type: 'text', text: 'ok' }], + }) ) const generateStream = mock( (_streamId: string, _messages: unknown, _options: unknown) => {} @@ -354,22 +356,24 @@ async function readAll(stream: ReadableStream) { describe('apple generated content', () => { test('returns reasoning and tool parts with their ids', async () => { - generateText.mockImplementationOnce(async () => [ - { type: 'reasoning', id: 'r1', text: 'thinking', signature: 'c2ln' }, - { - type: 'tool-call', - toolCallId: 'call-1', - toolName: 'weather', - input: '{"city":"Paris"}', - }, - { - type: 'tool-result', - toolCallId: 'call-1', - toolName: 'weather', - output: 'sunny', - }, - { type: 'text', text: 'It is sunny' }, - ]) + generateText.mockImplementationOnce(async () => ({ + content: [ + { type: 'reasoning', id: 'r1', text: 'thinking', signature: 'c2ln' }, + { + type: 'tool-call', + toolCallId: 'call-1', + toolName: 'weather', + input: '{"city":"Paris"}', + }, + { + type: 'tool-result', + toolCallId: 'call-1', + toolName: 'weather', + output: 'sunny', + }, + { type: 'text', text: 'It is sunny' }, + ], + })) const result = await apple().doGenerate(createCallOptions()) @@ -469,3 +473,63 @@ describe('apple generated content', () => { ]) }) }) + +describe('apple usage', () => { + test('maps native token usage', async () => { + generateText.mockImplementationOnce(async () => ({ + content: [{ type: 'text', text: 'ok' }], + usage: { + inputTokens: 120, + cachedInputTokens: 20, + outputTokens: 30, + reasoningTokens: 10, + }, + })) + + const result = await apple().doGenerate(createCallOptions()) + + expect(result.usage).toEqual({ + inputTokens: { + total: 120, + noCache: 100, + cacheRead: 20, + cacheWrite: undefined, + }, + outputTokens: { total: 30, text: 20, reasoning: 10 }, + }) + }) + + test('reports unknown usage as undefined instead of zero', async () => { + const result = await apple().doGenerate(createCallOptions()) + + expect(result.usage).toEqual({ + inputTokens: { + total: undefined, + noCache: undefined, + cacheRead: undefined, + cacheWrite: undefined, + }, + outputTokens: { total: undefined, text: undefined, reasoning: undefined }, + }) + }) + + test('reports stream usage on finish', async () => { + const { stream } = await apple().doStream(createCallOptions()) + const parts = readAll(stream) + const streamId = generateStream.mock.calls.at(-1)![0] + + onStreamUpdate.emit({ streamId, content: 'ok' }) + onStreamComplete.emit({ + streamId, + usage: { inputTokens: 12, outputTokens: 3 }, + }) + + const finish = (await parts).find((part) => part.type === 'finish') + expect(finish).toMatchObject({ + usage: { + inputTokens: { total: 12, noCache: undefined, cacheRead: undefined }, + outputTokens: { total: 3, text: undefined, reasoning: undefined }, + }, + }) + }) +}) diff --git a/packages/apple-llm/src/ai-sdk.ts b/packages/apple-llm/src/ai-sdk.ts index 4ba9540ef..0e7ffd637 100644 --- a/packages/apple-llm/src/ai-sdk.ts +++ b/packages/apple-llm/src/ai-sdk.ts @@ -12,6 +12,7 @@ import type { LanguageModelV4ProviderTool, LanguageModelV4StreamPart, LanguageModelV4ToolResultOutput, + LanguageModelV4Usage, SharedV4Warning, SpeechModelV4, SpeechModelV4CallOptions, @@ -28,7 +29,10 @@ import { import { createAppleLLMError, isAppleLLMErrorCode } from './errors' import NativeAppleEmbeddings from './NativeAppleEmbeddings' -import NativeAppleLLM, { type AppleMessage } from './NativeAppleLLM' +import NativeAppleLLM, { + type AppleMessage, + type AppleUsage, +} from './NativeAppleLLM' import NativeAppleSpeech from './NativeAppleSpeech' import NativeAppleTranscription from './NativeAppleTranscription' import NativeAppleUtils from './NativeAppleUtils' @@ -445,6 +449,30 @@ function prepareToolChoice( } } +function toUsage(usage: AppleUsage | undefined): LanguageModelV4Usage { + const cachedInputTokens = usage?.cachedInputTokens + const reasoningTokens = usage?.reasoningTokens + return { + inputTokens: { + total: usage?.inputTokens, + noCache: + usage && cachedInputTokens !== undefined + ? usage.inputTokens - cachedInputTokens + : undefined, + cacheRead: cachedInputTokens, + cacheWrite: undefined, + }, + outputTokens: { + total: usage?.outputTokens, + text: + usage && reasoningTokens !== undefined + ? usage.outputTokens - reasoningTokens + : undefined, + reasoning: reasoningTokens, + }, + } +} + function appendMessageContent(content: string, addition: string) { if (!addition) { return content @@ -838,7 +866,7 @@ class AppleLLMChatLanguageModel implements AppleLanguageModel { }) return { - content: response.map((part) => { + content: response.content.map((part) => { switch (part.type) { case 'text': return part @@ -869,19 +897,7 @@ class AppleLLMChatLanguageModel implements AppleLanguageModel { } }), finishReason: { unified: 'stop' as const, raw: 'stop' }, - usage: { - inputTokens: { - total: 0, - noCache: undefined, - cacheRead: undefined, - cacheWrite: undefined, - }, - outputTokens: { - total: 0, - text: undefined, - reasoning: undefined, - }, - }, + usage: toUsage(response.usage), warnings, } } finally { @@ -1058,19 +1074,7 @@ class AppleLLMChatLanguageModel implements AppleLanguageModel { controller.enqueue({ type: 'finish', finishReason: { unified: 'stop' as const, raw: 'stop' }, - usage: { - inputTokens: { - total: 0, - noCache: undefined, - cacheRead: undefined, - cacheWrite: undefined, - }, - outputTokens: { - total: 0, - text: undefined, - reasoning: undefined, - }, - }, + usage: toUsage(data.usage as AppleUsage | undefined), }) cleanup() controller.close() diff --git a/website/src/docs/apple/generating.md b/website/src/docs/apple/generating.md index 7a99cdb12..55c88e9b8 100644 --- a/website/src/docs/apple/generating.md +++ b/website/src/docs/apple/generating.md @@ -370,6 +370,14 @@ const info = await apple.getModelInfo() console.log(info.contextSize) ``` +Each response reports token usage in `result.usage`, which AI SDK features +such as `prepareStep` and telemetry read: + +- iOS 27 reports the usage measured by Foundation Models, for both the + on-device model and Private Cloud Compute. +- iOS 26.4 counts the transcript with the on-device model's tokenizer. +- Otherwise usage is `undefined`. + For reusable history policies such as summarization, rolling windows, and completed tool-call pruning, use `@react-native-ai/utils`. @@ -464,7 +472,7 @@ import { AppleFoundationModels } from '@react-native-ai/apple' const info = await AppleFoundationModels.getModelInfo('en-US', 'system') const tokens = await AppleFoundationModels.countTokens('A short prompt') -const result = await AppleFoundationModels.generateText( +const { content, usage } = await AppleFoundationModels.generateText( [{ role: 'user', content: 'Hello' }], { temperature: 0.7, maxTokens: 100 } ) From 803cc7f50e99269d5652f01c4427115e635ffd67 Mon Sep 17 00:00:00 2001 From: Mike Grabowski Date: Sat, 24 Oct 2026 23:46:21 +0400 Subject: [PATCH 3/8] fix(core-ai, llama): report unknown token usage as undefined core-ai always reported 0 tokens and llama fell back to 0 when llama.rn returned no timings. AI SDK treats 0 as a measured count, so totals looked real. Report undefined instead, as adk does. Co-Authored-By: Claude Opus 5.5 --- packages/core-ai/src/ai-sdk.ts | 4 ++-- packages/llama/src/ai-sdk.ts | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/core-ai/src/ai-sdk.ts b/packages/core-ai/src/ai-sdk.ts index d6ffc9cf7..a3823c2e7 100644 --- a/packages/core-ai/src/ai-sdk.ts +++ b/packages/core-ai/src/ai-sdk.ts @@ -564,13 +564,13 @@ function toFinishReason( function emptyUsage() { return { inputTokens: { - total: 0, + total: undefined, noCache: undefined, cacheRead: undefined, cacheWrite: undefined, }, outputTokens: { - total: 0, + total: undefined, text: undefined, reasoning: undefined, }, diff --git a/packages/llama/src/ai-sdk.ts b/packages/llama/src/ai-sdk.ts index 10a3ec1d0..04582920a 100644 --- a/packages/llama/src/ai-sdk.ts +++ b/packages/llama/src/ai-sdk.ts @@ -296,13 +296,13 @@ export class LlamaLanguageModel implements LanguageModelV4 { : convertFinishReason(response), usage: { inputTokens: { - total: response.timings?.prompt_n || 0, + total: response.timings?.prompt_n, noCache: undefined, cacheRead: undefined, cacheWrite: undefined, }, outputTokens: { - total: response.timings?.predicted_n || 0, + total: response.timings?.predicted_n, text: undefined, reasoning: undefined, }, @@ -373,13 +373,13 @@ export class LlamaLanguageModel implements LanguageModelV4 { : convertFinishReason(result), usage: { inputTokens: { - total: result.timings?.prompt_n || 0, + total: result.timings?.prompt_n, noCache: undefined, cacheRead: undefined, cacheWrite: undefined, }, outputTokens: { - total: result.timings?.predicted_n || 0, + total: result.timings?.predicted_n, text: undefined, reasoning: undefined, }, From 10b9ebc36f51c7301e35e1975a41f743ec3fe87f Mon Sep 17 00:00:00 2001 From: Mike Grabowski Date: Sat, 24 Oct 2026 23:47:21 +0400 Subject: [PATCH 4/8] feat(apple, llama): count tokens through language models @react-native-ai/utils summarizes history by calling countTokens on the model it is given, but no provider defined it, so it always fell back to four characters per token. Apple language models count with the on-device model's tokenizer (iOS 26.4+) and llama.rn models with their own tokenizer. AppleFoundationModels.countTokens stays available. Co-Authored-By: Claude Opus 5.5 --- packages/apple-llm/src/__tests__/ai-sdk.test.ts | 9 +++++++++ packages/apple-llm/src/ai-sdk.ts | 9 +++++++++ packages/llama/src/ai-sdk.ts | 9 +++++++++ packages/utils/README.md | 4 ++++ website/src/docs/apple/generating.md | 13 ++++++++----- 5 files changed, 39 insertions(+), 5 deletions(-) diff --git a/packages/apple-llm/src/__tests__/ai-sdk.test.ts b/packages/apple-llm/src/__tests__/ai-sdk.test.ts index 29c8b56da..2793aef40 100644 --- a/packages/apple-llm/src/__tests__/ai-sdk.test.ts +++ b/packages/apple-llm/src/__tests__/ai-sdk.test.ts @@ -25,6 +25,7 @@ const generateText = mock( content: [{ type: 'text', text: 'ok' }], }) ) +const countTokens = mock(async (text: string) => text.length) const generateStream = mock( (_streamId: string, _messages: unknown, _options: unknown) => {} ) @@ -39,6 +40,7 @@ mock.module('react-native', () => ({ ? { generateText, generateStream, + countTokens, cancelStream: () => {}, onStreamUpdate, onStreamComplete, @@ -533,3 +535,10 @@ describe('apple usage', () => { }) }) }) + +describe('apple token counting', () => { + test('counts tokens through the language model', async () => { + expect(await apple().countTokens('hello')).toBe(5) + expect(countTokens).toHaveBeenCalledWith('hello') + }) +}) diff --git a/packages/apple-llm/src/ai-sdk.ts b/packages/apple-llm/src/ai-sdk.ts index 0e7ffd637..fcf5a5232 100644 --- a/packages/apple-llm/src/ai-sdk.ts +++ b/packages/apple-llm/src/ai-sdk.ts @@ -43,6 +43,11 @@ export type AppleToolDefinitionSet = Record export interface AppleLanguageModel extends LanguageModelV4 { prepare: () => Promise updateTools: (tools: AppleToolDefinitionSet) => void + /** + * Counts tokens in text with the on-device model's tokenizer (iOS 26.4+). + * Private Cloud Compute has no tokenizer, so this approximates its count + */ + countTokens: (text: string) => Promise } export type AppleLanguageModelId = 'system' | 'private-cloud-compute' export type AppleBuiltInTool = 'ocr' | 'barcode' @@ -759,6 +764,10 @@ class AppleLLMChatLanguageModel implements AppleLanguageModel { async prepare(): Promise {} + countTokens(text: string): Promise { + return NativeAppleLLM.countTokens(text) + } + private prepareMessages(messages: LanguageModelV4Prompt): AppleMessage[] { return messages.map((message, index): AppleMessage => { switch (message.role) { diff --git a/packages/llama/src/ai-sdk.ts b/packages/llama/src/ai-sdk.ts index 04582920a..da85d7725 100644 --- a/packages/llama/src/ai-sdk.ts +++ b/packages/llama/src/ai-sdk.ts @@ -193,6 +193,15 @@ export class LlamaLanguageModel implements LanguageModelV4 { } } + /** + * Count tokens in text with the model's tokenizer + */ + async countTokens(text: string): Promise { + const context = this.context ?? (await this.prepare()) + const { tokens } = await context.tokenize(text) + return tokens.length + } + /** * Get the underlying LlamaContext (for advanced usage) */ diff --git a/packages/utils/README.md b/packages/utils/README.md index 2ac6a6051..9b73b23c0 100644 --- a/packages/utils/README.md +++ b/packages/utils/README.md @@ -14,3 +14,7 @@ const modelWithHistory = wrapLanguageModelWithHistory(model, { dropCompletedToolCalls: true, }) ``` + +`summarizeHistory` counts prompt tokens with the summarizer model's +`countTokens` method when it has one (Apple and llama.rn language models do). +Other models fall back to an estimate of four characters per token. diff --git a/website/src/docs/apple/generating.md b/website/src/docs/apple/generating.md index 55c88e9b8..c6e427f56 100644 --- a/website/src/docs/apple/generating.md +++ b/website/src/docs/apple/generating.md @@ -379,18 +379,21 @@ such as `prepareStep` and telemetry read: - Otherwise usage is `undefined`. For reusable history policies such as summarization, rolling windows, and -completed tool-call pruning, use `@react-native-ai/utils`. +completed tool-call pruning, use `@react-native-ai/utils`. Its summarizer uses +`countTokens` from the model you pass it. -Use low-level token counting as an estimate, not a fit guarantee: +Count tokens before sending a request as an estimate, not a fit guarantee: ```typescript -import { AppleFoundationModels } from '@react-native-ai/apple' - -const tokenCount = await AppleFoundationModels.countTokens( +const model = apple() +const tokenCount = await model.countTokens( 'Summarize this text in three bullet points.' ) ``` +Token counting requires iOS 26.4 and uses the on-device model's tokenizer, so +for Private Cloud Compute it is an approximation. + The full request also includes instructions, tool definitions, schema text, attachments, and generated output budget. If Apple reports a context overflow, trim or summarize and retry. From 4dd6d9cbe6361d32c2576f4772c1a4b84344e790 Mon Sep 17 00:00:00 2001 From: Mike Grabowski Date: Sat, 24 Oct 2026 23:48:23 +0400 Subject: [PATCH 5/8] feat(apple): stream tool calls on ios 26 and structured output On iOS 26, stream snapshots have no transcript entries, so streamText never reported Apple tool calls. Read them from the session transcript instead, which already holds the tool call and output entries when the first text chunk arrives. streamText with Output.object threw "Streaming JSON responses is not yet supported." Native streams now send the snapshot's JSON. Partial snapshots are not prefixes of the final JSON (even key order changes), so the provider sends the complete object once the stream finishes. Schema streams also report reasoning and tool entries on iOS 27. Co-Authored-By: Claude Opus 5.5 --- packages/apple-llm/ios/AppleLLMImpl.swift | 71 +++++++++++++------ .../apple-llm/src/__tests__/ai-sdk.test.ts | 64 +++++++++++++++++ packages/apple-llm/src/ai-sdk.ts | 21 ++++-- website/src/docs/apple/generating.md | 6 +- 4 files changed, 135 insertions(+), 27 deletions(-) diff --git a/packages/apple-llm/ios/AppleLLMImpl.swift b/packages/apple-llm/ios/AppleLLMImpl.swift index c77ffc22a..6c80e25bd 100644 --- a/packages/apple-llm/ios/AppleLLMImpl.swift +++ b/packages/apple-llm/ios/AppleLLMImpl.swift @@ -655,6 +655,19 @@ public class AppleLLMImpl: NSObject { providerOptions: [String: Any], onUpdate: @escaping (String, [[String: Any]]?) -> Void ) async throws -> [String: Any]? { + // Reasoning and tool activity are only sent when they change + var previousContent: String? + var previousEntries: NSArray = [] + func update(_ content: String, _ entries: [[String: Any]]) { + let entriesChanged = !(entries as NSArray).isEqual(previousEntries) + guard entriesChanged || content != previousContent else { + return + } + previousContent = content + previousEntries = entries as NSArray + onUpdate(content, entriesChanged ? entries : nil) + } + #if compiler(>=6.4) if #available(iOS 27, *) { let contextOptions = try createContextOptions( @@ -680,28 +693,28 @@ public class AppleLLMImpl: NSObject { } for try await chunk in responseStream { usage = chunk.usage - onUpdate(String(describing: chunk.content), nil) + update( + chunk.rawContent.jsonString, + AppleLLMTranscriptSerializer.serialize(chunk.transcriptEntries, includeResponses: false) + ) } - return usage.map(Self.usageDictionary) - } - - let responseStream = session.streamResponse( - options: generationOptions, - contextOptions: contextOptions - ) { - userPrompt - for attachment in preparedAttachments.values { - attachment + } else { + let responseStream = session.streamResponse( + options: generationOptions, + contextOptions: contextOptions + ) { + userPrompt + for attachment in preparedAttachments.values { + attachment + } + } + for try await chunk in responseStream { + usage = chunk.usage + update( + chunk.content, + AppleLLMTranscriptSerializer.serialize(chunk.transcriptEntries, includeResponses: false) + ) } - } - // Reasoning and tool activity are only sent when they change - var previousEntries: NSArray = [] - for try await chunk in responseStream { - usage = chunk.usage - let entries = AppleLLMTranscriptSerializer.serialize(chunk.transcriptEntries, includeResponses: false) - let entriesChanged = !(entries as NSArray).isEqual(previousEntries) - previousEntries = entries as NSArray - onUpdate(chunk.content, entriesChanged ? entries : nil) } return usage.map(Self.usageDictionary) } @@ -712,7 +725,17 @@ public class AppleLLMImpl: NSObject { throw AppleLLMError.unsupportedOS } + // iOS 26 snapshots have no transcript entries, so tool activity is read from + // the session transcript, which grows while the response streams let transcriptBefore = session.transcript + let newEntries = { + AppleLLMTranscriptSerializer.serialize( + session.transcript.dropFirst(transcriptBefore.count), + includeResponses: false + ) + } + + var content = "" if let generationSchema { let responseStream = session.streamResponse( to: userPrompt, @@ -721,14 +744,18 @@ public class AppleLLMImpl: NSObject { options: generationOptions ) for try await chunk in responseStream { - onUpdate(String(describing: chunk.content), nil) + content = chunk.rawContent.jsonString + update(content, newEntries()) } } else { let responseStream = session.streamResponse(to: userPrompt, options: generationOptions) for try await chunk in responseStream { - onUpdate(chunk.content, nil) + content = chunk.content + update(content, newEntries()) } } + // Entries added after the last snapshot + update(content, newEntries()) return await countUsage( session: session, diff --git a/packages/apple-llm/src/__tests__/ai-sdk.test.ts b/packages/apple-llm/src/__tests__/ai-sdk.test.ts index 2793aef40..80cab1693 100644 --- a/packages/apple-llm/src/__tests__/ai-sdk.test.ts +++ b/packages/apple-llm/src/__tests__/ai-sdk.test.ts @@ -536,6 +536,70 @@ describe('apple usage', () => { }) }) +describe('apple structured output stream', () => { + test('sends the final json once the stream completes', async () => { + const { stream } = await apple().doStream( + createCallOptions({ + responseFormat: { + type: 'json', + schema: { type: 'object', properties: { city: { type: 'string' } } }, + }, + }) + ) + const parts = readAll(stream) + const streamId = generateStream.mock.calls.at(-1)![0] + + onStreamUpdate.emit({ streamId, content: '{"city":"Par"}' }) + onStreamUpdate.emit({ streamId, content: '{"city":"Paris"}' }) + onStreamComplete.emit({ streamId }) + + expect( + (await parts).filter( + (part) => part.type !== 'stream-start' && part.type !== 'finish' + ) + ).toEqual([ + { type: 'text-start', id: streamId }, + { type: 'text-delta', id: streamId, delta: '{"city":"Paris"}' }, + { type: 'text-end', id: streamId }, + ]) + }) +}) + +describe('apple with the ai sdk', () => { + test('streams structured output and usage through streamText', async () => { + const { jsonSchema, Output, streamText } = await import('ai') + generateStream.mockImplementationOnce((streamId) => { + setTimeout(() => { + onStreamUpdate.emit({ streamId, content: '{"city":"Par"}' }) + onStreamUpdate.emit({ streamId, content: '{"city":"Paris"}' }) + onStreamComplete.emit({ + streamId, + usage: { inputTokens: 40, outputTokens: 6 }, + }) + }) + }) + + const result = streamText({ + model: apple(), + prompt: 'Where is the Eiffel Tower?', + output: Output.object({ + schema: jsonSchema<{ city: string }>({ + type: 'object', + properties: { city: { type: 'string' } }, + required: ['city'], + }), + }), + }) + + expect(await result.output).toEqual({ city: 'Paris' }) + expect(await result.usage).toMatchObject({ + inputTokens: 40, + outputTokens: 6, + totalTokens: 46, + }) + }) +}) + describe('apple token counting', () => { test('counts tokens through the language model', async () => { expect(await apple().countTokens('hello')).toBe(5) diff --git a/packages/apple-llm/src/ai-sdk.ts b/packages/apple-llm/src/ai-sdk.ts index fcf5a5232..6805240d6 100644 --- a/packages/apple-llm/src/ai-sdk.ts +++ b/packages/apple-llm/src/ai-sdk.ts @@ -937,10 +937,6 @@ class AppleLLMChatLanguageModel implements AppleLanguageModel { ? options.responseFormat.schema : undefined - if (schema) { - throw new Error('Streaming JSON responses is not yet supported.') - } - for (const tool of tools) { globalThis.__APPLE_LLM_TOOLS__[tool.id] = tool.execute } @@ -1048,6 +1044,14 @@ class AppleLLMChatLanguageModel implements AppleLanguageModel { } const nextRawContent = String(data.content ?? '') + + // Partial JSON snapshots are not prefixes of the final JSON, so + // structured output is sent once the stream completes + if (schema) { + previousRawContent = nextRawContent + return + } + const rawDelta = getSnapshotDelta( previousRawContent, nextRawContent @@ -1074,6 +1078,15 @@ class AppleLLMChatLanguageModel implements AppleLanguageModel { const completeListener = NativeAppleLLM.onStreamComplete((data) => { if (data.streamId === streamId) { endReasoning() + if (schema && previousRawContent) { + controller.enqueue({ type: 'text-start', id }) + controller.enqueue({ + type: 'text-delta', + delta: previousRawContent, + id, + }) + textStarted = true + } if (textStarted) { controller.enqueue({ type: 'text-end', diff --git a/website/src/docs/apple/generating.md b/website/src/docs/apple/generating.md index c6e427f56..c1d9e8045 100644 --- a/website/src/docs/apple/generating.md +++ b/website/src/docs/apple/generating.md @@ -66,7 +66,9 @@ for await (const delta of result.textStream) { } ``` -Streaming structured objects are not currently supported by this provider. +Structured output (`Output.object`) also works with `streamText`, but Apple +returns partial objects that are not prefixes of the final JSON, so the provider +sends the complete object once generation finishes instead of streaming it. ## Structured Output @@ -337,6 +339,8 @@ Important Apple-specific behavior: - Tool calls are provider-executed, so AI SDK step callbacks such as `stopWhen`, `onStepStart`, and `onStepEnd` do not run for native Apple tool execution. +- Tool calls and their results are reported with Apple's tool call IDs, in + `generateText` content and as stream parts in `streamText`. - You can update the registered tools on a model instance with `model.updateTools(...)` before generation. From e53ba0197ea8073e680dbc6ec72a355d3d0bcbef Mon Sep 17 00:00:00 2001 From: Mike Grabowski Date: Sat, 24 Oct 2026 23:49:12 +0400 Subject: [PATCH 6/8] refactor: align provider factories with ProviderV4 naming AI SDK 7 names provider factories embeddingModel and rerankingModel and gives providers a specificationVersion. - every provider factory sets specificationVersion: 'v4' - apple and llama add embeddingModel, and llama adds rerankingModel; the published textEmbeddingModel and rerankModel stay as deprecated aliases - core-ai drops textEmbeddingModel, which was never published - mlc adds createMlcProvider, so mlc is a callable provider like the others; mlc.languageModel() keeps working Docs, README and skills use the new names. Co-Authored-By: Claude Opus 5.5 --- README.md | 12 ++++++------ packages/adk/src/ai-sdk.ts | 2 ++ packages/apple-llm/src/ai-sdk.ts | 5 ++++- packages/core-ai/src/ai-sdk.ts | 4 ++-- packages/llama/src/ai-sdk.ts | 12 ++++++++++-- packages/mlc/src/ai-sdk.ts | 15 ++++++++++++--- packages/mlc/src/index.ts | 2 +- .../react-native-ai/references/apple-provider.md | 2 +- .../react-native-ai/references/llama-provider.md | 12 ++++++------ website/src/docs/apple/embeddings.md | 8 ++++---- website/src/docs/llama/embeddings.md | 6 +++--- website/src/docs/llama/getting-started.mdx | 10 +++++----- website/src/docs/llama/model-management.md | 4 ++-- website/src/docs/llama/reranking.md | 4 ++-- 14 files changed, 60 insertions(+), 38 deletions(-) diff --git a/README.md b/README.md index ac716486a..7b44b7259 100644 --- a/README.md +++ b/README.md @@ -91,7 +91,7 @@ const { text } = await generateText({ // Generate embeddings const { embedding } = await embed({ - model: apple.textEmbeddingModel(), + model: apple.embeddingModel(), value: 'Hello world', }) @@ -166,11 +166,11 @@ Run any GGUF model on-device using [llama.rn](https://github.com/mybigday/llama. #### Supported Features -| Feature | Method | Description | -| --------------- | ---------------------------- | --------------------------------------------- | -| Text Generation | `llama.languageModel()` | Chat, completion, streaming, reasoning models | -| Embeddings | `llama.textEmbeddingModel()` | Text embeddings for RAG and similarity search | -| Speech | `llama.speechModel()` | Text-to-speech with vocoder models | +| Feature | Method | Description | +| --------------- | ------------------------ | --------------------------------------------- | +| Text Generation | `llama.languageModel()` | Chat, completion, streaming, reasoning models | +| Embeddings | `llama.embeddingModel()` | Text embeddings for RAG and similarity search | +| Speech | `llama.speechModel()` | Text-to-speech with vocoder models | #### Installation diff --git a/packages/adk/src/ai-sdk.ts b/packages/adk/src/ai-sdk.ts index 79e7f4aba..079e1505c 100644 --- a/packages/adk/src/ai-sdk.ts +++ b/packages/adk/src/ai-sdk.ts @@ -59,6 +59,7 @@ async function checkNanoSupported(): Promise { export interface AdkProvider { (): LanguageModelV4 + readonly specificationVersion: 'v4' languageModel: () => LanguageModelV4 /** Device capability check (ML Kit `checkStatus !== 0`). Returns `false` on non-Android. */ isNanoSupported: () => Promise @@ -79,6 +80,7 @@ export function createAdkProvider( const provider = function () { return createLanguageModel() } + provider.specificationVersion = 'v4' as const provider.languageModel = createLanguageModel provider.isNanoSupported = () => checkNanoSupported() provider.isAvailable = async (modelType: AdkModelType = 'genai-nano') => { diff --git a/packages/apple-llm/src/ai-sdk.ts b/packages/apple-llm/src/ai-sdk.ts index 6805240d6..7d3e82a35 100644 --- a/packages/apple-llm/src/ai-sdk.ts +++ b/packages/apple-llm/src/ai-sdk.ts @@ -507,6 +507,7 @@ export function createAppleProvider({ ) { return createLanguageModel(options) } + provider.specificationVersion = 'v4' as const provider.isAvailable = () => NativeAppleLLM.isAvailable() provider.getModelInfo = ( options: { locale?: string; model?: AppleLanguageModelId } = {} @@ -516,9 +517,11 @@ export function createAppleProvider({ options.model ) as Promise provider.languageModel = createLanguageModel - provider.textEmbeddingModel = (options: AppleEmbeddingOptions = {}) => { + provider.embeddingModel = (options: AppleEmbeddingOptions = {}) => { return new AppleTextEmbeddingModel(options) } + /** @deprecated Use `embeddingModel` */ + provider.textEmbeddingModel = provider.embeddingModel provider.imageModel = (options: AppleImageModelOptions = {}) => { return new AppleImageModel(options) } diff --git a/packages/core-ai/src/ai-sdk.ts b/packages/core-ai/src/ai-sdk.ts index a3823c2e7..56610170e 100644 --- a/packages/core-ai/src/ai-sdk.ts +++ b/packages/core-ai/src/ai-sdk.ts @@ -38,14 +38,14 @@ export function createCoreAIProvider() { return provider.languageModel(config) } + provider.specificationVersion = 'v4' as const provider.getCapabilities = () => NativeCoreAI.getCapabilities() provider.languageModel = (config: CoreAIModelConfig) => { return new CoreAILanguageModel(config) } - provider.textEmbeddingModel = (config: CoreAIModelConfig) => { + provider.embeddingModel = (config: CoreAIModelConfig) => { return new CoreAITextEmbeddingModel(config) } - provider.embeddingModel = provider.textEmbeddingModel provider.imageModel = (config: CoreAIModelConfig) => { return new CoreAIImageGenerationModel(config) } diff --git a/packages/llama/src/ai-sdk.ts b/packages/llama/src/ai-sdk.ts index da85d7725..3a50fedc8 100644 --- a/packages/llama/src/ai-sdk.ts +++ b/packages/llama/src/ai-sdk.ts @@ -833,6 +833,8 @@ export function createLlamaProvider() { return provider.languageModel(modelPath, options) } + provider.specificationVersion = 'v4' as const + /** * Create a language model instance * @param modelPath - Path to the model file (from downloadModel() or getModelPath()) @@ -850,25 +852,31 @@ export function createLlamaProvider() { * @param modelPath - Path to the model file (from downloadModel() or getModelPath()) * @param options - Model configuration options */ - provider.textEmbeddingModel = ( + provider.embeddingModel = ( modelPath: string, options: LlamaEmbeddingOptions = {} ): LlamaEmbeddingModel => { return new LlamaEmbeddingModel(modelPath, options) } + /** @deprecated Use `embeddingModel` */ + provider.textEmbeddingModel = provider.embeddingModel + /** * Create a rerank model instance for document ranking * @param modelPath - Path to a reranker model file (from downloadModel() or getModelPath()) * @param options - Model configuration options */ - provider.rerankModel = ( + provider.rerankingModel = ( modelPath: string, options: LlamaRerankOptions = {} ): LlamaRerankModel => { return new LlamaRerankModel(modelPath, options) } + /** @deprecated Use `rerankingModel` */ + provider.rerankModel = provider.rerankingModel + /** * Create a speech model instance * @param modelPath - Path to the model file (from downloadModel() or getModelPath()) diff --git a/packages/mlc/src/ai-sdk.ts b/packages/mlc/src/ai-sdk.ts index 1eb70cc1d..26d2ef9cc 100644 --- a/packages/mlc/src/ai-sdk.ts +++ b/packages/mlc/src/ai-sdk.ts @@ -18,12 +18,21 @@ import NativeMLCEngine, { type Message, } from './NativeMLCEngine' -export const mlc = { - languageModel: (modelId: string = 'Llama-3.2-3B-Instruct') => { +export function createMlcProvider() { + const provider = function (modelId?: string) { + return provider.languageModel(modelId) + } + + provider.specificationVersion = 'v4' as const + provider.languageModel = (modelId: string = 'Llama-3.2-3B-Instruct') => { return new MlcChatLanguageModel(modelId) - }, + } + + return provider } +export const mlc = createMlcProvider() + const convertToolsToNativeFormat = ( tools: (LanguageModelV4FunctionTool | LanguageModelV4ProviderTool)[] ) => { diff --git a/packages/mlc/src/index.ts b/packages/mlc/src/index.ts index 45fa79e4d..9e96986ca 100644 --- a/packages/mlc/src/index.ts +++ b/packages/mlc/src/index.ts @@ -1,2 +1,2 @@ -export { mlc } from './ai-sdk' +export { createMlcProvider, mlc } from './ai-sdk' export { default as MLCEngine } from './NativeMLCEngine' diff --git a/skills/react-native-ai/references/apple-provider.md b/skills/react-native-ai/references/apple-provider.md index 72c6f0b12..c6e1630c7 100644 --- a/skills/react-native-ai/references/apple-provider.md +++ b/skills/react-native-ai/references/apple-provider.md @@ -57,7 +57,7 @@ the user in Settings. | Type | Method | Use Case | Documentation | | ------------- | ---------------------------- | --------------------------------------- | -------------------------------------------------------- | | Language | `apple.languageModel()` | Text generation, chat | https://www.react-native-ai.dev/docs/apple/generating | -| Embedding | `apple.textEmbeddingModel()` | RAG, similarity, prompt size estimation | https://www.react-native-ai.dev/docs/apple/embeddings | +| Embedding | `apple.embeddingModel()` | RAG, similarity, prompt size estimation | https://www.react-native-ai.dev/docs/apple/embeddings | | Transcription | `apple.transcriptionModel()` | Speech-to-text | https://www.react-native-ai.dev/docs/apple/transcription | | Speech | `apple.speechModel()` | Text-to-speech | https://www.react-native-ai.dev/docs/apple/speech | diff --git a/skills/react-native-ai/references/llama-provider.md b/skills/react-native-ai/references/llama-provider.md index 319480d48..1b6420f1f 100644 --- a/skills/react-native-ai/references/llama-provider.md +++ b/skills/react-native-ai/references/llama-provider.md @@ -89,12 +89,12 @@ const exists = await isModelDownloaded('owner/repo/model.gguf') ### 5. Model Types -| Type | Method | Notes | Documentation | -| --------- | ---------------------------- | --------------------------------------- | ------------------------------------------------------------------------------------ | -| Language | `llama.languageModel()` | Text generation, chat | https://www.react-native-ai.dev/docs/llama/generating | -| Embedding | `llama.textEmbeddingModel()` | RAG, similarity, prompt size estimation | https://www.react-native-ai.dev/docs/llama/embeddings | -| Rerank | `llama.rerankModel()` | Document ranking, RAG | https://www.react-native-ai.dev/docs/llama/reranking | -| Speech | `llama.speechModel()` | Requires `vocoderPath` in opts | https://www.react-native-ai.dev/docs/llama/model-management#creating-model-instances | +| Type | Method | Notes | Documentation | +| --------- | ------------------------ | --------------------------------------- | ------------------------------------------------------------------------------------ | +| Language | `llama.languageModel()` | Text generation, chat | https://www.react-native-ai.dev/docs/llama/generating | +| Embedding | `llama.embeddingModel()` | RAG, similarity, prompt size estimation | https://www.react-native-ai.dev/docs/llama/embeddings | +| Rerank | `llama.rerankingModel()` | Document ranking, RAG | https://www.react-native-ai.dev/docs/llama/reranking | +| Speech | `llama.speechModel()` | Requires `vocoderPath` in opts | https://www.react-native-ai.dev/docs/llama/model-management#creating-model-instances | ### 6. Lifecycle diff --git a/website/src/docs/apple/embeddings.md b/website/src/docs/apple/embeddings.md index d2d26387b..cd366e77a 100644 --- a/website/src/docs/apple/embeddings.md +++ b/website/src/docs/apple/embeddings.md @@ -29,7 +29,7 @@ import { embed } from 'ai' import { apple } from '@react-native-ai/apple' const { embedding } = await embed({ - model: apple.textEmbeddingModel(), + model: apple.embeddingModel(), value: 'Hello world', }) @@ -43,7 +43,7 @@ import { embedMany } from 'ai' import { apple } from '@react-native-ai/apple' const { embeddings } = await embedMany({ - model: apple.textEmbeddingModel(), + model: apple.embeddingModel(), values: ['Hello world', 'How are you?', 'Goodbye'], }) @@ -55,7 +55,7 @@ console.log(embeddings) The embeddings model supports multiple languages. You can specify the language using [ISO 639-1 codes](https://en.wikipedia.org/wiki/List_of_ISO_639_language_codes) or full names when creating the model: ```tsx -const model = apple.textEmbeddingModel({ language: 'fr' }) +const model = apple.embeddingModel({ language: 'fr' }) await embed({ model, @@ -73,7 +73,7 @@ For list of all supported languages, [check Apple documentation](https://develop Apple's NLContextualEmbedding requires downloading language-specific assets to the device. While the provider automatically prepares assets when needed, you can call `prepare()` ahead of time for better performance: ```tsx -const model = apple.textEmbeddingModel({ language: 'en' }) +const model = apple.embeddingModel({ language: 'en' }) // Call prepare() ahead of time to optimize first inference latency await model.prepare() diff --git a/website/src/docs/llama/embeddings.md b/website/src/docs/llama/embeddings.md index c88ee084f..30aa4836e 100644 --- a/website/src/docs/llama/embeddings.md +++ b/website/src/docs/llama/embeddings.md @@ -17,7 +17,7 @@ import { embed } from 'ai' const modelPath = await downloadModel('owner/repo/embedding-model.gguf') // Create embedding model with the path -const model = llama.textEmbeddingModel(modelPath) +const model = llama.embeddingModel(modelPath) await model.prepare() @@ -40,7 +40,7 @@ import { embedMany } from 'ai' const modelPath = await downloadModel('owner/repo/embedding-model.gguf') -const model = llama.textEmbeddingModel(modelPath) +const model = llama.embeddingModel(modelPath) await model.prepare() const { embeddings } = await embedMany({ @@ -61,7 +61,7 @@ console.log(embeddings[0].length) // Embedding dimension (e.g., 384, 768) Configure the embedding model with specific options: ```typescript -const model = llama.textEmbeddingModel(modelPath, { +const model = llama.embeddingModel(modelPath, { normalize: -1, // Normalization mode (default: -1) contextParams: { n_ctx: 2048, // Context size (default: 2048) diff --git a/website/src/docs/llama/getting-started.mdx b/website/src/docs/llama/getting-started.mdx index fc21f7025..0c844c413 100644 --- a/website/src/docs/llama/getting-started.mdx +++ b/website/src/docs/llama/getting-started.mdx @@ -50,11 +50,11 @@ For all other installation tips and tricks, refer to the [llama.rn Expo document The Llama provider supports multiple model types: -| Model Type | Method | Use Case | -| --------------- | ---------------------------- | ----------------------------------- | -| Language Model | `llama.languageModel()` | Text generation, chat, reasoning | -| Embedding Model | `llama.textEmbeddingModel()` | Text embeddings for RAG, similarity | -| Speech Model | `llama.speechModel()` | Text-to-speech with vocoder | +| Model Type | Method | Use Case | +| --------------- | ------------------------ | ----------------------------------- | +| Language Model | `llama.languageModel()` | Text generation, chat, reasoning | +| Embedding Model | `llama.embeddingModel()` | Text embeddings for RAG, similarity | +| Speech Model | `llama.speechModel()` | Text-to-speech with vocoder | ## Basic Usage diff --git a/website/src/docs/llama/model-management.md b/website/src/docs/llama/model-management.md index c8224a9dd..91ffd5e39 100644 --- a/website/src/docs/llama/model-management.md +++ b/website/src/docs/llama/model-management.md @@ -72,7 +72,7 @@ const modelPath = await downloadModel('bartowski/Qwen_Qwen3.5-2B-GGUF/Qwen_Qwen3 const languageModel = llama.languageModel(modelPath) // Embedding model -const embeddingModel = llama.textEmbeddingModel(modelPath) +const embeddingModel = llama.embeddingModel(modelPath) // Speech model (requires vocoder) const speechModel = llama.speechModel(modelPath, { @@ -162,7 +162,7 @@ Creates a language model instance. - `n_ctx`: Context size (default: 2048, or 4096 for multimodal) - `n_gpu_layers`: Number of GPU layers (default: 99) -### `llama.textEmbeddingModel(modelPath, options?)` +### `llama.embeddingModel(modelPath, options?)` Creates an embedding model instance. diff --git a/website/src/docs/llama/reranking.md b/website/src/docs/llama/reranking.md index 9d03e95ff..a903be2e5 100644 --- a/website/src/docs/llama/reranking.md +++ b/website/src/docs/llama/reranking.md @@ -19,7 +19,7 @@ const modelPath = await downloadModel( ) // Create rerank model with the path -const model = llama.rerankModel(modelPath) +const model = llama.rerankingModel(modelPath) await model.prepare() @@ -62,7 +62,7 @@ const { ranking } = await rerank({ Configure the rerank model with specific options: ```typescript -const model = llama.rerankModel(modelPath, { +const model = llama.rerankingModel(modelPath, { normalize: 1, // Score normalization (default: from model config) contextParams: { n_ctx: 2048, // Context size (default: 2048) From 0217bb7578ec3512b6437e4536c511e9a78f32c4 Mon Sep 17 00:00:00 2001 From: Mike Grabowski Date: Sun, 25 Oct 2026 00:04:40 +0400 Subject: [PATCH 7/8] fix(utils): call countTokens on its model countPromptTokens took countTokens off the model and called it without a receiver. The llama.rn model reads this.context in countTokens, so it threw and the summarizer silently fell back to four characters per token. Co-Authored-By: Claude Opus 5.5 --- packages/utils/src/__tests__/index.test.ts | 70 ++++++++++++++++++++++ packages/utils/src/index.ts | 7 ++- packages/utils/tsconfig.json | 3 +- 3 files changed, 76 insertions(+), 4 deletions(-) create mode 100644 packages/utils/src/__tests__/index.test.ts diff --git a/packages/utils/src/__tests__/index.test.ts b/packages/utils/src/__tests__/index.test.ts new file mode 100644 index 000000000..4cfd067ef --- /dev/null +++ b/packages/utils/src/__tests__/index.test.ts @@ -0,0 +1,70 @@ +import type { + LanguageModelV4, + LanguageModelV4CallOptions, + LanguageModelV4Prompt, +} from '@ai-sdk/provider' +import { describe, expect, mock, test } from 'bun:test' + +import { applyHistoryManagement } from '../index' + +const summaryResult = { + content: [ + { type: 'text' as const, text: 'Earlier: the user asked about Paris' }, + ], + finishReason: { unified: 'stop' as const, raw: 'stop' }, + usage: { + inputTokens: { + total: undefined, + noCache: undefined, + cacheRead: undefined, + cacheWrite: undefined, + }, + outputTokens: { total: undefined, text: undefined, reasoning: undefined }, + }, + warnings: [], +} + +// A model whose countTokens reads instance state, like the llama.rn model +class TokenCountingModel implements LanguageModelV4 { + readonly specificationVersion = 'v4' + readonly provider = 'test' + readonly modelId = 'test' + readonly supportedUrls = {} + + private tokensPerCharacter = 1 + + doGenerate = mock( + async (_options: LanguageModelV4CallOptions) => summaryResult + ) + doStream = mock(async (_options: LanguageModelV4CallOptions) => { + throw new Error('not used') + }) + + async countTokens(text: string) { + return text.length * this.tokensPerCharacter + } +} + +const prompt: LanguageModelV4Prompt = [ + { role: 'user', content: [{ type: 'text', text: 'Tell me about Paris' }] }, + { + role: 'assistant', + content: [{ type: 'text', text: 'Paris is in France' }], + }, + { role: 'user', content: [{ type: 'text', text: 'And Lyon?' }] }, +] + +describe('history summarization', () => { + test('counts tokens with the model as receiver', async () => { + const model = new TokenCountingModel() + + // About 70 characters, so the length / 4 fallback (about 18) stays under + // the threshold and only the model's own count goes over it + const result = await applyHistoryManagement(prompt, { + summarizeHistory: { threshold: 40, model }, + }) + + expect(model.doGenerate).toHaveBeenCalledTimes(1) + expect(result).toHaveLength(2) + }) +}) diff --git a/packages/utils/src/index.ts b/packages/utils/src/index.ts index 77bbe30a2..0223630e4 100644 --- a/packages/utils/src/index.ts +++ b/packages/utils/src/index.ts @@ -100,11 +100,12 @@ async function countPromptTokens( model: LanguageModelV4 ): Promise { const serializedPrompt = serializePromptForSummary(prompt) - const countTokens = (model as TokenCountCapableLanguageModel).countTokens + const tokenCountModel = model as TokenCountCapableLanguageModel - if (typeof countTokens === 'function') { + // Called on the model, since providers read their own state in countTokens + if (typeof tokenCountModel.countTokens === 'function') { try { - return await countTokens(serializedPrompt) + return await tokenCountModel.countTokens(serializedPrompt) } catch { return estimateTokenCount(serializedPrompt) } diff --git a/packages/utils/tsconfig.json b/packages/utils/tsconfig.json index bf5a36ddb..54f5777b4 100644 --- a/packages/utils/tsconfig.json +++ b/packages/utils/tsconfig.json @@ -1,4 +1,5 @@ { "extends": "../../tsconfig.base.json", - "include": ["src/**/*"] + "include": ["src/**/*"], + "exclude": ["src/**/__tests__/**"] } From d5062c175df01b4846a885e0c1e93feee2c6ca79 Mon Sep 17 00:00:00 2001 From: Mike Grabowski Date: Sun, 25 Oct 2026 00:04:44 +0400 Subject: [PATCH 8/8] docs(apple): say private cloud compute has no token counting api countTokens on a Private Cloud Compute model counts with the on-device model's tokenizer, which may not match the server model, and needs the on-device model to be available. Calling it an approximation claimed more than is known. Co-Authored-By: Claude Opus 5.5 --- packages/apple-llm/src/ai-sdk.ts | 3 ++- website/src/docs/apple/generating.md | 6 ++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/packages/apple-llm/src/ai-sdk.ts b/packages/apple-llm/src/ai-sdk.ts index 7d3e82a35..c4f5bd6f2 100644 --- a/packages/apple-llm/src/ai-sdk.ts +++ b/packages/apple-llm/src/ai-sdk.ts @@ -45,7 +45,8 @@ export interface AppleLanguageModel extends LanguageModelV4 { updateTools: (tools: AppleToolDefinitionSet) => void /** * Counts tokens in text with the on-device model's tokenizer (iOS 26.4+). - * Private Cloud Compute has no tokenizer, so this approximates its count + * Private Cloud Compute has no token counting API, so Private Cloud Compute + * models also count with the on-device tokenizer, which may not match */ countTokens: (text: string) => Promise } diff --git a/website/src/docs/apple/generating.md b/website/src/docs/apple/generating.md index c1d9e8045..4ac4bc919 100644 --- a/website/src/docs/apple/generating.md +++ b/website/src/docs/apple/generating.md @@ -395,8 +395,10 @@ const tokenCount = await model.countTokens( ) ``` -Token counting requires iOS 26.4 and uses the on-device model's tokenizer, so -for Private Cloud Compute it is an approximation. +Token counting requires iOS 26.4 and an available on-device model. Private +Cloud Compute has no token counting API, so `countTokens` on a Private Cloud +Compute model still uses the on-device model's tokenizer, whose count may not +match the server model. The full request also includes instructions, tool definitions, schema text, attachments, and generated output budget. If Apple reports a context overflow,