From 37c3e6db7ce2adba740f2f835c9d5be1c251e8ac Mon Sep 17 00:00:00 2001 From: Julian Kobrynski Date: Wed, 20 May 2026 16:03:50 +0200 Subject: [PATCH 01/11] docs(apple): document context window handling --- website/src/docs/apple/generating.md | 52 ++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/website/src/docs/apple/generating.md b/website/src/docs/apple/generating.md index 815edb020..8e58b4d0d 100644 --- a/website/src/docs/apple/generating.md +++ b/website/src/docs/apple/generating.md @@ -203,6 +203,58 @@ if (!apple.isAvailable()) { } ``` +## Context Window + +Apple Foundation Models have a fixed context window of 4096 tokens. This limit applies to the full request context, including system instructions, previous conversation messages, tool definitions, schemas, and the current user prompt. + +The `maxTokens` option only limits how many tokens the model can generate in its response. It does not increase the available context window or reserve enough room for a long prompt. + +If the full context is too large, Apple may fail generation with a context-window overflow error. The provider does not automatically remove messages from your prompt, because different apps need different memory strategies. Handle this at the application level by catching the error and choosing the recovery behavior that fits your product: + +- Start a new conversation without the previous transcript +- Keep a sliding window of recent messages +- Summarize older messages and include the summary instead of the full transcript +- Ask the user to shorten the prompt or start a new chat + +```typescript +import { apple } from '@react-native-ai/apple'; +import { generateText } from 'ai'; + +try { + const result = await generateText({ + model: apple(), + messages + }); +} catch (error) { + const message = error instanceof Error ? error.message : String(error); + + if (message.toLowerCase().includes('context')) { + // Apply your app's recovery strategy here. + // For example: retry with fewer messages or start a new chat. + } + + throw error; +} +``` + +For streaming calls, use `fullStream` when you need to inspect provider error parts: + +```typescript +import { apple } from '@react-native-ai/apple'; +import { streamText } from 'ai'; + +const result = streamText({ + model: apple(), + messages +}); + +for await (const part of result.fullStream) { + if (part.type === 'error') { + // Handle streamed errors here. + } +} +``` + ## Available Options Configure model behavior with generation options: From 654f793edd154fa8da1887e89adb23dabc04718a Mon Sep 17 00:00:00 2001 From: Julian Kobrynski Date: Thu, 21 May 2026 16:15:56 +0200 Subject: [PATCH 02/11] feat(apple): expose context window error code --- packages/apple-llm/ios/AppleLLM.mm | 4 +-- packages/apple-llm/ios/AppleLLMError.swift | 29 ++++++++++++++++++++++ packages/apple-llm/ios/AppleLLMImpl.swift | 28 ++++++++++++++++----- packages/apple-llm/src/NativeAppleLLM.ts | 1 + packages/apple-llm/src/ai-sdk.ts | 3 ++- packages/apple-llm/src/errors.ts | 22 ++++++++++++++++ packages/apple-llm/src/index.ts | 2 ++ packages/apple-llm/src/stream.ts | 3 ++- 8 files changed, 82 insertions(+), 10 deletions(-) create mode 100644 packages/apple-llm/src/errors.ts diff --git a/packages/apple-llm/ios/AppleLLM.mm b/packages/apple-llm/ios/AppleLLM.mm index 04ef22f66..fb56c2fe1 100644 --- a/packages/apple-llm/ios/AppleLLM.mm +++ b/packages/apple-llm/ios/AppleLLM.mm @@ -155,8 +155,8 @@ - (nonnull NSString *)generateStream:(nonnull NSArray *)messages options:(JS::Na onComplete:^(NSString *streamId) { [self emitOnStreamComplete:@{@"streamId": streamId}]; } - onError:^(NSString *streamId, NSString *error) { - [self emitOnStreamError:@{@"streamId": streamId, @"error": error}]; + onError:^(NSString *streamId, NSString *code, NSString *error) { + [self emitOnStreamError:@{@"streamId": streamId, @"code": code, @"error": error}]; } toolInvoker:callToolBlock]; diff --git a/packages/apple-llm/ios/AppleLLMError.swift b/packages/apple-llm/ios/AppleLLMError.swift index 570ef9c53..9c35987d5 100644 --- a/packages/apple-llm/ios/AppleLLMError.swift +++ b/packages/apple-llm/ios/AppleLLMError.swift @@ -17,6 +17,7 @@ enum AppleLLMError: Error, LocalizedError { case invalidSchema(String) case toolCallError(Error) case unknownToolCallError + case contextWindowExceeded var errorDescription: String? { switch self { @@ -38,9 +39,36 @@ enum AppleLLMError: Error, LocalizedError { return "Error calling tool: \(error.localizedDescription)" case .unknownToolCallError: return "Unknown tool call error" + case .contextWindowExceeded: + return "Context window exceeded" } } + + var reactNativeCode: String { + switch self { + case .modelUnavailable: + return "MODEL_UNAVAILABLE" + case .unsupportedOS: + return "UNSUPPORTED_OS" + case .generationError: + return "GENERATION_ERROR" + case .streamNotFound: + return "STREAM_NOT_FOUND" + case .invalidMessage: + return "INVALID_MESSAGE" + case .conflictingSamplingMethods: + return "CONFLICTING_SAMPLING_METHODS" + case .invalidSchema: + return "INVALID_SCHEMA" + case .toolCallError: + return "TOOL_CALL_ERROR" + case .unknownToolCallError: + return "UNKNOWN_TOOL_CALL_ERROR" + case .contextWindowExceeded: + return "CONTEXT_WINDOW_EXCEEDED" + } + } var code: Int { switch self { @@ -53,6 +81,7 @@ enum AppleLLMError: Error, LocalizedError { case .invalidSchema: return 7 case .unknownToolCallError: return 8 case .toolCallError: return 9 + case .contextWindowExceeded: return 10 } } } diff --git a/packages/apple-llm/ios/AppleLLMImpl.swift b/packages/apple-llm/ios/AppleLLMImpl.swift index 7cb71f1e6..93d6332f8 100644 --- a/packages/apple-llm/ios/AppleLLMImpl.swift +++ b/packages/apple-llm/ios/AppleLLMImpl.swift @@ -74,16 +74,17 @@ public class AppleLLMImpl: NSObject { resolve(response.toModelMessages()) } } catch { - reject("AppleLLM", error.localizedDescription, error) + let appleError = self.createAppleLLMError(from: error) + reject(appleError.reactNativeCode, appleError.localizedDescription, appleError) } } } else { let error = AppleLLMError.unsupportedOS - reject("AppleLLM", error.localizedDescription, error) + reject(error.reactNativeCode, error.localizedDescription, error) } #else let error = AppleLLMError.unsupportedOS - reject("AppleLLM", error.localizedDescription, error) + reject(error.reactNativeCode, error.localizedDescription, error) #endif } @@ -93,14 +94,15 @@ public class AppleLLMImpl: NSObject { options: [String: Any], onUpdate: @escaping (String, String) -> Void, onComplete: @escaping (String) -> Void, - onError: @escaping (String, String) -> Void, + onError: @escaping (String, String, String) -> Void, toolInvoker: @escaping ToolInvoker ) throws -> String { #if canImport(FoundationModels) if #available(iOS 26, *) { let streamId = UUID().uuidString guard SystemLanguageModel.default.availability == .available else { - onError(streamId, "Apple Intelligence model is not available") + let error = AppleLLMError.modelUnavailable + onError(streamId, error.reactNativeCode, error.localizedDescription) return streamId } @@ -140,7 +142,8 @@ public class AppleLLMImpl: NSObject { onComplete(streamId) } } catch { - onError(streamId, error.localizedDescription) + let appleError = self.createAppleLLMError(from: error) + onError(streamId, appleError.reactNativeCode, appleError.localizedDescription) } // Clean up task from map when completed @@ -171,6 +174,19 @@ public class AppleLLMImpl: NSObject { // MARK: - Private Methods #if canImport(FoundationModels) + + @available(iOS 26, *) + private func createAppleLLMError(from error: Error) -> AppleLLMError { + if case LanguageModelSession.GenerationError.exceededContextWindowSize = error { + return .contextWindowExceeded + } + + if let appleError = error as? AppleLLMError { + return appleError + } + + return .generationError(error.localizedDescription) + } @available(iOS 26, *) private func createTools(from options: [String: Any], toolInvoker: @escaping ToolInvoker) throws -> [any Tool] { diff --git a/packages/apple-llm/src/NativeAppleLLM.ts b/packages/apple-llm/src/NativeAppleLLM.ts index eebd9ae85..7741bb1b2 100644 --- a/packages/apple-llm/src/NativeAppleLLM.ts +++ b/packages/apple-llm/src/NativeAppleLLM.ts @@ -30,6 +30,7 @@ export type StreamCompleteEvent = { export type StreamErrorEvent = { streamId: string + code: string error: string } diff --git a/packages/apple-llm/src/ai-sdk.ts b/packages/apple-llm/src/ai-sdk.ts index ff85d1c5c..b5b853a14 100644 --- a/packages/apple-llm/src/ai-sdk.ts +++ b/packages/apple-llm/src/ai-sdk.ts @@ -21,6 +21,7 @@ import { ToolCallOptions, } from '@ai-sdk/provider-utils' +import { createAppleLLMError } from './errors' import NativeAppleEmbeddings from './NativeAppleEmbeddings' import NativeAppleLLM, { type AppleMessage } from './NativeAppleLLM' import NativeAppleSpeech from './NativeAppleSpeech' @@ -464,7 +465,7 @@ class AppleLLMChatLanguageModel implements LanguageModelV3 { if (data.streamId === streamId) { controller.enqueue({ type: 'error', - error: data.error, + error: createAppleLLMError(data.error, data.code), }) cleanup() controller.close() diff --git a/packages/apple-llm/src/errors.ts b/packages/apple-llm/src/errors.ts new file mode 100644 index 000000000..d4749098f --- /dev/null +++ b/packages/apple-llm/src/errors.ts @@ -0,0 +1,22 @@ +export type AppleLLMError = Error & { + code?: string +} + +export const AppleLLMErrorCodes = { + ContextWindowExceeded: 'CONTEXT_WINDOW_EXCEEDED', + ModelUnavailable: 'MODEL_UNAVAILABLE', +} as const + +export type AppleLLMErrorCode = + (typeof AppleLLMErrorCodes)[keyof typeof AppleLLMErrorCodes] + +export function createAppleLLMError( + message: string, + code?: string +): AppleLLMError { + const error = new Error(message) as AppleLLMError + if (code) { + error.code = code + } + return error +} diff --git a/packages/apple-llm/src/index.ts b/packages/apple-llm/src/index.ts index 77ba09e5b..5ddf59daf 100644 --- a/packages/apple-llm/src/index.ts +++ b/packages/apple-llm/src/index.ts @@ -1,4 +1,6 @@ export { apple, createAppleProvider } from './ai-sdk' +export type { AppleLLMError, AppleLLMErrorCode } from './errors' +export { AppleLLMErrorCodes } from './errors' export { default as AppleEmbeddings } from './NativeAppleEmbeddings' export { default as AppleFoundationModels } from './NativeAppleLLM' export { default as AppleSpeech, VoiceInfo } from './NativeAppleSpeech' diff --git a/packages/apple-llm/src/stream.ts b/packages/apple-llm/src/stream.ts index d1d6fc2b8..759bafca5 100644 --- a/packages/apple-llm/src/stream.ts +++ b/packages/apple-llm/src/stream.ts @@ -1,5 +1,6 @@ import type { LanguageModelV2StreamPart } from '@ai-sdk/provider' +import { createAppleLLMError } from './errors' import NativeAppleLLMSpec, { type AppleGenerationOptions, type AppleMessage, @@ -67,7 +68,7 @@ export function generateStream( if (data.streamId === streamId) { controller.enqueue({ type: 'error', - error: data.error, + error: createAppleLLMError(data.error, data.code), }) cleanup() controller.close() From ab9a040d71f91774ae45122355ec90f60cfdca83 Mon Sep 17 00:00:00 2001 From: Julian Kobrynski Date: Thu, 21 May 2026 16:40:33 +0200 Subject: [PATCH 03/11] fix(expo-example): handle text stream errors --- apps/expo-example/src/screens/ChatScreen/index.tsx | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/apps/expo-example/src/screens/ChatScreen/index.tsx b/apps/expo-example/src/screens/ChatScreen/index.tsx index 1663ccb93..7318477c8 100644 --- a/apps/expo-example/src/screens/ChatScreen/index.tsx +++ b/apps/expo-example/src/screens/ChatScreen/index.tsx @@ -103,6 +103,7 @@ export default function ChatScreen() { setToolExecutionReporter(({ toolName, args, result }) => { addToolExecutionMessage(chatId, toolName, args, result) }) + let streamError: unknown const result = streamText({ model: selectedAdapter.model, messages: [ @@ -122,6 +123,9 @@ export default function ChatScreen() { additionalInstructions: 'If the user asks, tell who you are (assistant) and what is this (Callstack AI demo app).', }), + onError: ({ error }) => { + streamError = error + }, }) let accumulated = '' @@ -132,6 +136,8 @@ export default function ChatScreen() { updateMessageContent(chatId, assistantMessageId, accumulated) } + if (streamError) throw streamError + if (accumulated.trim().length === 0) { updateMessageContent( chatId, From 280fd8244f0d14ea3731bc59c27fe238be8fcb01 Mon Sep 17 00:00:00 2001 From: Julian Kobrynski Date: Fri, 22 May 2026 11:52:42 +0200 Subject: [PATCH 04/11] docs(apple): document text stream error handling --- website/src/docs/apple/generating.md | 56 +++++++++++++++++++++++++--- 1 file changed, 51 insertions(+), 5 deletions(-) diff --git a/website/src/docs/apple/generating.md b/website/src/docs/apple/generating.md index 8e58b4d0d..63c2bdb5d 100644 --- a/website/src/docs/apple/generating.md +++ b/website/src/docs/apple/generating.md @@ -217,7 +217,11 @@ If the full context is too large, Apple may fail generation with a context-windo - Ask the user to shorten the prompt or start a new chat ```typescript -import { apple } from '@react-native-ai/apple'; +import { + AppleLLMErrorCodes, + type AppleLLMError, + apple, +} from '@react-native-ai/apple'; import { generateText } from 'ai'; try { @@ -226,9 +230,9 @@ try { messages }); } catch (error) { - const message = error instanceof Error ? error.message : String(error); + const appleError = error as AppleLLMError; - if (message.toLowerCase().includes('context')) { + if (appleError.code === AppleLLMErrorCodes.ContextWindowExceeded) { // Apply your app's recovery strategy here. // For example: retry with fewer messages or start a new chat. } @@ -240,7 +244,11 @@ try { For streaming calls, use `fullStream` when you need to inspect provider error parts: ```typescript -import { apple } from '@react-native-ai/apple'; +import { + AppleLLMErrorCodes, + type AppleLLMError, + apple, +} from '@react-native-ai/apple'; import { streamText } from 'ai'; const result = streamText({ @@ -250,8 +258,46 @@ const result = streamText({ for await (const part of result.fullStream) { if (part.type === 'error') { - // Handle streamed errors here. + const error = part.error as AppleLLMError; + + if (error.code === AppleLLMErrorCodes.ContextWindowExceeded) { + // Apply your app's recovery strategy here. + } + } +} +``` + +If you only consume `textStream`, pass `onError` to `streamText`. The AI SDK does not emit error parts through the text-only stream, so capture the error there and handle it after the stream finishes: + +```typescript +import { + AppleLLMErrorCodes, + type AppleLLMError, + apple, +} from '@react-native-ai/apple'; +import { streamText } from 'ai'; + +let streamError: unknown; +const result = streamText({ + model: apple(), + messages, + onError: ({ error }) => { + streamError = error; + }, +}); + +for await (const delta of result.textStream) { + console.log(delta); +} + +if (streamError) { + const error = streamError as AppleLLMError; + + if (error.code === AppleLLMErrorCodes.ContextWindowExceeded) { + // Apply your app's recovery strategy here. } + + throw streamError; } ``` From 87450882e10118e865e7a34ea8162a0e6619f317 Mon Sep 17 00:00:00 2001 From: Julian Kobrynski Date: Fri, 22 May 2026 15:08:27 +0200 Subject: [PATCH 05/11] fix(apple): scope context window error code --- packages/apple-llm/ios/AppleLLM.mm | 6 ++++- packages/apple-llm/ios/AppleLLMError.swift | 22 +++------------- packages/apple-llm/ios/AppleLLMImpl.swift | 30 ++++++++++++---------- packages/apple-llm/src/NativeAppleLLM.ts | 2 +- packages/apple-llm/src/errors.ts | 1 - 5 files changed, 26 insertions(+), 35 deletions(-) diff --git a/packages/apple-llm/ios/AppleLLM.mm b/packages/apple-llm/ios/AppleLLM.mm index fb56c2fe1..75aa89c31 100644 --- a/packages/apple-llm/ios/AppleLLM.mm +++ b/packages/apple-llm/ios/AppleLLM.mm @@ -156,7 +156,11 @@ - (nonnull NSString *)generateStream:(nonnull NSArray *)messages options:(JS::Na [self emitOnStreamComplete:@{@"streamId": streamId}]; } onError:^(NSString *streamId, NSString *code, NSString *error) { - [self emitOnStreamError:@{@"streamId": streamId, @"code": code, @"error": error}]; + NSMutableDictionary *payload = [@{@"streamId": streamId, @"error": error} mutableCopy]; + if (code.length > 0) { + payload[@"code"] = code; + } + [self emitOnStreamError:payload]; } toolInvoker:callToolBlock]; diff --git a/packages/apple-llm/ios/AppleLLMError.swift b/packages/apple-llm/ios/AppleLLMError.swift index 9c35987d5..506039e8d 100644 --- a/packages/apple-llm/ios/AppleLLMError.swift +++ b/packages/apple-llm/ios/AppleLLMError.swift @@ -45,28 +45,12 @@ enum AppleLLMError: Error, LocalizedError { } - var reactNativeCode: String { + var contextWindowErrorCode: String? { switch self { - case .modelUnavailable: - return "MODEL_UNAVAILABLE" - case .unsupportedOS: - return "UNSUPPORTED_OS" - case .generationError: - return "GENERATION_ERROR" - case .streamNotFound: - return "STREAM_NOT_FOUND" - case .invalidMessage: - return "INVALID_MESSAGE" - case .conflictingSamplingMethods: - return "CONFLICTING_SAMPLING_METHODS" - case .invalidSchema: - return "INVALID_SCHEMA" - case .toolCallError: - return "TOOL_CALL_ERROR" - case .unknownToolCallError: - return "UNKNOWN_TOOL_CALL_ERROR" case .contextWindowExceeded: return "CONTEXT_WINDOW_EXCEEDED" + default: + return nil } } diff --git a/packages/apple-llm/ios/AppleLLMImpl.swift b/packages/apple-llm/ios/AppleLLMImpl.swift index 93d6332f8..3fa462a71 100644 --- a/packages/apple-llm/ios/AppleLLMImpl.swift +++ b/packages/apple-llm/ios/AppleLLMImpl.swift @@ -74,17 +74,21 @@ public class AppleLLMImpl: NSObject { resolve(response.toModelMessages()) } } catch { - let appleError = self.createAppleLLMError(from: error) - reject(appleError.reactNativeCode, appleError.localizedDescription, appleError) + if let appleError = self.createContextWindowError(from: error), + let code = appleError.contextWindowErrorCode { + reject(code, appleError.localizedDescription, appleError) + } else { + reject("AppleLLM", error.localizedDescription, error) + } } } } else { let error = AppleLLMError.unsupportedOS - reject(error.reactNativeCode, error.localizedDescription, error) + reject("AppleLLM", error.localizedDescription, error) } #else let error = AppleLLMError.unsupportedOS - reject(error.reactNativeCode, error.localizedDescription, error) + reject("AppleLLM", error.localizedDescription, error) #endif } @@ -102,7 +106,7 @@ public class AppleLLMImpl: NSObject { let streamId = UUID().uuidString guard SystemLanguageModel.default.availability == .available else { let error = AppleLLMError.modelUnavailable - onError(streamId, error.reactNativeCode, error.localizedDescription) + onError(streamId, "", error.localizedDescription) return streamId } @@ -142,8 +146,12 @@ public class AppleLLMImpl: NSObject { onComplete(streamId) } } catch { - let appleError = self.createAppleLLMError(from: error) - onError(streamId, appleError.reactNativeCode, appleError.localizedDescription) + if let appleError = self.createContextWindowError(from: error), + let code = appleError.contextWindowErrorCode { + onError(streamId, code, appleError.localizedDescription) + } else { + onError(streamId, "", error.localizedDescription) + } } // Clean up task from map when completed @@ -176,16 +184,12 @@ public class AppleLLMImpl: NSObject { #if canImport(FoundationModels) @available(iOS 26, *) - private func createAppleLLMError(from error: Error) -> AppleLLMError { + private func createContextWindowError(from error: Error) -> AppleLLMError? { if case LanguageModelSession.GenerationError.exceededContextWindowSize = error { return .contextWindowExceeded } - if let appleError = error as? AppleLLMError { - return appleError - } - - return .generationError(error.localizedDescription) + return nil } @available(iOS 26, *) diff --git a/packages/apple-llm/src/NativeAppleLLM.ts b/packages/apple-llm/src/NativeAppleLLM.ts index 7741bb1b2..7977f1a26 100644 --- a/packages/apple-llm/src/NativeAppleLLM.ts +++ b/packages/apple-llm/src/NativeAppleLLM.ts @@ -30,7 +30,7 @@ export type StreamCompleteEvent = { export type StreamErrorEvent = { streamId: string - code: string + code?: string error: string } diff --git a/packages/apple-llm/src/errors.ts b/packages/apple-llm/src/errors.ts index d4749098f..87afe7987 100644 --- a/packages/apple-llm/src/errors.ts +++ b/packages/apple-llm/src/errors.ts @@ -4,7 +4,6 @@ export type AppleLLMError = Error & { export const AppleLLMErrorCodes = { ContextWindowExceeded: 'CONTEXT_WINDOW_EXCEEDED', - ModelUnavailable: 'MODEL_UNAVAILABLE', } as const export type AppleLLMErrorCode = From 01ca4d3a815b4296431e55fa0f55292e068e999b Mon Sep 17 00:00:00 2001 From: Julian Kobrynski Date: Fri, 22 May 2026 15:08:59 +0200 Subject: [PATCH 06/11] docs(apple): clarify context overflow recovery --- website/src/docs/apple/generating.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/website/src/docs/apple/generating.md b/website/src/docs/apple/generating.md index 63c2bdb5d..e7b86de36 100644 --- a/website/src/docs/apple/generating.md +++ b/website/src/docs/apple/generating.md @@ -209,9 +209,9 @@ Apple Foundation Models have a fixed context window of 4096 tokens. This limit a The `maxTokens` option only limits how many tokens the model can generate in its response. It does not increase the available context window or reserve enough room for a long prompt. -If the full context is too large, Apple may fail generation with a context-window overflow error. The provider does not automatically remove messages from your prompt, because different apps need different memory strategies. Handle this at the application level by catching the error and choosing the recovery behavior that fits your product: +If the full context is too large, Apple may fail generation with a context-window overflow error. The provider does not automatically estimate tokens, remove messages from your prompt, or retry the request, because token estimates can vary by language and different apps need different memory strategies. Handle this at the application level by catching the error and choosing the recovery behavior that fits your product: -- Start a new conversation without the previous transcript +- Start a new conversation without the previous transcript, which is Apple's recommended baseline after this error - Keep a sliding window of recent messages - Summarize older messages and include the summary instead of the full transcript - Ask the user to shorten the prompt or start a new chat From 51c7f181721d9f251339b8004499af90aa657527 Mon Sep 17 00:00:00 2001 From: Julian Kobrynski Date: Mon, 25 May 2026 10:36:23 +0200 Subject: [PATCH 07/11] fix(apple): downcast generation errors before matching --- packages/apple-llm/ios/AppleLLMImpl.swift | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/apple-llm/ios/AppleLLMImpl.swift b/packages/apple-llm/ios/AppleLLMImpl.swift index 3fa462a71..bb9a6569c 100644 --- a/packages/apple-llm/ios/AppleLLMImpl.swift +++ b/packages/apple-llm/ios/AppleLLMImpl.swift @@ -185,7 +185,11 @@ public class AppleLLMImpl: NSObject { @available(iOS 26, *) private func createContextWindowError(from error: Error) -> AppleLLMError? { - if case LanguageModelSession.GenerationError.exceededContextWindowSize = error { + guard let generationError = error as? LanguageModelSession.GenerationError else { + return nil + } + + if case .exceededContextWindowSize = generationError { return .contextWindowExceeded } From 5d3d46a333229b0ee7658144665c88da51367938 Mon Sep 17 00:00:00 2001 From: Julian Kobrynski Date: Tue, 26 May 2026 12:33:46 +0200 Subject: [PATCH 08/11] feat(apple): expose token counting api --- packages/apple-llm/ios/AppleLLM.mm | 6 ++++ packages/apple-llm/ios/AppleLLMImpl.swift | 34 ++++++++++++++++++ .../apple-llm/src/AppleFoundationModels.ts | 36 +++++++++++++++++++ packages/apple-llm/src/NativeAppleLLM.ts | 1 + packages/apple-llm/src/index.ts | 2 +- website/src/docs/apple/generating.md | 16 +++++++++ 6 files changed, 94 insertions(+), 1 deletion(-) create mode 100644 packages/apple-llm/src/AppleFoundationModels.ts diff --git a/packages/apple-llm/ios/AppleLLM.mm b/packages/apple-llm/ios/AppleLLM.mm index 75aa89c31..4c99c1a86 100644 --- a/packages/apple-llm/ios/AppleLLM.mm +++ b/packages/apple-llm/ios/AppleLLM.mm @@ -125,6 +125,12 @@ - (void)generateText:(nonnull NSArray *)messages [_llm generateText:messages options:opts resolve:resolve reject:reject toolInvoker:callToolBlock]; } +- (void)countTokens:(nonnull NSString *)text + resolve:(nonnull RCTPromiseResolveBlock)resolve + reject:(nonnull RCTPromiseRejectBlock)reject { + [_llm countTokens:text resolve:resolve reject:reject]; +} + - (void)cancelStream:(nonnull NSString *)streamId { [_llm cancelStream:streamId]; } diff --git a/packages/apple-llm/ios/AppleLLMImpl.swift b/packages/apple-llm/ios/AppleLLMImpl.swift index bb9a6569c..af94f93ba 100644 --- a/packages/apple-llm/ios/AppleLLMImpl.swift +++ b/packages/apple-llm/ios/AppleLLMImpl.swift @@ -33,6 +33,40 @@ public class AppleLLMImpl: NSObject { #endif } + @objc + public func countTokens( + _ text: String, + resolve: @escaping (Any?) -> Void, + reject: @escaping (String, String, Error?) -> Void + ) { +#if canImport(FoundationModels) + if #available(iOS 26.4, *) { + guard SystemLanguageModel.default.availability == .available else { + reject( + "MODEL_UNAVAILABLE", + "Apple Intelligence model is not available", + nil + ) + return + } + Task { + do { + let tokenCount = try await SystemLanguageModel.default.tokenCount(for: text) + resolve(tokenCount) + } catch { + reject("AppleLLM", error.localizedDescription, error) + } + } + } else { + let error = AppleLLMError.unsupportedOS + reject("AppleLLM", error.localizedDescription, error) + } +#else + let error = AppleLLMError.unsupportedOS + reject("AppleLLM", error.localizedDescription, error) +#endif + } + @objc public func generateText( _ messages: [[String: Any]], diff --git a/packages/apple-llm/src/AppleFoundationModels.ts b/packages/apple-llm/src/AppleFoundationModels.ts new file mode 100644 index 000000000..4859ffd94 --- /dev/null +++ b/packages/apple-llm/src/AppleFoundationModels.ts @@ -0,0 +1,36 @@ +import { createAppleLLMError } from './errors' +import NativeAppleLLM, { + type AppleGenerationOptions, + type AppleMessage, + type Spec, +} from './NativeAppleLLM' + +const tokenCountingUnavailableMessage = + 'Apple Foundation Models token counting is unavailable. It requires iOS 26.4 or newer and a native build with countTokens support.' + +const nativeAppleLLM = NativeAppleLLM as Spec & { + countTokens?: (text: string) => Promise +} + +const AppleFoundationModels: Spec = { + isAvailable: () => NativeAppleLLM.isAvailable(), + countTokens: (text) => { + if (typeof nativeAppleLLM.countTokens !== 'function') { + return Promise.reject( + createAppleLLMError(tokenCountingUnavailableMessage) + ) + } + + return nativeAppleLLM.countTokens(text) + }, + generateText: (messages: AppleMessage[], options: AppleGenerationOptions) => + NativeAppleLLM.generateText(messages, options), + generateStream: (messages: AppleMessage[], options: AppleGenerationOptions) => + NativeAppleLLM.generateStream(messages, options), + cancelStream: (streamId) => NativeAppleLLM.cancelStream(streamId), + onStreamUpdate: NativeAppleLLM.onStreamUpdate, + onStreamComplete: NativeAppleLLM.onStreamComplete, + onStreamError: NativeAppleLLM.onStreamError, +} + +export default AppleFoundationModels diff --git a/packages/apple-llm/src/NativeAppleLLM.ts b/packages/apple-llm/src/NativeAppleLLM.ts index 7977f1a26..37c412bf4 100644 --- a/packages/apple-llm/src/NativeAppleLLM.ts +++ b/packages/apple-llm/src/NativeAppleLLM.ts @@ -36,6 +36,7 @@ export type StreamErrorEvent = { export interface Spec extends TurboModule { isAvailable(): boolean + countTokens(text: string): Promise generateText( messages: AppleMessage[], options: AppleGenerationOptions diff --git a/packages/apple-llm/src/index.ts b/packages/apple-llm/src/index.ts index 5ddf59daf..0e73c5e56 100644 --- a/packages/apple-llm/src/index.ts +++ b/packages/apple-llm/src/index.ts @@ -1,8 +1,8 @@ export { apple, createAppleProvider } from './ai-sdk' +export { default as AppleFoundationModels } from './AppleFoundationModels' export type { AppleLLMError, AppleLLMErrorCode } from './errors' export { AppleLLMErrorCodes } from './errors' export { default as AppleEmbeddings } from './NativeAppleEmbeddings' -export { default as AppleFoundationModels } from './NativeAppleLLM' export { default as AppleSpeech, VoiceInfo } from './NativeAppleSpeech' export { default as AppleTranscription } from './NativeAppleTranscription' export { default as AppleUtils } from './NativeAppleUtils' diff --git a/website/src/docs/apple/generating.md b/website/src/docs/apple/generating.md index e7b86de36..14f9ee7b0 100644 --- a/website/src/docs/apple/generating.md +++ b/website/src/docs/apple/generating.md @@ -343,3 +343,19 @@ const options = { temperature: 0.7, maxTokens: 100 } const result = await AppleFoundationModels.generateText(messages, options) ``` + +On iOS 26.4 and newer, you can also count the number of tokens in a string +before sending it to the model: + +```tsx +import { AppleFoundationModels } from '@react-native-ai/apple' + +const tokenCount = await AppleFoundationModels.countTokens( + 'Summarize this text in three bullet points.' +) +``` + +Token counting is useful for estimating prompt size, but it is not a complete +guarantee that a generation request will fit in the model context window. The +full context also includes instructions, previous messages in the transcript, +tools, schemas, and generated output. From 458261002f1f1b0b6ad7a5f75b56181941eaf12b Mon Sep 17 00:00:00 2001 From: Julian Kobrynski Date: Tue, 26 May 2026 14:52:50 +0200 Subject: [PATCH 09/11] feat(expo-example): show apple token count in chat input --- .../ChatScreen/AppleTokenCounter.android.tsx | 8 +++ .../ChatScreen/AppleTokenCounter.ios.tsx | 66 +++++++++++++++++++ .../src/screens/ChatScreen/ChatInputBar.tsx | 14 +++- .../src/screens/ChatScreen/ChatMessages.tsx | 8 ++- .../src/screens/ChatScreen/index.tsx | 4 ++ 5 files changed, 98 insertions(+), 2 deletions(-) create mode 100644 apps/expo-example/src/screens/ChatScreen/AppleTokenCounter.android.tsx create mode 100644 apps/expo-example/src/screens/ChatScreen/AppleTokenCounter.ios.tsx diff --git a/apps/expo-example/src/screens/ChatScreen/AppleTokenCounter.android.tsx b/apps/expo-example/src/screens/ChatScreen/AppleTokenCounter.android.tsx new file mode 100644 index 000000000..f91db1b2c --- /dev/null +++ b/apps/expo-example/src/screens/ChatScreen/AppleTokenCounter.android.tsx @@ -0,0 +1,8 @@ +type AppleTokenCounterProps = { + input: string + isVisible: boolean +} + +export function AppleTokenCounter(_props: AppleTokenCounterProps) { + return null +} diff --git a/apps/expo-example/src/screens/ChatScreen/AppleTokenCounter.ios.tsx b/apps/expo-example/src/screens/ChatScreen/AppleTokenCounter.ios.tsx new file mode 100644 index 000000000..406c172de --- /dev/null +++ b/apps/expo-example/src/screens/ChatScreen/AppleTokenCounter.ios.tsx @@ -0,0 +1,66 @@ +import { AppleFoundationModels } from '@react-native-ai/apple' +import React, { useEffect, useState } from 'react' +import { StyleSheet, Text } from 'react-native' + +import { colors } from '../../theme/colors' + +const APPLE_CONTEXT_WINDOW_TOKENS = 4096 +const TOKEN_COUNT_DEBOUNCE_MS = 250 + +type AppleTokenCounterProps = { + input: string + isVisible: boolean +} + +export function AppleTokenCounter({ + input, + isVisible, +}: AppleTokenCounterProps) { + const [tokenCount, setTokenCount] = useState(0) + + useEffect(() => { + if (!isVisible) { + setTokenCount(null) + return + } + + if (!input.trim()) { + setTokenCount(0) + return + } + + let isActive = true + const timeoutId = setTimeout(() => { + AppleFoundationModels.countTokens(input) + .then((count) => { + if (isActive) setTokenCount(count) + }) + .catch(() => { + if (isActive) setTokenCount(null) + }) + }, TOKEN_COUNT_DEBOUNCE_MS) + + return () => { + isActive = false + clearTimeout(timeoutId) + } + }, [input, isVisible]) + + if (!isVisible || tokenCount === null) return null + + return ( + + {tokenCount}/{APPLE_CONTEXT_WINDOW_TOKENS} + + ) +} + +const styles = StyleSheet.create({ + tokenCounter: { + color: colors.secondaryLabel as any, + fontSize: 12, + lineHeight: 16, + textAlign: 'right', + marginBottom: 6, + }, +}) diff --git a/apps/expo-example/src/screens/ChatScreen/ChatInputBar.tsx b/apps/expo-example/src/screens/ChatScreen/ChatInputBar.tsx index 0bb571bc1..f9a502d09 100644 --- a/apps/expo-example/src/screens/ChatScreen/ChatInputBar.tsx +++ b/apps/expo-example/src/screens/ChatScreen/ChatInputBar.tsx @@ -7,13 +7,19 @@ import { AdaptiveGlass } from '../../components/AdaptiveGlass' import { Button, ContextMenu, Host } from '../../components/expo-ui' import { RecordButton } from '../../components/RecordButton' import { colors } from '../../theme/colors' +import { AppleTokenCounter } from './AppleTokenCounter' type ChatInputBarProps = { onSend: (message: string) => void isGenerating: boolean + showAppleTokenCount: boolean } -export function ChatInputBar({ onSend, isGenerating }: ChatInputBarProps) { +export function ChatInputBar({ + onSend, + isGenerating, + showAppleTokenCount, +}: ChatInputBarProps) { const [input, setInput] = useState('') const handleSend = () => { @@ -25,6 +31,9 @@ export function ChatInputBar({ onSend, isGenerating }: ChatInputBarProps) { return ( + + + @@ -131,6 +140,9 @@ const styles = StyleSheet.create({ paddingHorizontal: 16, minHeight: 40, }, + tokenCounterContainer: { + alignItems: 'center', + }, textInput: { flex: 1, fontSize: 17, diff --git a/apps/expo-example/src/screens/ChatScreen/ChatMessages.tsx b/apps/expo-example/src/screens/ChatScreen/ChatMessages.tsx index 1b670ea88..ee5b3f5f2 100644 --- a/apps/expo-example/src/screens/ChatScreen/ChatMessages.tsx +++ b/apps/expo-example/src/screens/ChatScreen/ChatMessages.tsx @@ -32,6 +32,7 @@ type ChatMessagesProps = { onSend: (message: string) => void isGenerating: boolean genUiEnabled: boolean + showAppleTokenCount: boolean } export function ChatMessages({ @@ -40,6 +41,7 @@ export function ChatMessages({ onSend, isGenerating, genUiEnabled, + showAppleTokenCount, }: ChatMessagesProps) { const ref = useRef(null) @@ -117,7 +119,11 @@ export function ChatMessages({ )} - + ) diff --git a/apps/expo-example/src/screens/ChatScreen/index.tsx b/apps/expo-example/src/screens/ChatScreen/index.tsx index 4a18f411e..6dd84fcb7 100644 --- a/apps/expo-example/src/screens/ChatScreen/index.tsx +++ b/apps/expo-example/src/screens/ChatScreen/index.tsx @@ -182,6 +182,9 @@ export default function ChatScreen() { }, [selectedAdapter, selectedModelAvailability]) const headerSubtitle = selectedAdapter?.display.label ?? 'No model selected' + const showAppleTokenCount = + selectedAdapter?.model.provider === 'apple' && + selectedModelAvailability === 'yes' return ( @@ -205,6 +208,7 @@ export default function ChatScreen() { isGenerating={isGenerating} selectedModelLabel={selectedAdapter.display.label} genUiEnabled={genUiEnabled} + showAppleTokenCount={showAppleTokenCount} /> )} From 1ae1bc6529f5a9ac3a57f6b441eddcd8523cd242 Mon Sep 17 00:00:00 2001 From: Julian Kobrynski Date: Tue, 26 May 2026 15:18:05 +0200 Subject: [PATCH 10/11] fix(expo-example): adjust apple token counter alignment --- .../src/screens/ChatScreen/AppleTokenCounter.ios.tsx | 1 - 1 file changed, 1 deletion(-) diff --git a/apps/expo-example/src/screens/ChatScreen/AppleTokenCounter.ios.tsx b/apps/expo-example/src/screens/ChatScreen/AppleTokenCounter.ios.tsx index 406c172de..3dbdb9c9b 100644 --- a/apps/expo-example/src/screens/ChatScreen/AppleTokenCounter.ios.tsx +++ b/apps/expo-example/src/screens/ChatScreen/AppleTokenCounter.ios.tsx @@ -60,7 +60,6 @@ const styles = StyleSheet.create({ color: colors.secondaryLabel as any, fontSize: 12, lineHeight: 16, - textAlign: 'right', marginBottom: 6, }, }) From 22bc7cf369b5b02fe824132303cd6535b1f579d8 Mon Sep 17 00:00:00 2001 From: Artur Morys-Magiera Date: Mon, 1 Jun 2026 14:26:32 +0200 Subject: [PATCH 11/11] docs: context window size information for Apple models --- website/src/docs/apple/generating.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/website/src/docs/apple/generating.md b/website/src/docs/apple/generating.md index 14f9ee7b0..9949edfbf 100644 --- a/website/src/docs/apple/generating.md +++ b/website/src/docs/apple/generating.md @@ -359,3 +359,6 @@ Token counting is useful for estimating prompt size, but it is not a complete guarantee that a generation request will fit in the model context window. The full context also includes instructions, previous messages in the transcript, tools, schemas, and generated output. + +The maximum context window size for Apple's Foundation models is 4096 tokens +per session. More information can be found [here](https://developer.apple.com/documentation/technotes/tn3193-managing-the-on-device-foundation-model-s-context-window).