Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
type AppleTokenCounterProps = {
input: string
isVisible: boolean
}

export function AppleTokenCounter(_props: AppleTokenCounterProps) {
return null
}
65 changes: 65 additions & 0 deletions apps/expo-example/src/screens/ChatScreen/AppleTokenCounter.ios.tsx
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
import { AppleFoundationModels } from '@react-native-ai/apple'
import React, { useEffect, useState } from 'react'
import { StyleSheet, Text } from 'react-native'

import { colors } from '../../theme/colors'

const APPLE_CONTEXT_WINDOW_TOKENS = 4096
const TOKEN_COUNT_DEBOUNCE_MS = 250

type AppleTokenCounterProps = {
input: string
isVisible: boolean
}

export function AppleTokenCounter({
input,
isVisible,
}: AppleTokenCounterProps) {
const [tokenCount, setTokenCount] = useState<number | null>(0)

useEffect(() => {
if (!isVisible) {
setTokenCount(null)
return
}

if (!input.trim()) {
setTokenCount(0)
return
}

let isActive = true
const timeoutId = setTimeout(() => {
AppleFoundationModels.countTokens(input)
.then((count) => {
if (isActive) setTokenCount(count)
})
.catch(() => {
if (isActive) setTokenCount(null)
})
}, TOKEN_COUNT_DEBOUNCE_MS)

return () => {
isActive = false
clearTimeout(timeoutId)
}
}, [input, isVisible])

if (!isVisible || tokenCount === null) return null

return (
<Text style={styles.tokenCounter}>
{tokenCount}/{APPLE_CONTEXT_WINDOW_TOKENS}
</Text>
)
}

const styles = StyleSheet.create({
tokenCounter: {
color: colors.secondaryLabel as any,
fontSize: 12,
lineHeight: 16,
marginBottom: 6,
},
})
14 changes: 13 additions & 1 deletion apps/expo-example/src/screens/ChatScreen/ChatInputBar.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -7,13 +7,19 @@ import { AdaptiveGlass } from '../../components/AdaptiveGlass'
import { Button, ContextMenu, Host } from '../../components/expo-ui'
import { RecordButton } from '../../components/RecordButton'
import { colors } from '../../theme/colors'
import { AppleTokenCounter } from './AppleTokenCounter'

type ChatInputBarProps = {
onSend: (message: string) => void
isGenerating: boolean
showAppleTokenCount: boolean
}

export function ChatInputBar({ onSend, isGenerating }: ChatInputBarProps) {
export function ChatInputBar({
onSend,
isGenerating,
showAppleTokenCount,
}: ChatInputBarProps) {
const [input, setInput] = useState('')

const handleSend = () => {
Expand All @@ -25,6 +31,9 @@ export function ChatInputBar({ onSend, isGenerating }: ChatInputBarProps) {

return (
<AdaptiveGlass style={styles.inputBar}>
<View style={styles.tokenCounterContainer}>
<AppleTokenCounter input={input} isVisible={showAppleTokenCount} />
</View>
<View style={styles.inputRow}>
<AdaptiveGlass isInteractive style={styles.plusButton}>
<Host matchContents>
Expand Down Expand Up @@ -131,6 +140,9 @@ const styles = StyleSheet.create({
paddingHorizontal: 16,
minHeight: 40,
},
tokenCounterContainer: {
alignItems: 'center',
},
textInput: {
flex: 1,
fontSize: 17,
Expand Down
8 changes: 7 additions & 1 deletion apps/expo-example/src/screens/ChatScreen/ChatMessages.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ type ChatMessagesProps = {
onSend: (message: string) => void
isGenerating: boolean
genUiEnabled: boolean
showAppleTokenCount: boolean
}

export function ChatMessages({
Expand All @@ -40,6 +41,7 @@ export function ChatMessages({
onSend,
isGenerating,
genUiEnabled,
showAppleTokenCount,
}: ChatMessagesProps) {
const ref = useRef<ScrollView>(null)

Expand Down Expand Up @@ -117,7 +119,11 @@ export function ChatMessages({
)}
</ScrollView>
<Reanimated.View style={inputBarBottomPadding}>
<ChatInputBar onSend={onSend} isGenerating={isGenerating} />
<ChatInputBar
onSend={onSend}
isGenerating={isGenerating}
showAppleTokenCount={showAppleTokenCount}
/>
</Reanimated.View>
</>
)
Expand Down
10 changes: 10 additions & 0 deletions apps/expo-example/src/screens/ChatScreen/index.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,7 @@ export default function ChatScreen() {
setToolExecutionReporter(({ toolName, args, result }) => {
addToolExecutionMessage(chatId, toolName, args, result)
})
let streamError: unknown
const result = streamText({
model: selectedAdapter.model,
messages: [
Expand All @@ -129,6 +130,9 @@ export default function ChatScreen() {
'If the user asks, tell who you are (assistant) and what is this (Callstack AI demo app).',
})
: 'You are a helpful assistant. If the user asks, tell who you are (assistant) and what is this (Callstack AI demo app).',
onError: ({ error }) => {
streamError = error
},
})

let accumulated = ''
Expand All @@ -139,6 +143,8 @@ export default function ChatScreen() {
updateMessageContent(chatId, assistantMessageId, accumulated)
}

if (streamError) throw streamError

if (accumulated.trim().length === 0) {
updateMessageContent(
chatId,
Expand Down Expand Up @@ -176,6 +182,9 @@ export default function ChatScreen() {
}, [selectedAdapter, selectedModelAvailability])

const headerSubtitle = selectedAdapter?.display.label ?? 'No model selected'
const showAppleTokenCount =
selectedAdapter?.model.provider === 'apple' &&
selectedModelAvailability === 'yes'

return (
<SafeAreaView style={styles.container} edges={['top']}>
Expand All @@ -199,6 +208,7 @@ export default function ChatScreen() {
isGenerating={isGenerating}
selectedModelLabel={selectedAdapter.display.label}
genUiEnabled={genUiEnabled}
showAppleTokenCount={showAppleTokenCount}
/>
)}
</View>
Expand Down
14 changes: 12 additions & 2 deletions packages/apple-llm/ios/AppleLLM.mm
Original file line number Diff line number Diff line change
Expand Up @@ -125,6 +125,12 @@ - (void)generateText:(nonnull NSArray *)messages
[_llm generateText:messages options:opts resolve:resolve reject:reject toolInvoker:callToolBlock];
}

- (void)countTokens:(nonnull NSString *)text
resolve:(nonnull RCTPromiseResolveBlock)resolve
reject:(nonnull RCTPromiseRejectBlock)reject {
[_llm countTokens:text resolve:resolve reject:reject];
}

- (void)cancelStream:(nonnull NSString *)streamId {
[_llm cancelStream:streamId];
}
Expand Down Expand Up @@ -155,8 +161,12 @@ - (nonnull NSString *)generateStream:(nonnull NSArray *)messages options:(JS::Na
onComplete:^(NSString *streamId) {
[self emitOnStreamComplete:@{@"streamId": streamId}];
}
onError:^(NSString *streamId, NSString *error) {
[self emitOnStreamError:@{@"streamId": streamId, @"error": error}];
onError:^(NSString *streamId, NSString *code, NSString *error) {
NSMutableDictionary *payload = [@{@"streamId": streamId, @"error": error} mutableCopy];
if (code.length > 0) {
payload[@"code"] = code;
}
[self emitOnStreamError:payload];
}
toolInvoker:callToolBlock];

Expand Down
13 changes: 13 additions & 0 deletions packages/apple-llm/ios/AppleLLMError.swift
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ enum AppleLLMError: Error, LocalizedError {
case invalidSchema(String)
case toolCallError(Error)
case unknownToolCallError
case contextWindowExceeded

var errorDescription: String? {
switch self {
Expand All @@ -38,9 +39,20 @@ enum AppleLLMError: Error, LocalizedError {
return "Error calling tool: \(error.localizedDescription)"
case .unknownToolCallError:
return "Unknown tool call error"
case .contextWindowExceeded:
return "Context window exceeded"
}

}

var contextWindowErrorCode: String? {
switch self {
case .contextWindowExceeded:
return "CONTEXT_WINDOW_EXCEEDED"
default:
return nil
}
}

var code: Int {
switch self {
Expand All @@ -53,6 +65,7 @@ enum AppleLLMError: Error, LocalizedError {
case .invalidSchema: return 7
case .unknownToolCallError: return 8
case .toolCallError: return 9
case .contextWindowExceeded: return 10
}
}
}
66 changes: 62 additions & 4 deletions packages/apple-llm/ios/AppleLLMImpl.swift
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,40 @@ public class AppleLLMImpl: NSObject {
#endif
}

@objc
public func countTokens(
_ text: String,
resolve: @escaping (Any?) -> Void,
reject: @escaping (String, String, Error?) -> Void
) {
#if canImport(FoundationModels)
if #available(iOS 26.4, *) {
guard SystemLanguageModel.default.availability == .available else {
reject(
"MODEL_UNAVAILABLE",
"Apple Intelligence model is not available",
nil
)
return
}
Task {
do {
let tokenCount = try await SystemLanguageModel.default.tokenCount(for: text)
resolve(tokenCount)
} catch {
reject("AppleLLM", error.localizedDescription, error)
}
}
} else {
let error = AppleLLMError.unsupportedOS
reject("AppleLLM", error.localizedDescription, error)
}
#else
let error = AppleLLMError.unsupportedOS
reject("AppleLLM", error.localizedDescription, error)
#endif
}

@objc
public func generateText(
_ messages: [[String: Any]],
Expand Down Expand Up @@ -74,7 +108,12 @@ public class AppleLLMImpl: NSObject {
resolve(response.toModelMessages())
}
} catch {
reject("AppleLLM", error.localizedDescription, error)
if let appleError = self.createContextWindowError(from: error),
let code = appleError.contextWindowErrorCode {
reject(code, appleError.localizedDescription, appleError)
} else {
reject("AppleLLM", error.localizedDescription, error)
}
}
}
} else {
Expand All @@ -93,14 +132,15 @@ public class AppleLLMImpl: NSObject {
options: [String: Any],
onUpdate: @escaping (String, String) -> Void,
onComplete: @escaping (String) -> Void,
onError: @escaping (String, String) -> Void,
onError: @escaping (String, String, String) -> Void,
toolInvoker: @escaping ToolInvoker
) throws -> String {
#if canImport(FoundationModels)
if #available(iOS 26, *) {
let streamId = UUID().uuidString
guard SystemLanguageModel.default.availability == .available else {
onError(streamId, "Apple Intelligence model is not available")
let error = AppleLLMError.modelUnavailable
onError(streamId, "", error.localizedDescription)
return streamId
}

Expand Down Expand Up @@ -140,7 +180,12 @@ public class AppleLLMImpl: NSObject {
onComplete(streamId)
}
} catch {
onError(streamId, error.localizedDescription)
if let appleError = self.createContextWindowError(from: error),
let code = appleError.contextWindowErrorCode {
onError(streamId, code, appleError.localizedDescription)
} else {
onError(streamId, "", error.localizedDescription)
}
}

// Clean up task from map when completed
Expand Down Expand Up @@ -171,6 +216,19 @@ public class AppleLLMImpl: NSObject {

// MARK: - Private Methods
#if canImport(FoundationModels)

@available(iOS 26, *)
private func createContextWindowError(from error: Error) -> AppleLLMError? {
guard let generationError = error as? LanguageModelSession.GenerationError else {
return nil
}

if case .exceededContextWindowSize = generationError {
return .contextWindowExceeded
}

return nil
}

@available(iOS 26, *)
private func createTools(from options: [String: Any], toolInvoker: @escaping ToolInvoker) throws -> [any Tool] {
Expand Down
36 changes: 36 additions & 0 deletions packages/apple-llm/src/AppleFoundationModels.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
import { createAppleLLMError } from './errors'
import NativeAppleLLM, {
type AppleGenerationOptions,
type AppleMessage,
type Spec,
} from './NativeAppleLLM'

const tokenCountingUnavailableMessage =
'Apple Foundation Models token counting is unavailable. It requires iOS 26.4 or newer and a native build with countTokens support.'

const nativeAppleLLM = NativeAppleLLM as Spec & {
countTokens?: (text: string) => Promise<number>
}

const AppleFoundationModels: Spec = {
isAvailable: () => NativeAppleLLM.isAvailable(),
countTokens: (text) => {
if (typeof nativeAppleLLM.countTokens !== 'function') {
return Promise.reject(
createAppleLLMError(tokenCountingUnavailableMessage)
)
}

return nativeAppleLLM.countTokens(text)
},
generateText: (messages: AppleMessage[], options: AppleGenerationOptions) =>
NativeAppleLLM.generateText(messages, options),
generateStream: (messages: AppleMessage[], options: AppleGenerationOptions) =>
NativeAppleLLM.generateStream(messages, options),
cancelStream: (streamId) => NativeAppleLLM.cancelStream(streamId),
onStreamUpdate: NativeAppleLLM.onStreamUpdate,
onStreamComplete: NativeAppleLLM.onStreamComplete,
onStreamError: NativeAppleLLM.onStreamError,
}

export default AppleFoundationModels
Loading
Loading