Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 6 additions & 6 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,7 @@ const { text } = await generateText({

// Generate embeddings
const { embedding } = await embed({
model: apple.textEmbeddingModel(),
model: apple.embeddingModel(),
value: 'Hello world',
})

Expand Down Expand Up @@ -166,11 +166,11 @@ Run any GGUF model on-device using [llama.rn](https://github.com/mybigday/llama.

#### Supported Features

| Feature | Method | Description |
| --------------- | ---------------------------- | --------------------------------------------- |
| Text Generation | `llama.languageModel()` | Chat, completion, streaming, reasoning models |
| Embeddings | `llama.textEmbeddingModel()` | Text embeddings for RAG and similarity search |
| Speech | `llama.speechModel()` | Text-to-speech with vocoder models |
| Feature | Method | Description |
| --------------- | ------------------------ | --------------------------------------------- |
| Text Generation | `llama.languageModel()` | Chat, completion, streaming, reasoning models |
| Embeddings | `llama.embeddingModel()` | Text embeddings for RAG and similarity search |
| Speech | `llama.speechModel()` | Text-to-speech with vocoder models |

#### Installation

Expand Down
2 changes: 2 additions & 0 deletions packages/adk/src/ai-sdk.ts
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@ async function checkNanoSupported(): Promise<boolean> {

export interface AdkProvider {
(): LanguageModelV4
readonly specificationVersion: 'v4'
languageModel: () => LanguageModelV4
/** Device capability check (ML Kit `checkStatus !== 0`). Returns `false` on non-Android. */
isNanoSupported: () => Promise<boolean>
Expand All @@ -79,6 +80,7 @@ export function createAdkProvider(
const provider = function () {
return createLanguageModel()
}
provider.specificationVersion = 'v4' as const
provider.languageModel = createLanguageModel
provider.isNanoSupported = () => checkNanoSupported()
provider.isAvailable = async (modelType: AdkModelType = 'genai-nano') => {
Expand Down
8 changes: 6 additions & 2 deletions packages/apple-llm/ios/AppleLLM.mm
Original file line number Diff line number Diff line change
Expand Up @@ -208,8 +208,12 @@ - (void)generateStream:(nonnull NSString *)streamId messages:(nonnull NSArray *)
}
[self emitOnStreamUpdate:payload];
}
onComplete:^(NSString *streamId) {
[self emitOnStreamComplete:@{@"streamId": streamId}];
onComplete:^(NSString *streamId, NSDictionary *usage) {
NSMutableDictionary *payload = [@{@"streamId": streamId} mutableCopy];
if (usage != nil) {
payload[@"usage"] = usage;
}
[self emitOnStreamComplete:payload];
}
onError:^(NSString *streamId, NSString *code, NSString *error) {
NSMutableDictionary *payload = [@{@"streamId": streamId, @"error": error} mutableCopy];
Expand Down
171 changes: 135 additions & 36 deletions packages/apple-llm/ios/AppleLLMImpl.swift
Original file line number Diff line number Diff line change
Expand Up @@ -254,7 +254,7 @@ public class AppleLLMImpl: NSObject {
messages: [[String: Any]],
options: [String: Any],
onUpdate: @escaping (String, String, [[String: Any]]?) -> Void,
onComplete: @escaping (String) -> Void,
onComplete: @escaping (String, [String: Any]?) -> Void,
onError: @escaping (String, String, String) -> Void,
toolInvoker: @escaping ToolInvoker
) {
Expand All @@ -275,7 +275,7 @@ public class AppleLLMImpl: NSObject {
let generationSchema = try self.createGenerationSchema(from: options)

do {
try await self.streamResponse(
let usage = try await self.streamResponse(
with: session,
userPrompt: userPrompt,
promptAttachments: promptAttachments,
Expand All @@ -287,7 +287,7 @@ public class AppleLLMImpl: NSObject {
}

if !Task.isCancelled {
onComplete(streamId)
onComplete(streamId, usage)
}
} catch {
if Task.isCancelled {
Expand Down Expand Up @@ -525,7 +525,7 @@ public class AppleLLMImpl: NSObject {
generationSchema: GenerationSchema?,
generationOptions: GenerationOptions,
providerOptions: [String: Any]
) async throws -> [[String: Any]] {
) async throws -> [String: Any] {
#if compiler(>=6.4)
if #available(iOS 27, *) {
let contextOptions = try createContextOptions(
Expand All @@ -548,7 +548,7 @@ public class AppleLLMImpl: NSObject {
attachment
}
}
return response.toModelMessages()
return Self.generationResult(content: response.toModelMessages(), usage: Self.usageDictionary(response.usage))
}

let response = try await session.respond(
Expand All @@ -560,7 +560,7 @@ public class AppleLLMImpl: NSObject {
attachment
}
}
return response.toModelMessages()
return Self.generationResult(content: response.toModelMessages(), usage: Self.usageDictionary(response.usage))
}
#endif

Expand All @@ -569,18 +569,80 @@ public class AppleLLMImpl: NSObject {
throw AppleLLMError.unsupportedOS
}

let transcriptBefore = session.transcript
let content: [[String: Any]]
if let generationSchema {
let response = try await session.respond(
to: userPrompt,
schema: generationSchema,
includeSchemaInPrompt: true,
options: generationOptions
)
return response.toModelMessages()
content = response.toModelMessages()
} else {
let response = try await session.respond(to: userPrompt, options: generationOptions)
content = response.toModelMessages()
}

let response = try await session.respond(to: userPrompt, options: generationOptions)
return response.toModelMessages()
let usage = await countUsage(
session: session,
transcriptBefore: transcriptBefore,
userPrompt: userPrompt,
providerOptions: providerOptions
)
return Self.generationResult(content: content, usage: usage)
}

private static func generationResult(content: [[String: Any]], usage: [String: Any]?) -> [String: Any] {
var result: [String: Any] = ["content": content]
if let usage {
result["usage"] = usage
}
return result
}

#if compiler(>=6.4)
@available(iOS 27, *)
private static func usageDictionary(_ usage: LanguageModelSession.Usage) -> [String: Any] {
return [
"inputTokens": usage.input.totalTokenCount,
"cachedInputTokens": usage.input.cachedTokenCount,
"outputTokens": usage.output.totalTokenCount,
"reasoningTokens": usage.output.reasoningTokenCount,
]
}
#endif

/// iOS 26 responses report no usage, so the system model's tokenizer (iOS 26.4)
/// counts the transcript before and after the response. Private Cloud Compute
/// has no tokenizer, so its usage stays unknown
@available(iOS 26, *)
private func countUsage(
session: LanguageModelSession,
transcriptBefore: Transcript,
userPrompt: String,
providerOptions: [String: Any]
) async -> [String: Any]? {
#if compiler(>=6.3)
guard #available(iOS 26.4, *), (providerOptions["model"] as? String ?? "system") == "system" else {
return nil
}

let model = SystemLanguageModel.default
guard let historyTokens = try? await model.tokenCount(for: transcriptBefore),
let promptTokens = try? await model.tokenCount(for: userPrompt),
let totalTokens = try? await model.tokenCount(for: session.transcript) else {
return nil
}

let inputTokens = historyTokens + promptTokens
return [
"inputTokens": inputTokens,
"outputTokens": max(totalTokens - inputTokens, 0),
]
#else
return nil
#endif
}

@available(iOS 26, *)
Expand All @@ -592,7 +654,20 @@ public class AppleLLMImpl: NSObject {
generationOptions: GenerationOptions,
providerOptions: [String: Any],
onUpdate: @escaping (String, [[String: Any]]?) -> Void
) async throws {
) async throws -> [String: Any]? {
// Reasoning and tool activity are only sent when they change
var previousContent: String?
var previousEntries: NSArray = []
func update(_ content: String, _ entries: [[String: Any]]) {
let entriesChanged = !(entries as NSArray).isEqual(previousEntries)
guard entriesChanged || content != previousContent else {
return
}
previousContent = content
previousEntries = entries as NSArray
onUpdate(content, entriesChanged ? entries : nil)
}

#if compiler(>=6.4)
if #available(iOS 27, *) {
let contextOptions = try createContextOptions(
Expand All @@ -604,6 +679,7 @@ public class AppleLLMImpl: NSObject {
Self.removeTemporaryFiles(preparedAttachments.temporaryFileURLs)
}

var usage: LanguageModelSession.Usage?
if let generationSchema {
let responseStream = session.streamResponse(
schema: generationSchema,
Expand All @@ -616,29 +692,31 @@ public class AppleLLMImpl: NSObject {
}
}
for try await chunk in responseStream {
onUpdate(String(describing: chunk.content), nil)
usage = chunk.usage
update(
chunk.rawContent.jsonString,
AppleLLMTranscriptSerializer.serialize(chunk.transcriptEntries, includeResponses: false)
)
}
return
}

let responseStream = session.streamResponse(
options: generationOptions,
contextOptions: contextOptions
) {
userPrompt
for attachment in preparedAttachments.values {
attachment
} else {
let responseStream = session.streamResponse(
options: generationOptions,
contextOptions: contextOptions
) {
userPrompt
for attachment in preparedAttachments.values {
attachment
}
}
for try await chunk in responseStream {
usage = chunk.usage
update(
chunk.content,
AppleLLMTranscriptSerializer.serialize(chunk.transcriptEntries, includeResponses: false)
)
}
}
// Reasoning and tool activity are only sent when they change
var previousEntries: NSArray = []
for try await chunk in responseStream {
let entries = AppleLLMTranscriptSerializer.serialize(chunk.transcriptEntries, includeResponses: false)
let entriesChanged = !(entries as NSArray).isEqual(previousEntries)
previousEntries = entries as NSArray
onUpdate(chunk.content, entriesChanged ? entries : nil)
}
return
return usage.map(Self.usageDictionary)
}
#endif

Expand All @@ -647,6 +725,17 @@ public class AppleLLMImpl: NSObject {
throw AppleLLMError.unsupportedOS
}

// iOS 26 snapshots have no transcript entries, so tool activity is read from
// the session transcript, which grows while the response streams
let transcriptBefore = session.transcript
let newEntries = {
AppleLLMTranscriptSerializer.serialize(
session.transcript.dropFirst(transcriptBefore.count),
includeResponses: false
)
}

var content = ""
if let generationSchema {
let responseStream = session.streamResponse(
to: userPrompt,
Expand All @@ -655,15 +744,25 @@ public class AppleLLMImpl: NSObject {
options: generationOptions
)
for try await chunk in responseStream {
onUpdate(String(describing: chunk.content), nil)
content = chunk.rawContent.jsonString
update(content, newEntries())
}
} else {
let responseStream = session.streamResponse(to: userPrompt, options: generationOptions)
for try await chunk in responseStream {
content = chunk.content
update(content, newEntries())
}
return
}
// Entries added after the last snapshot
update(content, newEntries())

let responseStream = session.streamResponse(to: userPrompt, options: generationOptions)
for try await chunk in responseStream {
onUpdate(chunk.content, nil)
}
return await countUsage(
session: session,
transcriptBefore: transcriptBefore,
userPrompt: userPrompt,
providerOptions: providerOptions
)
}

#if compiler(>=6.4)
Expand Down
34 changes: 0 additions & 34 deletions packages/apple-llm/src/AppleFoundationModels.ts

This file was deleted.

Loading
Loading