From e49064516f52567cac257a0741a2521d94e95362 Mon Sep 17 00:00:00 2001 From: omesser Date: Wed, 2 Sep 2026 15:04:02 +0300 Subject: [PATCH 1/5] Measure the reply-contract failure rate against a live local model (#175) Co-Authored-By: Claude Opus 5 --- src-tauri/src/model.rs | 143 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 143 insertions(+) diff --git a/src-tauri/src/model.rs b/src-tauri/src/model.rs index b57ae3198..f33d46a92 100644 --- a/src-tauri/src/model.rs +++ b/src-tauri/src/model.rs @@ -1520,4 +1520,147 @@ mod tests { "the abandoned Wake must not land on the new Character" ); } + + /// #175: how often a live local model breaks the reply contract, as the + /// before number #144 argues from. Ignored because it needs a server and + /// spends real seconds; it is the harness, not a check of our own code. + /// + /// The classifier is `ModelDirector::wake` itself rather than a copy of + /// it, so the measurement cannot drift from what the app actually does: + /// a proposal naming a declared Behavior is accepted, an empty name is + /// `as_speech` catching prose, and `Failed` is the turn `StaticDirector` + /// takes. One session throughout, because that is how the buddy runs. + /// + /// ```sh + /// AI_BUDDY_DIRECTOR_BASE_URL=http://localhost:11434 \ + /// AI_BUDDY_DIRECTOR_MODEL=gemma4 \ + /// cargo test -p ai-buddy measure_the_reply_contract -- --ignored --nocapture + /// ``` + #[test] + #[ignore] + fn measure_the_reply_contract_failure_rate() { + use ai_buddy_core::director::{Context, Happened, ModelDirector, Wake}; + use ai_buddy_core::engine::State; + use ai_buddy_core::sensing::Activity; + use std::path::Path; + use std::time::{Instant, SystemTime}; + + const WAKES: usize = 40; + + let endpoint = endpoint().expect("AI_BUDDY_DIRECTOR_BASE_URL and _MODEL in the env"); + let model = endpoint.model().to_string(); + let origin = endpoint.origin(); + + // A real shipped package, so the prompt is the one production sends: + // its Personality Prompt and its declared Behavior names. + let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../characters/cat"); + let files = crate::package::read(&root).expect("the shipped cat package reads"); + let cat = ai_buddy_core::character::load(&files).expect("and loads"); + let behaviors: Vec = cat.behaviors.keys().cloned().collect(); + + let director = ModelDirector::new(endpoint, behaviors.clone()); + + // Vary the wake so the prompts differ: the reactive verbs plus ambient. + let occasions = [ + (Happened::Poke, State::Grounded, "the display floor"), + (Happened::Throw, State::Falling, "nothing"), + (Happened::Summon, State::Grounded, "a Terminal window"), + (Happened::Perch, State::Perched, "a Safari window"), + (Happened::Ambient, State::Grounded, "the top of the Dock"), + ]; + + let (mut accepted, mut speech, mut failed) = (0usize, 0usize, 0usize); + // A reply whose first line names a declared Behavior in the wrong + // case is the contract kept and our matcher refusing it: `knows` + // compares exactly. Counting it apart separates what the model got + // wrong from what we do. + let mut case_only = 0usize; + let mut examples: Vec = Vec::new(); + let started = Instant::now(); + + for turn in 0..WAKES { + let (happened, state, standing) = &occasions[turn % occasions.len()]; + let context = Context { + activity: Activity { + frontmost_application: Some("Terminal".to_string()), + switched: turn % 3 == 0, + idle: Duration::from_secs((turn as u64 % 7) * 30), + at: SystemTime::now(), + hour: 9 + (turn as u8 % 12), + minute: ((turn as u32 * 7) % 60) as u8, + displays_asleep: false, + }, + recent: Vec::new(), + personality: cat.personality.clone(), + state: *state, + happened: *happened, + standing: standing.to_string(), + }; + + match director.wake(&context) { + Wake::Proposed(proposal) if !proposal.behavior.is_empty() => { + accepted += 1; + } + Wake::Proposed(proposal) => { + // `as_speech` hands back the whole reply, so its first + // line is the name the model actually offered. + let said = proposal.dialogue.unwrap_or_default(); + let offered = said + .lines() + .next() + .unwrap_or_default() + .trim() + .trim_end_matches(['.', ':', '!']) + .to_string(); + let near = behaviors + .iter() + .any(|declared| declared.eq_ignore_ascii_case(&offered)); + if near { + case_only += 1; + } else { + speech += 1; + } + if examples.len() < 6 { + let tag = if near { "case-only" } else { "speech" }; + examples.push(format!(" {tag}: {}", said.replace('\n', " | "))); + } + } + Wake::Failed => { + failed += 1; + if examples.len() < 5 { + examples.push(" failed: unparsable or transport error".to_string()); + } + } + } + } + + let percent = |n: usize| (n as f64) * 100.0 / (WAKES as f64); + println!("\n#175 reply-contract outcomes over {WAKES} wakes"); + println!(" model: {model} at {origin}"); + println!(" behaviors: {}", behaviors.join(", ")); + println!(" elapsed: {:.0}s", started.elapsed().as_secs_f64()); + println!(" accepted: {accepted:>3} ({:.0}%)", percent(accepted)); + println!( + " case-only: {case_only:>3} ({:.0}%) contract kept, matcher refused", + percent(case_only) + ); + println!( + " speech: {speech:>3} ({:.0}%) genuine prose", + percent(speech) + ); + println!(" failed: {failed:>3} ({:.0}%)", percent(failed)); + println!( + " the model broke the contract on {:.0}% of wakes", + percent(speech + failed) + ); + for line in &examples { + println!("{line}"); + } + + assert_eq!( + accepted + case_only + speech + failed, + WAKES, + "every wake lands in exactly one bucket" + ); + } } From 53499d62e64af897eda6ac72a4b39744f944bc60 Mon Sep 17 00:00:00 2001 From: omesser Date: Wed, 2 Sep 2026 15:10:12 +0300 Subject: [PATCH 2/5] Record the measured reply-contract rates, and the case bug they exposed Co-Authored-By: Claude Opus 5 --- README.md | 42 +++++++++++++++++++++++++++++++++--------- 1 file changed, 33 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index ebaf89530..4462b95fa 100644 --- a/README.md +++ b/README.md @@ -213,15 +213,39 @@ director: http://localhost:11434 model "llama3.2" is not served; it has gemma4:l Neither line stops anything: a wake that fails already falls back to Static per turn. The line exists so a buddy that went quiet is not a mystery. -**Size and the reply contract.** The Director asks for a Behavior name on one -line and an optional spoken line on the next. Small models break that shape -more often than hosted ones do, and every break is silent — an unparsable -reply becomes speech, a failed one becomes Static. A local reasoning model -(Qwen3, gpt-oss) has a second failure: it thinks inside the same token budget -on chat-completions, so a tight cap can be spent before it writes anything. -That is why the local cap defaults to 512 rather than 80. Constrained -decoding is the real fix, and four of the five servers support it through -`response_format`; #144 decides that shape. +**Size and the reply contract, measured.** The Director asks for a Behavior +name on one line and an optional spoken line on the next, and every break is +silent — an unparsable reply becomes speech, a failed one becomes Static. +`measure_the_reply_contract_failure_rate` (an `#[ignore]`d test in +`src-tauri/src/model.rs`) counts the outcomes over 40 varied wakes against a +live server, classifying with `ModelDirector::wake` itself so the measurement +cannot drift from what the app does. On an Apple-silicon Mac, loading +`characters/cat`: + +| Server | Model | Contract kept | Prose instead | Failed | +|---|---|---|---|---| +| Ollama | `gemma4:latest`, 8B Q4_K_M, 9.6 GB | 95% | 2% | 2% | +| Ollama | `llama3.2:1b`, 1B | 78% | 22% | 0% | +| oMLX | `gemma-4-e2b-it-4bit` | 100% | 0% | 0% | + +Size drives the break rate less than instruction tuning does: the 1B model +breaks the shape on 22% of wakes, while a 4-bit `e2b` build kept it on all +forty. Constrained decoding is still the durable fix, and four of the five +servers support it through `response_format`; #144 decides that shape. + +**What the measurement found first, though, was ours.** `ModelDirector::knows` +compares a proposed Behavior name to the declared ones with `==`, and every +model above answers `Prowl` where the manifest declares `prowl`. So a reply +that kept the contract is refused, falls through `as_speech`, and reaches the +user as dialogue with the Behavior name stuck on the front — the buddy talks +and never acts. Accepted-as-declared was 0%, 2% and 0% in the three runs +above; the "contract kept" column is what a case-insensitive comparison would +have accepted. #175 measured it; fixing it is a separate change. + +A local reasoning model (Qwen3, gpt-oss) has a second failure: it thinks +inside the same token budget on chat-completions, so a tight cap can be spent +before it writes anything. That is why the local cap defaults to 512 rather +than 80. ## Development From 6e71f4896da0e10ebe0372a37ae9ff33d26f111e Mon Sep 17 00:00:00 2001 From: omesser Date: Wed, 2 Sep 2026 15:39:08 +0300 Subject: [PATCH 3/5] Let the benchmark's sample size be set, and say what it cannot tell Co-Authored-By: Claude Opus 5 --- README.md | 6 ++++-- src-tauri/src/model.rs | 22 +++++++++++++++++----- 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 4462b95fa..b9c571ca8 100644 --- a/README.md +++ b/README.md @@ -219,8 +219,10 @@ silent — an unparsable reply becomes speech, a failed one becomes Static. `measure_the_reply_contract_failure_rate` (an `#[ignore]`d test in `src-tauri/src/model.rs`) counts the outcomes over 40 varied wakes against a live server, classifying with `ModelDirector::wake` itself so the measurement -cannot drift from what the app does. On an Apple-silicon Mac, loading -`characters/cat`: +cannot drift from what the app does. Nothing pins `temperature` or a seed — +the app sends neither — so forty wakes tell 5% from 50% but not 5% from 8%; +`AI_BUDDY_BENCH_WAKES` raises the sample when a tighter number is worth the +minutes. On an Apple-silicon Mac, loading `characters/cat`: | Server | Model | Contract kept | Prose instead | Failed | |---|---|---|---|---| diff --git a/src-tauri/src/model.rs b/src-tauri/src/model.rs index f33d46a92..65de15a8a 100644 --- a/src-tauri/src/model.rs +++ b/src-tauri/src/model.rs @@ -1536,6 +1536,8 @@ mod tests { /// AI_BUDDY_DIRECTOR_MODEL=gemma4 \ /// cargo test -p ai-buddy measure_the_reply_contract -- --ignored --nocapture /// ``` + /// + /// `AI_BUDDY_BENCH_WAKES` sets the sample size; it defaults to 40. #[test] #[ignore] fn measure_the_reply_contract_failure_rate() { @@ -1545,7 +1547,16 @@ mod tests { use std::path::Path; use std::time::{Instant, SystemTime}; - const WAKES: usize = 40; + // Forty tells 5% from 50%, which is what the question needs. It does + // not tell 5% from 8%: nothing pins `temperature` or a seed, because + // the app sends neither and this measures the app, so runs of the + // same model wander by a few points. Raise it when a tighter number + // is worth the minutes. + let wakes: usize = std::env::var("AI_BUDDY_BENCH_WAKES") + .ok() + .and_then(|raw| raw.parse().ok()) + .filter(|&n: &usize| n > 0) + .unwrap_or(40); let endpoint = endpoint().expect("AI_BUDDY_DIRECTOR_BASE_URL and _MODEL in the env"); let model = endpoint.model().to_string(); @@ -1578,7 +1589,7 @@ mod tests { let mut examples: Vec = Vec::new(); let started = Instant::now(); - for turn in 0..WAKES { + for turn in 0..wakes { let (happened, state, standing) = &occasions[turn % occasions.len()]; let context = Context { activity: Activity { @@ -1634,8 +1645,8 @@ mod tests { } } - let percent = |n: usize| (n as f64) * 100.0 / (WAKES as f64); - println!("\n#175 reply-contract outcomes over {WAKES} wakes"); + let percent = |n: usize| (n as f64) * 100.0 / (wakes as f64); + println!("\n#175 reply-contract outcomes over {wakes} wakes"); println!(" model: {model} at {origin}"); println!(" behaviors: {}", behaviors.join(", ")); println!(" elapsed: {:.0}s", started.elapsed().as_secs_f64()); @@ -1653,13 +1664,14 @@ mod tests { " the model broke the contract on {:.0}% of wakes", percent(speech + failed) ); + println!(" (sampling is the server's own; runs of one model wander a few points)"); for line in &examples { println!("{line}"); } assert_eq!( accepted + case_only + speech + failed, - WAKES, + wakes, "every wake lands in exactly one bucket" ); } From c79a29c6fb0167cf4d423fe7469b275584d5d60e Mon Sep 17 00:00:00 2001 From: omesser Date: Wed, 2 Sep 2026 15:50:38 +0300 Subject: [PATCH 4/5] Re-take the numbers after #233, and report the 1B as the range it is Co-Authored-By: Claude Opus 5 --- README.md | 50 ++++++++++++++++++++++++++++++-------------------- 1 file changed, 30 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index b9c571ca8..544a87387 100644 --- a/README.md +++ b/README.md @@ -220,29 +220,39 @@ silent — an unparsable reply becomes speech, a failed one becomes Static. `src-tauri/src/model.rs`) counts the outcomes over 40 varied wakes against a live server, classifying with `ModelDirector::wake` itself so the measurement cannot drift from what the app does. Nothing pins `temperature` or a seed — -the app sends neither — so forty wakes tell 5% from 50% but not 5% from 8%; -`AI_BUDDY_BENCH_WAKES` raises the sample when a tighter number is worth the -minutes. On an Apple-silicon Mac, loading `characters/cat`: +the app sends neither — and the run shares one session, so replies condition +each other; `AI_BUDDY_BENCH_WAKES` raises the sample when a tighter number is +worth the minutes. On an Apple-silicon Mac, loading `characters/cat`: -| Server | Model | Contract kept | Prose instead | Failed | +| Server | Model | Behavior played | Prose instead | Failed | |---|---|---|---|---| | Ollama | `gemma4:latest`, 8B Q4_K_M, 9.6 GB | 95% | 2% | 2% | -| Ollama | `llama3.2:1b`, 1B | 78% | 22% | 0% | -| oMLX | `gemma-4-e2b-it-4bit` | 100% | 0% | 0% | - -Size drives the break rate less than instruction tuning does: the 1B model -breaks the shape on 22% of wakes, while a 4-bit `e2b` build kept it on all -forty. Constrained decoding is still the durable fix, and four of the five -servers support it through `response_format`; #144 decides that shape. - -**What the measurement found first, though, was ours.** `ModelDirector::knows` -compares a proposed Behavior name to the declared ones with `==`, and every -model above answers `Prowl` where the manifest declares `prowl`. So a reply -that kept the contract is refused, falls through `as_speech`, and reaches the -user as dialogue with the Behavior name stuck on the front — the buddy talks -and never acts. Accepted-as-declared was 0%, 2% and 0% in the three runs -above; the "contract kept" column is what a case-insensitive comparison would -have accepted. #175 measured it; fixing it is a separate change. +| Ollama | `llama3.2:1b`, 1B | 12–78% | 22–88% | 0% | +| oMLX | `gemma-4-e2b-it-4bit` | 100%¹ | 0% | 0% | + +¹ Measured before #233 as "contract kept"; the server needs a key, so the row +has not been re-taken since. The other two rows are post-#233 runs. + +Instruction tuning matters more than size: a 4-bit `e2b` build kept the shape +on all forty wakes, while the 1B did not. **The 1B is a range because it is +genuinely unstable** — four runs of the same model gave 12%, 30%, 65% and 78%. +One session carries the whole run, so once a small model starts inventing +names (`snub`, `scratching`, `grooming` for the declared `groom`) it keeps +doing it, and a run locks into a good or bad groove rather than averaging out. +The 8B repeated 95% twice. Read the 1B as "unreliable", not as a percentage. +Constrained decoding is the durable fix, and four of the five servers support +it through `response_format`; #144 decides that shape. + +**The measurement's first finding was ours, and it is fixed.** `knows` +compared a proposed Behavior name to the declared ones with `==`, and every +model here answers `Prowl` where the manifest declares `prowl`, so a reply +that kept the contract was refused, fell through `as_speech`, and reached the +user as dialogue with the name stuck on the front — the buddy talked and never +acted. Behaviors played were 0%, 2% and 0% before. #233 compares without case +and hands back the Character's own spelling, which is what the Engine looks a +Behavior up by; the numbers above are what the same harness measures after it. +The comparison is exact-modulo-case, not fuzzy: `Greeter` and `grooming` still +become speech, because no Character declares them. A local reasoning model (Qwen3, gpt-oss) has a second failure: it thinks inside the same token budget on chat-completions, so a tight cap can be spent From cf86eec36f07cc96f268ac834b0421ed79d59a05 Mon Sep 17 00:00:00 2001 From: omesser Date: Wed, 2 Sep 2026 16:07:31 +0300 Subject: [PATCH 5/5] Take the oMLX row after #233, and name the sample lines as a cause Co-Authored-By: Claude Opus 5 --- README.md | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 544a87387..73a3aa0ac 100644 --- a/README.md +++ b/README.md @@ -228,18 +228,25 @@ worth the minutes. On an Apple-silicon Mac, loading `characters/cat`: |---|---|---|---|---| | Ollama | `gemma4:latest`, 8B Q4_K_M, 9.6 GB | 95% | 2% | 2% | | Ollama | `llama3.2:1b`, 1B | 12–78% | 22–88% | 0% | -| oMLX | `gemma-4-e2b-it-4bit` | 100%¹ | 0% | 0% | +| oMLX | `gemma-4-e2b-it-4bit` | 90% | 10% | 0% | -¹ Measured before #233 as "contract kept"; the server needs a key, so the row -has not been re-taken since. The other two rows are post-#233 runs. +Every row is a post-#233 run. -Instruction tuning matters more than size: a 4-bit `e2b` build kept the shape -on all forty wakes, while the 1B did not. **The 1B is a range because it is -genuinely unstable** — four runs of the same model gave 12%, 30%, 65% and 78%. +Instruction tuning matters more than size: a 4-bit `e2b` build plays a +Behavior on nine wakes in ten, while the 1B swings between one and eight. +**The 1B is a range because it is genuinely unstable** — four runs of the same +model gave 12%, 30%, 65% and 78%. One session carries the whole run, so once a small model starts inventing names (`snub`, `scratching`, `grooming` for the declared `groom`) it keeps doing it, and a run locks into a good or bad groove rather than averaging out. The 8B repeated 95% twice. Read the 1B as "unreliable", not as a percentage. +**Some of the remaining breaks are the personality file's doing.** The prose +the `e2b` build returned instead of a Behavior was twice the verbatim line +`"What is that one? Show me."` — one of the sample lines in +`characters/cat/personality.txt`. A small model reads sample lines as replies +to imitate whole, and a sample line carries no Behavior name, so quoting one +breaks the contract by construction. Worth weighing when writing them (#156). + Constrained decoding is the durable fix, and four of the five servers support it through `response_format`; #144 decides that shape. @@ -248,7 +255,8 @@ compared a proposed Behavior name to the declared ones with `==`, and every model here answers `Prowl` where the manifest declares `prowl`, so a reply that kept the contract was refused, fell through `as_speech`, and reached the user as dialogue with the name stuck on the front — the buddy talked and never -acted. Behaviors played were 0%, 2% and 0% before. #233 compares without case +acted. Behaviors played were 0%, 2% and 0% before; they are 95%, 12–78% and +90% after. #233 compares without case and hands back the Character's own spelling, which is what the Engine looks a Behavior up by; the numbers above are what the same harness measures after it. The comparison is exact-modulo-case, not fuzzy: `Greeter` and `grooming` still