diff --git a/README.md b/README.md index ebaf89530..73a3aa0ac 100644 --- a/README.md +++ b/README.md @@ -213,15 +213,59 @@ director: http://localhost:11434 model "llama3.2" is not served; it has gemma4:l Neither line stops anything: a wake that fails already falls back to Static per turn. The line exists so a buddy that went quiet is not a mystery. -**Size and the reply contract.** The Director asks for a Behavior name on one -line and an optional spoken line on the next. Small models break that shape -more often than hosted ones do, and every break is silent — an unparsable -reply becomes speech, a failed one becomes Static. A local reasoning model -(Qwen3, gpt-oss) has a second failure: it thinks inside the same token budget -on chat-completions, so a tight cap can be spent before it writes anything. -That is why the local cap defaults to 512 rather than 80. Constrained -decoding is the real fix, and four of the five servers support it through -`response_format`; #144 decides that shape. +**Size and the reply contract, measured.** The Director asks for a Behavior +name on one line and an optional spoken line on the next, and every break is +silent — an unparsable reply becomes speech, a failed one becomes Static. +`measure_the_reply_contract_failure_rate` (an `#[ignore]`d test in +`src-tauri/src/model.rs`) counts the outcomes over 40 varied wakes against a +live server, classifying with `ModelDirector::wake` itself so the measurement +cannot drift from what the app does. Nothing pins `temperature` or a seed — +the app sends neither — and the run shares one session, so replies condition +each other; `AI_BUDDY_BENCH_WAKES` raises the sample when a tighter number is +worth the minutes. On an Apple-silicon Mac, loading `characters/cat`: + +| Server | Model | Behavior played | Prose instead | Failed | +|---|---|---|---|---| +| Ollama | `gemma4:latest`, 8B Q4_K_M, 9.6 GB | 95% | 2% | 2% | +| Ollama | `llama3.2:1b`, 1B | 12–78% | 22–88% | 0% | +| oMLX | `gemma-4-e2b-it-4bit` | 90% | 10% | 0% | + +Every row is a post-#233 run. + +Instruction tuning matters more than size: a 4-bit `e2b` build plays a +Behavior on nine wakes in ten, while the 1B swings between one and eight. +**The 1B is a range because it is genuinely unstable** — four runs of the same +model gave 12%, 30%, 65% and 78%. +One session carries the whole run, so once a small model starts inventing +names (`snub`, `scratching`, `grooming` for the declared `groom`) it keeps +doing it, and a run locks into a good or bad groove rather than averaging out. +The 8B repeated 95% twice. Read the 1B as "unreliable", not as a percentage. +**Some of the remaining breaks are the personality file's doing.** The prose +the `e2b` build returned instead of a Behavior was twice the verbatim line +`"What is that one? Show me."` — one of the sample lines in +`characters/cat/personality.txt`. A small model reads sample lines as replies +to imitate whole, and a sample line carries no Behavior name, so quoting one +breaks the contract by construction. Worth weighing when writing them (#156). + +Constrained decoding is the durable fix, and four of the five servers support +it through `response_format`; #144 decides that shape. + +**The measurement's first finding was ours, and it is fixed.** `knows` +compared a proposed Behavior name to the declared ones with `==`, and every +model here answers `Prowl` where the manifest declares `prowl`, so a reply +that kept the contract was refused, fell through `as_speech`, and reached the +user as dialogue with the name stuck on the front — the buddy talked and never +acted. Behaviors played were 0%, 2% and 0% before; they are 95%, 12–78% and +90% after. #233 compares without case +and hands back the Character's own spelling, which is what the Engine looks a +Behavior up by; the numbers above are what the same harness measures after it. +The comparison is exact-modulo-case, not fuzzy: `Greeter` and `grooming` still +become speech, because no Character declares them. + +A local reasoning model (Qwen3, gpt-oss) has a second failure: it thinks +inside the same token budget on chat-completions, so a tight cap can be spent +before it writes anything. That is why the local cap defaults to 512 rather +than 80. ## Development diff --git a/src-tauri/src/model.rs b/src-tauri/src/model.rs index b57ae3198..65de15a8a 100644 --- a/src-tauri/src/model.rs +++ b/src-tauri/src/model.rs @@ -1520,4 +1520,159 @@ mod tests { "the abandoned Wake must not land on the new Character" ); } + + /// #175: how often a live local model breaks the reply contract, as the + /// before number #144 argues from. Ignored because it needs a server and + /// spends real seconds; it is the harness, not a check of our own code. + /// + /// The classifier is `ModelDirector::wake` itself rather than a copy of + /// it, so the measurement cannot drift from what the app actually does: + /// a proposal naming a declared Behavior is accepted, an empty name is + /// `as_speech` catching prose, and `Failed` is the turn `StaticDirector` + /// takes. One session throughout, because that is how the buddy runs. + /// + /// ```sh + /// AI_BUDDY_DIRECTOR_BASE_URL=http://localhost:11434 \ + /// AI_BUDDY_DIRECTOR_MODEL=gemma4 \ + /// cargo test -p ai-buddy measure_the_reply_contract -- --ignored --nocapture + /// ``` + /// + /// `AI_BUDDY_BENCH_WAKES` sets the sample size; it defaults to 40. + #[test] + #[ignore] + fn measure_the_reply_contract_failure_rate() { + use ai_buddy_core::director::{Context, Happened, ModelDirector, Wake}; + use ai_buddy_core::engine::State; + use ai_buddy_core::sensing::Activity; + use std::path::Path; + use std::time::{Instant, SystemTime}; + + // Forty tells 5% from 50%, which is what the question needs. It does + // not tell 5% from 8%: nothing pins `temperature` or a seed, because + // the app sends neither and this measures the app, so runs of the + // same model wander by a few points. Raise it when a tighter number + // is worth the minutes. + let wakes: usize = std::env::var("AI_BUDDY_BENCH_WAKES") + .ok() + .and_then(|raw| raw.parse().ok()) + .filter(|&n: &usize| n > 0) + .unwrap_or(40); + + let endpoint = endpoint().expect("AI_BUDDY_DIRECTOR_BASE_URL and _MODEL in the env"); + let model = endpoint.model().to_string(); + let origin = endpoint.origin(); + + // A real shipped package, so the prompt is the one production sends: + // its Personality Prompt and its declared Behavior names. + let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../characters/cat"); + let files = crate::package::read(&root).expect("the shipped cat package reads"); + let cat = ai_buddy_core::character::load(&files).expect("and loads"); + let behaviors: Vec = cat.behaviors.keys().cloned().collect(); + + let director = ModelDirector::new(endpoint, behaviors.clone()); + + // Vary the wake so the prompts differ: the reactive verbs plus ambient. + let occasions = [ + (Happened::Poke, State::Grounded, "the display floor"), + (Happened::Throw, State::Falling, "nothing"), + (Happened::Summon, State::Grounded, "a Terminal window"), + (Happened::Perch, State::Perched, "a Safari window"), + (Happened::Ambient, State::Grounded, "the top of the Dock"), + ]; + + let (mut accepted, mut speech, mut failed) = (0usize, 0usize, 0usize); + // A reply whose first line names a declared Behavior in the wrong + // case is the contract kept and our matcher refusing it: `knows` + // compares exactly. Counting it apart separates what the model got + // wrong from what we do. + let mut case_only = 0usize; + let mut examples: Vec = Vec::new(); + let started = Instant::now(); + + for turn in 0..wakes { + let (happened, state, standing) = &occasions[turn % occasions.len()]; + let context = Context { + activity: Activity { + frontmost_application: Some("Terminal".to_string()), + switched: turn % 3 == 0, + idle: Duration::from_secs((turn as u64 % 7) * 30), + at: SystemTime::now(), + hour: 9 + (turn as u8 % 12), + minute: ((turn as u32 * 7) % 60) as u8, + displays_asleep: false, + }, + recent: Vec::new(), + personality: cat.personality.clone(), + state: *state, + happened: *happened, + standing: standing.to_string(), + }; + + match director.wake(&context) { + Wake::Proposed(proposal) if !proposal.behavior.is_empty() => { + accepted += 1; + } + Wake::Proposed(proposal) => { + // `as_speech` hands back the whole reply, so its first + // line is the name the model actually offered. + let said = proposal.dialogue.unwrap_or_default(); + let offered = said + .lines() + .next() + .unwrap_or_default() + .trim() + .trim_end_matches(['.', ':', '!']) + .to_string(); + let near = behaviors + .iter() + .any(|declared| declared.eq_ignore_ascii_case(&offered)); + if near { + case_only += 1; + } else { + speech += 1; + } + if examples.len() < 6 { + let tag = if near { "case-only" } else { "speech" }; + examples.push(format!(" {tag}: {}", said.replace('\n', " | "))); + } + } + Wake::Failed => { + failed += 1; + if examples.len() < 5 { + examples.push(" failed: unparsable or transport error".to_string()); + } + } + } + } + + let percent = |n: usize| (n as f64) * 100.0 / (wakes as f64); + println!("\n#175 reply-contract outcomes over {wakes} wakes"); + println!(" model: {model} at {origin}"); + println!(" behaviors: {}", behaviors.join(", ")); + println!(" elapsed: {:.0}s", started.elapsed().as_secs_f64()); + println!(" accepted: {accepted:>3} ({:.0}%)", percent(accepted)); + println!( + " case-only: {case_only:>3} ({:.0}%) contract kept, matcher refused", + percent(case_only) + ); + println!( + " speech: {speech:>3} ({:.0}%) genuine prose", + percent(speech) + ); + println!(" failed: {failed:>3} ({:.0}%)", percent(failed)); + println!( + " the model broke the contract on {:.0}% of wakes", + percent(speech + failed) + ); + println!(" (sampling is the server's own; runs of one model wander a few points)"); + for line in &examples { + println!("{line}"); + } + + assert_eq!( + accepted + case_only + speech + failed, + wakes, + "every wake lands in exactly one bucket" + ); + } }