From 534c1cbfb2ac3645036b5dfb90d10868c3699803 Mon Sep 17 00:00:00 2001
From: Steven Enamakel <enamakel@tinyhumans.ai>
Date: Wed, 8 Apr 2026 00:47:44 -0700
Subject: [PATCH 1/4] feat(settings): add 'Use Vision Model' option to Screen
 Intelligence Panel

- Introduced a new checkbox in the Screen Intelligence Panel to toggle the use of a vision model for richer context extraction from screenshots.
- Updated state management to handle the new option and integrated it into the configuration and processing logic.
- Adjusted related tests and configurations to support the new feature, ensuring compatibility across the application.
---
 .../panels/ScreenIntelligencePanel.tsx        | 18 ++++
 .../ScreenIntelligencePanel.test.tsx          |  2 +
 app/src/utils/tauriCommands/accessibility.ts  |  1 +
 app/src/utils/tauriCommands/config.ts         |  1 +
 src/openhuman/config/ops.rs                   |  4 +
 src/openhuman/config/schema/accessibility.rs  | 11 +++
 src/openhuman/config/schemas.rs               |  2 +
 .../screen_intelligence/processing_worker.rs  | 96 ++++++++++++++-----
 8 files changed, 110 insertions(+), 25 deletions(-)

diff --git a/app/src/components/settings/panels/ScreenIntelligencePanel.tsx b/app/src/components/settings/panels/ScreenIntelligencePanel.tsx
index 8f35d85e644..53cc2dedf15 100644
--- a/app/src/components/settings/panels/ScreenIntelligencePanel.tsx
+++ b/app/src/components/settings/panels/ScreenIntelligencePanel.tsx
@@ -71,6 +71,7 @@ const ScreenIntelligencePanel = () => {
     'all_except_blacklist'
   );
   const [baselineFps, setBaselineFps] = useState<string>('1');
+  const [useVisionModel, setUseVisionModel] = useState<boolean>(true);
   const [keepScreenshots, setKeepScreenshots] = useState<boolean>(false);
   const [allowlistText, setAllowlistText] = useState('');
   const [denylistText, setDenylistText] = useState('');
@@ -105,6 +106,7 @@ const ScreenIntelligencePanel = () => {
       status.config.policy_mode === 'whitelist_only' ? 'whitelist_only' : 'all_except_blacklist'
     );
     setBaselineFps(String(status.config.baseline_fps ?? 1));
+    setUseVisionModel(status.config.use_vision_model ?? true);
     setKeepScreenshots(status.config.keep_screenshots ?? false);
     setAllowlistText((status.config.allowlist ?? []).join('\n'));
     setDenylistText((status.config.denylist ?? []).join('\n'));
@@ -145,6 +147,7 @@ const ScreenIntelligencePanel = () => {
         enabled,
         policy_mode: policyMode,
         baseline_fps: Number.isFinite(fps) && fps > 0 ? fps : 1,
+        use_vision_model: useVisionModel,
         keep_screenshots: keepScreenshots,
         allowlist: allowlistText
           .split('\n')
@@ -282,6 +285,21 @@ const ScreenIntelligencePanel = () => {
             />
           </label>
 
+          <label className="flex items-center justify-between rounded-xl border border-stone-200 bg-stone-50 px-3 py-2">
+            <div>
+              <span className="text-sm text-stone-700">Use Vision Model</span>
+              <p className="text-xs text-stone-400">
+                Send screenshots to a vision LLM for richer context. When off, only OCR text is used
+                with a text LLM — faster and no vision model required.
+              </p>
+            </div>
+            <input
+              type="checkbox"
+              checked={useVisionModel}
+              onChange={event => setUseVisionModel(event.target.checked)}
+            />
+          </label>
+
           <label className="flex items-center justify-between rounded-xl border border-stone-200 bg-stone-50 px-3 py-2">
             <div>
               <span className="text-sm text-stone-700">Keep Screenshots</span>
diff --git a/app/src/components/settings/panels/__tests__/ScreenIntelligencePanel.test.tsx b/app/src/components/settings/panels/__tests__/ScreenIntelligencePanel.test.tsx
index e5717618380..cb30858d96e 100644
--- a/app/src/components/settings/panels/__tests__/ScreenIntelligencePanel.test.tsx
+++ b/app/src/components/settings/panels/__tests__/ScreenIntelligencePanel.test.tsx
@@ -78,6 +78,7 @@ const baseStatus: AccessibilityStatus = {
     session_ttl_secs: 300,
     panic_stop_hotkey: 'Cmd+Shift+.',
     autocomplete_enabled: true,
+    use_vision_model: true,
     keep_screenshots: false,
     allowlist: ['Code'],
     denylist: ['1Password'],
@@ -189,6 +190,7 @@ describe('ScreenIntelligencePanel', () => {
       enabled: true,
       policy_mode: 'all_except_blacklist',
       baseline_fps: 1,
+      use_vision_model: true,
       keep_screenshots: false,
       allowlist: ['Code'],
       denylist: ['1Password'],
diff --git a/app/src/utils/tauriCommands/accessibility.ts b/app/src/utils/tauriCommands/accessibility.ts
index 59dcc1bf96d..71324a4bc99 100644
--- a/app/src/utils/tauriCommands/accessibility.ts
+++ b/app/src/utils/tauriCommands/accessibility.ts
@@ -46,6 +46,7 @@ export interface AccessibilityConfig {
   session_ttl_secs: number;
   panic_stop_hotkey: string;
   autocomplete_enabled: boolean;
+  use_vision_model: boolean;
   keep_screenshots: boolean;
   allowlist: string[];
   denylist: string[];
diff --git a/app/src/utils/tauriCommands/config.ts b/app/src/utils/tauriCommands/config.ts
index 46d0f834539..9a238e1bd07 100644
--- a/app/src/utils/tauriCommands/config.ts
+++ b/app/src/utils/tauriCommands/config.ts
@@ -41,6 +41,7 @@ export interface ScreenIntelligenceSettingsUpdate {
   baseline_fps?: number | null;
   vision_enabled?: boolean | null;
   autocomplete_enabled?: boolean | null;
+  use_vision_model?: boolean | null;
   keep_screenshots?: boolean | null;
   allowlist?: string[] | null;
   denylist?: string[] | null;
diff --git a/src/openhuman/config/ops.rs b/src/openhuman/config/ops.rs
index af3e50d639c..f5c259540dd 100644
--- a/src/openhuman/config/ops.rs
+++ b/src/openhuman/config/ops.rs
@@ -164,6 +164,7 @@ pub struct ScreenIntelligenceSettingsPatch {
     pub baseline_fps: Option<f32>,
     pub vision_enabled: Option<bool>,
     pub autocomplete_enabled: Option<bool>,
+    pub use_vision_model: Option<bool>,
     pub keep_screenshots: Option<bool>,
     pub allowlist: Option<Vec<String>>,
     pub denylist: Option<Vec<String>>,
@@ -282,6 +283,9 @@ pub async fn apply_screen_intelligence_settings(
     if let Some(autocomplete_enabled) = update.autocomplete_enabled {
         config.screen_intelligence.autocomplete_enabled = autocomplete_enabled;
     }
+    if let Some(use_vision_model) = update.use_vision_model {
+        config.screen_intelligence.use_vision_model = use_vision_model;
+    }
     if let Some(keep_screenshots) = update.keep_screenshots {
         config.screen_intelligence.keep_screenshots = keep_screenshots;
     }
diff --git a/src/openhuman/config/schema/accessibility.rs b/src/openhuman/config/schema/accessibility.rs
index 80e76385dc8..7ed04092b4c 100644
--- a/src/openhuman/config/schema/accessibility.rs
+++ b/src/openhuman/config/schema/accessibility.rs
@@ -19,6 +19,12 @@ pub struct ScreenIntelligenceConfig {
     pub panic_stop_hotkey: String,
     #[serde(default = "default_autocomplete_enabled")]
     pub autocomplete_enabled: bool,
+    /// When `true`, Pass 2 sends the screenshot to a vision-capable LLM for
+    /// visual context extraction.  When `false`, only Apple Vision OCR (Pass 1)
+    /// feeds into the text synthesis LLM (Pass 3) — no vision model required.
+    /// Default: `true`.
+    #[serde(default = "default_use_vision_model")]
+    pub use_vision_model: bool,
     /// When `true`, captured screenshots are saved to `{workspace_dir}/screenshots/`
     /// instead of being discarded after vision processing. Default: `false`.
     #[serde(default)]
@@ -61,6 +67,10 @@ fn default_autocomplete_enabled() -> bool {
     true
 }
 
+fn default_use_vision_model() -> bool {
+    true
+}
+
 impl Default for ScreenIntelligenceConfig {
     fn default() -> Self {
         Self {
@@ -72,6 +82,7 @@ impl Default for ScreenIntelligenceConfig {
             session_ttl_secs: default_session_ttl_secs(),
             panic_stop_hotkey: default_panic_stop_hotkey(),
             autocomplete_enabled: default_autocomplete_enabled(),
+            use_vision_model: default_use_vision_model(),
             keep_screenshots: false,
             allowlist: vec![],
             denylist: vec![
diff --git a/src/openhuman/config/schemas.rs b/src/openhuman/config/schemas.rs
index e70d5dfe1b0..21eb6ef878b 100644
--- a/src/openhuman/config/schemas.rs
+++ b/src/openhuman/config/schemas.rs
@@ -45,6 +45,7 @@ struct ScreenIntelligenceSettingsUpdate {
     baseline_fps: Option<f32>,
     vision_enabled: Option<bool>,
     autocomplete_enabled: Option<bool>,
+    use_vision_model: Option<bool>,
     keep_screenshots: Option<bool>,
     allowlist: Option<Vec<String>>,
     denylist: Option<Vec<String>>,
@@ -587,6 +588,7 @@ fn handle_update_screen_intelligence_settings(params: Map<String, Value>) -> Con
             baseline_fps: update.baseline_fps,
             vision_enabled: update.vision_enabled,
             autocomplete_enabled: update.autocomplete_enabled,
+            use_vision_model: update.use_vision_model,
             keep_screenshots: update.keep_screenshots,
             allowlist: update.allowlist,
             denylist: update.denylist,
diff --git a/src/openhuman/screen_intelligence/processing_worker.rs b/src/openhuman/screen_intelligence/processing_worker.rs
index 193dcf2cd2f..ddcb4476659 100644
--- a/src/openhuman/screen_intelligence/processing_worker.rs
+++ b/src/openhuman/screen_intelligence/processing_worker.rs
@@ -171,7 +171,16 @@ pub(crate) async fn run(
 
 // ── Analysis pipeline ───────────────────────────────────────────────────
 
-/// Run the full 3-pass analysis pipeline on a captured frame.
+/// Run the analysis pipeline on a captured frame.
+///
+/// When `use_vision_model` is `true` (default), the full 3-pass pipeline runs:
+///   1. Apple Vision OCR → raw text
+///   2. Vision LLM (Ollama) → visual context from screenshot
+///   3. Synthesis LLM → final document combining OCR + vision context
+///
+/// When `use_vision_model` is `false`, Pass 2 is skipped and the synthesis LLM
+/// works from OCR text + app metadata only — no vision-capable model required.
+///
 /// Public within the crate so `engine.rs` can call it for flush/diagnostics.
 pub(crate) async fn analyze_frame(
     _engine: &AccessibilityEngine,
@@ -218,17 +227,21 @@ pub(crate) async fn analyze_frame(
     .map_err(|_| "Apple Vision OCR timed out after 30s".to_string())??;
     tracing::debug!("[processing_worker] OCR extracted {} chars", ocr_text.len());
 
-    // ── Pass 2: Vision LLM for context ──────────────────────────────
-    let compressed = super::image_processing::compress_screenshot(&image_ref, None, None)
-        .map_err(|e| format!("image compression failed: {e}"))?;
-    let vision_image_ref = compressed.data_uri;
+    // ── Resolve use_vision_model flag from config ─────────────────
+    let use_vision_model = config.screen_intelligence.use_vision_model;
 
-    tracing::debug!(
-        "[processing_worker] pass 2/3: vision LLM (model={})",
-        config.local_ai.vision_model_id,
-    );
-    let service = local_ai::global(&config);
-    let vision_prompt = r#"Describe this screenshot briefly. Answer each on its own line:
+    // ── Pass 2: Vision LLM for context (skipped when use_vision_model=false) ──
+    let vision_context = if use_vision_model {
+        let compressed = super::image_processing::compress_screenshot(&image_ref, None, None)
+            .map_err(|e| format!("image compression failed: {e}"))?;
+        let vision_image_ref = compressed.data_uri;
+
+        tracing::debug!(
+            "[processing_worker] pass 2/3: vision LLM (model={})",
+            config.local_ai.vision_model_id,
+        );
+        let service = local_ai::global(&config);
+        let vision_prompt = r#"Describe this screenshot briefly. Answer each on its own line:
 
 APP: Name the application and the specific page/view/tab shown.
 DOING: What is the user actively doing? (e.g. writing code, reading email, browsing, chatting)
@@ -236,31 +249,42 @@ FOCUS: What's the main content area about? (e.g. a PR review for auth refactor,
 MOOD: Is anything urgent, broken, or notable? (errors, notifications, warnings — or "nothing notable")
 
 One line per answer. No text extraction. Be specific and concise."#;
-    let vision_context = service
-        .vision_prompt(&config, vision_prompt, &[vision_image_ref], Some(150))
-        .await?
-        .trim()
-        .to_string();
+        Some(
+            service
+                .vision_prompt(&config, vision_prompt, &[vision_image_ref], Some(150))
+                .await?
+                .trim()
+                .to_string(),
+        )
+    } else {
+        tracing::debug!("[processing_worker] pass 2/3: SKIPPED (use_vision_model=false)");
+        None
+    };
 
     // ── Pass 3: Synthesis LLM — final document ──────────────────────
     let app_label = frame.app_name.as_deref().unwrap_or("Unknown");
     let window_label = frame.window_title.as_deref().unwrap_or("");
     let ocr_truncated = truncate_tail(&ocr_text, 4000);
 
+    let pass_label = if use_vision_model { "3/3" } else { "2/2" };
     tracing::debug!(
-        "[processing_worker] pass 3/3: synthesis LLM (ocr={} chars, vision={} chars)",
+        "[processing_worker] pass {}: synthesis LLM (ocr={} chars, vision={} chars)",
+        pass_label,
         ocr_truncated.len(),
-        vision_context.len(),
+        vision_context.as_ref().map_or(0, |s| s.len()),
     );
 
-    let synthesis_prompt = format!(
-        r#"You are summarizing what a user is doing on their computer right now.
+    let service = local_ai::global(&config);
+
+    let synthesis_prompt = if let Some(ref vc) = vision_context {
+        format!(
+            r#"You are summarizing what a user is doing on their computer right now.
 
 Application: {app_label}
 Window: {window_label}
 
 Visual context from the screenshot:
-{vision_context}
+{vc}
 
 Extracted text from screen (OCR):
 {ocr_truncated}
@@ -273,14 +297,36 @@ Write a clear, informative summary in plain text. Include:
 - Brief context that would help someone understand this moment later
 
 Be specific and informative. Write in present tense. ~300-500 words."#
-    );
+        )
+    } else {
+        format!(
+            r#"You are summarizing what a user is doing on their computer right now.
+You only have the application name, window title, and OCR-extracted text to work with.
+
+Application: {app_label}
+Window: {window_label}
+
+Extracted text from screen (OCR):
+{ocr_truncated}
+
+Write a clear, informative summary in plain text. Include:
+- What application and specific view/page the user has open (infer from window title and OCR text)
+- What they are actively doing (coding, reading, chatting, browsing, etc.)
+- Key content visible — synthesize the OCR text into a coherent description
+- Any notable items: errors, notifications, deadlines, action items
+- Brief context that would help someone understand this moment later
+
+Be specific and informative. Write in present tense. ~300-500 words."#
+        )
+    };
 
+    let fallback_text = vision_context.as_deref().unwrap_or("");
     let synthesis = service
         .prompt(&config, &synthesis_prompt, Some(700), true)
         .await
         .unwrap_or_else(|e| {
             tracing::debug!("[processing_worker] synthesis failed, using fallback: {e}");
-            format!("{}\n\n{}", vision_context, ocr_truncated)
+            format!("{}\n\n{}", fallback_text, ocr_truncated)
         });
 
     tracing::debug!(
@@ -293,10 +339,10 @@ Be specific and informative. Write in present tense. ~300-500 words."#
         captured_at_ms: frame.captured_at_ms,
         app_name: frame.app_name,
         window_title: frame.window_title,
-        ui_state: truncate_tail(&vision_context, 500),
+        ui_state: truncate_tail(vision_context.as_deref().unwrap_or(&ocr_truncated), 500),
         key_text: truncate_tail(&synthesis, 4000),
         actionable_notes: String::new(),
-        confidence: 0.9,
+        confidence: if use_vision_model { 0.9 } else { 0.75 },
     })
 }
 

From 6425e3ca538a853789f2c52c128d1e82f884a815 Mon Sep 17 00:00:00 2001
From: Steven Enamakel <enamakel@tinyhumans.ai>
Date: Wed, 8 Apr 2026 00:49:55 -0700
Subject: [PATCH 2/4] feat(cli): add --no-vision-model option for screen
 intelligence

- Introduced a new command-line option `--no-vision-model` to allow users to skip the vision model and use OCR and text LLM only.
- Updated the CLI options parsing to handle the new flag and modified the bootstrap logic to respect this setting.
- Enhanced usage documentation to reflect the new option and its alias `--ocr-only` for clarity.
---
 src/core/screen_intelligence_cli.rs | 80 ++++++++++++++++++++---------
 1 file changed, 57 insertions(+), 23 deletions(-)

diff --git a/src/core/screen_intelligence_cli.rs b/src/core/screen_intelligence_cli.rs
index d3e478f956b..dfa1d751fdd 100644
--- a/src/core/screen_intelligence_cli.rs
+++ b/src/core/screen_intelligence_cli.rs
@@ -46,6 +46,7 @@ struct CliOpts {
     ttl_secs: u64,
     keep: bool,
     limit: usize,
+    no_vision_model: bool,
 }
 
 fn parse_opts(args: &[String]) -> Result<(CliOpts, Vec<String>)> {
@@ -53,11 +54,16 @@ fn parse_opts(args: &[String]) -> Result<(CliOpts, Vec<String>)> {
     let mut ttl_secs: u64 = 300;
     let mut keep = false;
     let mut limit: usize = 10;
+    let mut no_vision_model = false;
     let mut rest = Vec::new();
     let mut i = 0;
 
     while i < args.len() {
         match args[i].as_str() {
+            "--no-vision-model" | "--ocr-only" => {
+                no_vision_model = true;
+                i += 1;
+            }
             "--ttl" => {
                 let val = args
                     .get(i + 1)
@@ -101,6 +107,7 @@ fn parse_opts(args: &[String]) -> Result<(CliOpts, Vec<String>)> {
             ttl_secs,
             keep,
             limit,
+            no_vision_model,
         },
         rest,
     ))
@@ -109,14 +116,26 @@ fn parse_opts(args: &[String]) -> Result<(CliOpts, Vec<String>)> {
 /// Bootstrap the screen intelligence engine with config.
 async fn bootstrap_engine(
     verbose: bool,
+) -> Result<Arc<crate::openhuman::screen_intelligence::AccessibilityEngine>> {
+    bootstrap_engine_with_opts(verbose, false).await
+}
+
+/// Bootstrap with CLI overrides.
+async fn bootstrap_engine_with_opts(
+    verbose: bool,
+    no_vision_model: bool,
 ) -> Result<Arc<crate::openhuman::screen_intelligence::AccessibilityEngine>> {
     use crate::openhuman::config::Config;
     use crate::openhuman::screen_intelligence::global_engine;
 
-    let config = Config::load_or_init()
+    let mut config = Config::load_or_init()
         .await
         .map_err(|e| anyhow::anyhow!("config load failed: {e}"))?;
 
+    if no_vision_model {
+        config.screen_intelligence.use_vision_model = false;
+    }
+
     let engine = global_engine();
     let _ = engine
         .apply_config(config.screen_intelligence.clone())
@@ -124,9 +143,10 @@ async fn bootstrap_engine(
 
     if verbose {
         log::info!(
-            "[screen-intelligence-cli] engine initialized, enabled={}, vision={}, keep_screenshots={}, workspace={}",
+            "[screen-intelligence-cli] engine initialized, enabled={}, vision={}, use_vision_model={}, keep_screenshots={}, workspace={}",
             config.screen_intelligence.enabled,
             config.screen_intelligence.vision_enabled,
+            config.screen_intelligence.use_vision_model,
             config.screen_intelligence.keep_screenshots,
             config.workspace_dir.display(),
         );
@@ -148,15 +168,17 @@ fn run_server(args: &[String]) -> Result<()> {
     let (opts, rest) = parse_opts(args)?;
 
     if rest.iter().any(|a| is_help(a)) {
-        println!("Usage: openhuman screen-intelligence run [--ttl <secs>] [--keep] [-v]");
+        println!("Usage: openhuman screen-intelligence run [--ttl <secs>] [--keep] [--no-vision-model] [-v]");
         println!();
         println!("Start the screen intelligence capture + vision loop.");
-        println!("Captures screenshots at baseline FPS, sends to vision model,");
+        println!("Captures screenshots at baseline FPS, runs OCR and vision analysis,");
         println!("and logs summaries. Blocks until TTL expires or Ctrl+C.");
         println!();
-        println!("  --ttl <secs>     Session duration (default: 300)");
-        println!("  --keep           Keep screenshots on disk after vision processing");
-        println!("  -v, --verbose    Enable debug logging");
+        println!("  --ttl <secs>        Session duration (default: 300)");
+        println!("  --keep              Keep screenshots on disk after vision processing");
+        println!("  --no-vision-model   Skip the vision LLM — use OCR + text LLM only");
+        println!("  --ocr-only          Alias for --no-vision-model");
+        println!("  -v, --verbose       Enable debug logging");
         return Ok(());
     }
 
@@ -170,25 +192,31 @@ fn run_server(args: &[String]) -> Result<()> {
         .build()?;
 
     rt.block_on(async {
-        let config = crate::openhuman::config::Config::load_or_init()
+        let mut config = crate::openhuman::config::Config::load_or_init()
             .await
             .map_err(|e| anyhow::anyhow!("config load failed: {e}"))?;
 
+        if opts.no_vision_model {
+            config.screen_intelligence.use_vision_model = false;
+        }
+
         let server_config = crate::openhuman::screen_intelligence::server::SiServerConfig {
             ttl_secs: opts.ttl_secs,
             log_interval_secs: 5,
             keep_screenshots: opts.keep,
         };
 
+        let mode_label = if config.screen_intelligence.use_vision_model {
+            format!("vision LLM ({})", config.local_ai.vision_model_id)
+        } else {
+            "OCR + text LLM (no vision model)".to_string()
+        };
+
         eprintln!();
         eprintln!("  Screen Intelligence");
         eprintln!("  ───────────────────");
         eprintln!("  TTL:              {}s", opts.ttl_secs);
-        eprintln!(
-            "  Vision:           {}",
-            config.screen_intelligence.vision_enabled
-        );
-        eprintln!("  Vision model:     {}", config.local_ai.vision_model_id);
+        eprintln!("  Mode:             {}", mode_label);
         eprintln!(
             "  FPS:              {}",
             config.screen_intelligence.baseline_fps
@@ -334,13 +362,15 @@ fn run_capture(args: &[String]) -> Result<()> {
 /// `openhuman screen-intelligence start` — start a capture + vision session.
 fn run_start_session(args: &[String]) -> Result<()> {
     if args.iter().any(|a| is_help(a)) {
-        println!("Usage: openhuman screen-intelligence start [--ttl <secs>] [-v]");
+        println!("Usage: openhuman screen-intelligence start [--ttl <secs>] [--no-vision-model] [-v]");
         println!();
         println!("Start a screen intelligence capture session with vision analysis.");
         println!("The session runs until TTL expires or Ctrl+C is pressed.");
         println!();
-        println!("  --ttl <secs>     Session duration (default: 300, max: 3600)");
-        println!("  -v, --verbose    Enable debug logging");
+        println!("  --ttl <secs>        Session duration (default: 300, max: 3600)");
+        println!("  --no-vision-model   Skip the vision LLM — use OCR + text LLM only");
+        println!("  --ocr-only          Alias for --no-vision-model");
+        println!("  -v, --verbose       Enable debug logging");
         return Ok(());
     }
 
@@ -355,7 +385,7 @@ fn run_start_session(args: &[String]) -> Result<()> {
         .build()?;
 
     rt.block_on(async {
-        let engine = bootstrap_engine(opts.verbose).await?;
+        let engine = bootstrap_engine_with_opts(opts.verbose, opts.no_vision_model).await?;
 
         let params = crate::openhuman::screen_intelligence::StartSessionParams {
             consent: true,
@@ -501,6 +531,7 @@ fn run_doctor(args: &[String]) -> Result<()> {
             eprintln!("  Config:");
             eprintln!("    enabled:           {}", si.enabled);
             eprintln!("    vision_enabled:    {}", si.vision_enabled);
+            eprintln!("    use_vision_model:  {}", si.use_vision_model);
             eprintln!("    baseline_fps:      {}", si.baseline_fps);
             eprintln!("    keep_screenshots:  {}", si.keep_screenshots);
             eprintln!("    local_ai.enabled:  {}", la.enabled);
@@ -536,6 +567,7 @@ fn run_doctor(args: &[String]) -> Result<()> {
                 "config": config.as_ref().map(|c| serde_json::json!({
                     "enabled": c.screen_intelligence.enabled,
                     "vision_enabled": c.screen_intelligence.vision_enabled,
+                    "use_vision_model": c.screen_intelligence.use_vision_model,
                     "baseline_fps": c.screen_intelligence.baseline_fps,
                     "keep_screenshots": c.screen_intelligence.keep_screenshots,
                     "local_ai_enabled": c.local_ai.enabled,
@@ -640,10 +672,10 @@ fn print_help() {
     println!("Boots only the screen intelligence engine (accessibility capture + local-AI");
     println!("vision) without the full desktop app, Socket.IO, or skills runtime.\n");
     println!("Usage:");
-    println!("  openhuman screen-intelligence run       [--ttl <secs>] [-v]");
+    println!("  openhuman screen-intelligence run       [--ttl <secs>] [--no-vision-model] [-v]");
     println!("  openhuman screen-intelligence status     [-v]");
     println!("  openhuman screen-intelligence capture    [--keep] [-v]");
-    println!("  openhuman screen-intelligence start      [--ttl <secs>] [-v]");
+    println!("  openhuman screen-intelligence start      [--ttl <secs>] [--no-vision-model] [-v]");
     println!("  openhuman screen-intelligence stop       [-v]");
     println!("  openhuman screen-intelligence doctor     [-v]");
     println!("  openhuman screen-intelligence vision     [--limit <n>] [-v]");
@@ -658,8 +690,10 @@ fn print_help() {
     println!("  vision    Print recent vision summaries from the active session");
     println!();
     println!("Common options:");
-    println!("  --ttl <secs>     Session TTL (default: 300)");
-    println!("  --limit <n>      Max vision summaries for 'vision' (default: 10)");
-    println!("  --keep           Save screenshot to disk (for 'capture')");
-    println!("  -v, --verbose    Enable debug logging");
+    println!("  --ttl <secs>        Session TTL (default: 300)");
+    println!("  --limit <n>         Max vision summaries for 'vision' (default: 10)");
+    println!("  --keep              Save screenshot to disk (for 'capture')");
+    println!("  --no-vision-model   Skip vision LLM — use OCR + text LLM only");
+    println!("  --ocr-only          Alias for --no-vision-model");
+    println!("  -v, --verbose       Enable debug logging");
 }

From ae0f186a566a1a9d823828de9fe40a76dac884bd Mon Sep 17 00:00:00 2001
From: Steven Enamakel <enamakel@tinyhumans.ai>
Date: Wed, 8 Apr 2026 01:03:29 -0700
Subject: [PATCH 3/4] fix(screen-intelligence): read use_vision_model from
 engine runtime config

The processing worker was reading use_vision_model from the persisted
config file (Config::load_or_init), so the CLI --no-vision-model flag
had no effect. Now reads from the engine's in-memory runtime config
which the CLI correctly overrides via apply_config(). Also moves image
compression before OCR pass.
---
 .../screen_intelligence/processing_worker.rs  | 22 ++++++++++++++-----
 1 file changed, 16 insertions(+), 6 deletions(-)

diff --git a/src/openhuman/screen_intelligence/processing_worker.rs b/src/openhuman/screen_intelligence/processing_worker.rs
index ddcb4476659..bb1157cea2c 100644
--- a/src/openhuman/screen_intelligence/processing_worker.rs
+++ b/src/openhuman/screen_intelligence/processing_worker.rs
@@ -183,7 +183,7 @@ pub(crate) async fn run(
 ///
 /// Public within the crate so `engine.rs` can call it for flush/diagnostics.
 pub(crate) async fn analyze_frame(
-    _engine: &AccessibilityEngine,
+    engine: &AccessibilityEngine,
     frame: CaptureFrame,
 ) -> Result<VisionSummary, String> {
     let image_ref = frame
@@ -201,6 +201,12 @@ pub(crate) async fn analyze_frame(
         }
     }
 
+    // ── Read use_vision_model from the engine's runtime config ─────
+    // The CLI `--no-vision-model` flag overrides this at runtime without
+    // persisting to disk, so we read from the engine state, not from the
+    // persisted config file.
+    let use_vision_model = engine.inner.lock().await.config.use_vision_model;
+
     // ── Validate config before doing any work ─────────────────────────
     let config = Config::load_or_init()
         .await
@@ -217,6 +223,15 @@ pub(crate) async fn analyze_frame(
         ));
     }
 
+    tracing::debug!(
+        "[processing_worker] use_vision_model={} (from engine runtime config)",
+        use_vision_model,
+    );
+
+    // ── Image compression (always runs — used by vision LLM and/or storage) ──
+    let compressed = super::image_processing::compress_screenshot(&image_ref, None, None)
+        .map_err(|e| format!("image compression failed: {e}"))?;
+
     // ── Pass 1: OCR via Apple Vision ────────────────────────────────
     tracing::debug!("[processing_worker] pass 1/3: Apple Vision OCR");
     let ocr_text = tokio::time::timeout(
@@ -227,13 +242,8 @@ pub(crate) async fn analyze_frame(
     .map_err(|_| "Apple Vision OCR timed out after 30s".to_string())??;
     tracing::debug!("[processing_worker] OCR extracted {} chars", ocr_text.len());
 
-    // ── Resolve use_vision_model flag from config ─────────────────
-    let use_vision_model = config.screen_intelligence.use_vision_model;
-
     // ── Pass 2: Vision LLM for context (skipped when use_vision_model=false) ──
     let vision_context = if use_vision_model {
-        let compressed = super::image_processing::compress_screenshot(&image_ref, None, None)
-            .map_err(|e| format!("image compression failed: {e}"))?;
         let vision_image_ref = compressed.data_uri;
 
         tracing::debug!(

From 7fb8506b43d9d4ac0850f2cd5566484c1952893d Mon Sep 17 00:00:00 2001
From: Steven Enamakel <enamakel@tinyhumans.ai>
Date: Wed, 8 Apr 2026 01:05:25 -0700
Subject: [PATCH 4/4] fix: add use_vision_model to test fixtures and fix
 rustfmt

Add the new use_vision_model field to all AccessibilityConfig test
fixtures so TypeScript compilation passes. Also includes rustfmt
auto-fix for screen_intelligence_cli.rs.
---
 .../__tests__/ScreenIntelligenceDebugPanel.test.tsx           | 1 +
 .../settings/panels/__tests__/AccessibilityPanel.test.tsx     | 1 +
 .../onboarding/steps/__tests__/ScreenPermissionsStep.test.tsx | 1 +
 app/src/services/__tests__/coreRpcClient.test.ts              | 1 +
 app/src/store/__tests__/accessibilitySlice.test.ts            | 1 +
 src/core/screen_intelligence_cli.rs                           | 4 +++-
 6 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/app/src/components/intelligence/__tests__/ScreenIntelligenceDebugPanel.test.tsx b/app/src/components/intelligence/__tests__/ScreenIntelligenceDebugPanel.test.tsx
index 9d1c82c6aa3..aa1462b51f4 100644
--- a/app/src/components/intelligence/__tests__/ScreenIntelligenceDebugPanel.test.tsx
+++ b/app/src/components/intelligence/__tests__/ScreenIntelligenceDebugPanel.test.tsx
@@ -74,6 +74,7 @@ const sampleStatus: AccessibilityStatus = {
     session_ttl_secs: 300,
     panic_stop_hotkey: 'Cmd+Shift+.',
     autocomplete_enabled: true,
+    use_vision_model: true,
     keep_screenshots: false,
     allowlist: [],
     denylist: [],
diff --git a/app/src/components/settings/panels/__tests__/AccessibilityPanel.test.tsx b/app/src/components/settings/panels/__tests__/AccessibilityPanel.test.tsx
index fe180b9580c..b01ec551260 100644
--- a/app/src/components/settings/panels/__tests__/AccessibilityPanel.test.tsx
+++ b/app/src/components/settings/panels/__tests__/AccessibilityPanel.test.tsx
@@ -46,6 +46,7 @@ const status: AccessibilityStatus = {
     session_ttl_secs: 300,
     panic_stop_hotkey: 'Cmd+Shift+.',
     autocomplete_enabled: true,
+    use_vision_model: true,
     keep_screenshots: false,
     allowlist: [],
     denylist: ['wallet'],
diff --git a/app/src/pages/onboarding/steps/__tests__/ScreenPermissionsStep.test.tsx b/app/src/pages/onboarding/steps/__tests__/ScreenPermissionsStep.test.tsx
index a5258fb68ed..4e854e3a53e 100644
--- a/app/src/pages/onboarding/steps/__tests__/ScreenPermissionsStep.test.tsx
+++ b/app/src/pages/onboarding/steps/__tests__/ScreenPermissionsStep.test.tsx
@@ -62,6 +62,7 @@ const deniedStatus: AccessibilityStatus = {
     session_ttl_secs: 300,
     panic_stop_hotkey: 'Cmd+Shift+.',
     autocomplete_enabled: true,
+    use_vision_model: true,
     keep_screenshots: false,
     allowlist: [],
     denylist: [],
diff --git a/app/src/services/__tests__/coreRpcClient.test.ts b/app/src/services/__tests__/coreRpcClient.test.ts
index 1a2b6290c8f..f3d6edce8b0 100644
--- a/app/src/services/__tests__/coreRpcClient.test.ts
+++ b/app/src/services/__tests__/coreRpcClient.test.ts
@@ -41,6 +41,7 @@ function sampleAccessibilityStatus(
       session_ttl_secs: 300,
       panic_stop_hotkey: 'Cmd+Shift+.',
       autocomplete_enabled: true,
+      use_vision_model: true,
       keep_screenshots: false,
       allowlist: [],
       denylist: [],
diff --git a/app/src/store/__tests__/accessibilitySlice.test.ts b/app/src/store/__tests__/accessibilitySlice.test.ts
index 561c4f04133..3cd7711404f 100644
--- a/app/src/store/__tests__/accessibilitySlice.test.ts
+++ b/app/src/store/__tests__/accessibilitySlice.test.ts
@@ -45,6 +45,7 @@ const sampleStatus: AccessibilityStatus = {
     session_ttl_secs: 300,
     panic_stop_hotkey: 'Cmd+Shift+.',
     autocomplete_enabled: true,
+    use_vision_model: true,
     keep_screenshots: false,
     allowlist: [],
     denylist: ['wallet'],
diff --git a/src/core/screen_intelligence_cli.rs b/src/core/screen_intelligence_cli.rs
index dfa1d751fdd..ce165f151d7 100644
--- a/src/core/screen_intelligence_cli.rs
+++ b/src/core/screen_intelligence_cli.rs
@@ -362,7 +362,9 @@ fn run_capture(args: &[String]) -> Result<()> {
 /// `openhuman screen-intelligence start` — start a capture + vision session.
 fn run_start_session(args: &[String]) -> Result<()> {
     if args.iter().any(|a| is_help(a)) {
-        println!("Usage: openhuman screen-intelligence start [--ttl <secs>] [--no-vision-model] [-v]");
+        println!(
+            "Usage: openhuman screen-intelligence start [--ttl <secs>] [--no-vision-model] [-v]"
+        );
         println!();
         println!("Start a screen intelligence capture session with vision analysis.");
         println!("The session runs until TTL expires or Ctrl+C is pressed.");