feat(screen-intelligence): OCR-only mode without vision model

app/src/components/intelligence/__tests__/ScreenIntelligenceDebugPanel.test.tsx

@@ -74,6 +74,7 @@ const sampleStatus: AccessibilityStatus = {
     session_ttl_secs: 300,
     panic_stop_hotkey: 'Cmd+Shift+.',
     autocomplete_enabled: true,
+    use_vision_model: true,
     keep_screenshots: false,
     allowlist: [],
     denylist: [],

app/src/components/settings/panels/ScreenIntelligencePanel.tsx

@@ -71,6 +71,7 @@ const ScreenIntelligencePanel = () => {
     'all_except_blacklist'
   );
   const [baselineFps, setBaselineFps] = useState<string>('1');
+  const [useVisionModel, setUseVisionModel] = useState<boolean>(true);
   const [keepScreenshots, setKeepScreenshots] = useState<boolean>(false);
   const [allowlistText, setAllowlistText] = useState('');
   const [denylistText, setDenylistText] = useState('');
@@ -105,6 +106,7 @@ const ScreenIntelligencePanel = () => {
       status.config.policy_mode === 'whitelist_only' ? 'whitelist_only' : 'all_except_blacklist'
     );
     setBaselineFps(String(status.config.baseline_fps ?? 1));
+    setUseVisionModel(status.config.use_vision_model ?? true);
     setKeepScreenshots(status.config.keep_screenshots ?? false);
     setAllowlistText((status.config.allowlist ?? []).join('\n'));
     setDenylistText((status.config.denylist ?? []).join('\n'));
@@ -145,6 +147,7 @@ const ScreenIntelligencePanel = () => {
         enabled,
         policy_mode: policyMode,
         baseline_fps: Number.isFinite(fps) && fps > 0 ? fps : 1,
+        use_vision_model: useVisionModel,
         keep_screenshots: keepScreenshots,
         allowlist: allowlistText
           .split('\n')
@@ -282,6 +285,21 @@ const ScreenIntelligencePanel = () => {
             />
           </label>
 
+          <label className="flex items-center justify-between rounded-xl border border-stone-200 bg-stone-50 px-3 py-2">
+            <div>
+              <span className="text-sm text-stone-700">Use Vision Model</span>
+              <p className="text-xs text-stone-400">
+                Send screenshots to a vision LLM for richer context. When off, only OCR text is used
+                with a text LLM — faster and no vision model required.
+              </p>
+            </div>
+            <input
+              type="checkbox"
+              checked={useVisionModel}
+              onChange={event => setUseVisionModel(event.target.checked)}
+            />
+          </label>
+
           <label className="flex items-center justify-between rounded-xl border border-stone-200 bg-stone-50 px-3 py-2">
             <div>
               <span className="text-sm text-stone-700">Keep Screenshots</span>

app/src/components/settings/panels/__tests__/AccessibilityPanel.test.tsx

@@ -46,6 +46,7 @@ const status: AccessibilityStatus = {
     session_ttl_secs: 300,
     panic_stop_hotkey: 'Cmd+Shift+.',
     autocomplete_enabled: true,
+    use_vision_model: true,
     keep_screenshots: false,
     allowlist: [],
     denylist: ['wallet'],

app/src/components/settings/panels/__tests__/ScreenIntelligencePanel.test.tsx

@@ -78,6 +78,7 @@ const baseStatus: AccessibilityStatus = {
     session_ttl_secs: 300,
     panic_stop_hotkey: 'Cmd+Shift+.',
     autocomplete_enabled: true,
+    use_vision_model: true,
     keep_screenshots: false,
     allowlist: ['Code'],
     denylist: ['1Password'],
@@ -189,6 +190,7 @@ describe('ScreenIntelligencePanel', () => {
       enabled: true,
       policy_mode: 'all_except_blacklist',
       baseline_fps: 1,
+      use_vision_model: true,
       keep_screenshots: false,
       allowlist: ['Code'],
       denylist: ['1Password'],

app/src/pages/onboarding/steps/__tests__/ScreenPermissionsStep.test.tsx

@@ -62,6 +62,7 @@ const deniedStatus: AccessibilityStatus = {
     session_ttl_secs: 300,
     panic_stop_hotkey: 'Cmd+Shift+.',
     autocomplete_enabled: true,
+    use_vision_model: true,
     keep_screenshots: false,
     allowlist: [],
     denylist: [],

app/src/services/__tests__/coreRpcClient.test.ts

@@ -41,6 +41,7 @@ function sampleAccessibilityStatus(
       session_ttl_secs: 300,
       panic_stop_hotkey: 'Cmd+Shift+.',
       autocomplete_enabled: true,
+      use_vision_model: true,
       keep_screenshots: false,
       allowlist: [],
       denylist: [],

app/src/store/__tests__/accessibilitySlice.test.ts

@@ -45,6 +45,7 @@ const sampleStatus: AccessibilityStatus = {
     session_ttl_secs: 300,
     panic_stop_hotkey: 'Cmd+Shift+.',
     autocomplete_enabled: true,
+    use_vision_model: true,
     keep_screenshots: false,
     allowlist: [],
     denylist: ['wallet'],

app/src/utils/tauriCommands/accessibility.ts

@@ -46,6 +46,7 @@ export interface AccessibilityConfig {
   session_ttl_secs: number;
   panic_stop_hotkey: string;
   autocomplete_enabled: boolean;
+  use_vision_model: boolean;
   keep_screenshots: boolean;
   allowlist: string[];
   denylist: string[];

app/src/utils/tauriCommands/config.ts

@@ -41,6 +41,7 @@ export interface ScreenIntelligenceSettingsUpdate {
   baseline_fps?: number | null;
   vision_enabled?: boolean | null;
   autocomplete_enabled?: boolean | null;
+  use_vision_model?: boolean | null;
   keep_screenshots?: boolean | null;
   allowlist?: string[] | null;
   denylist?: string[] | null;

src/core/screen_intelligence_cli.rs

@@ -46,18 +46,24 @@ struct CliOpts {
     ttl_secs: u64,
     keep: bool,
     limit: usize,
+    no_vision_model: bool,
 }
 
 fn parse_opts(args: &[String]) -> Result<(CliOpts, Vec<String>)> {
     let mut verbose = false;
     let mut ttl_secs: u64 = 300;
     let mut keep = false;
     let mut limit: usize = 10;
+    let mut no_vision_model = false;
     let mut rest = Vec::new();
     let mut i = 0;
 
     while i < args.len() {
         match args[i].as_str() {
+            "--no-vision-model" | "--ocr-only" => {
+                no_vision_model = true;
+                i += 1;
+            }
             "--ttl" => {
                 let val = args
                     .get(i + 1)
@@ -101,6 +107,7 @@ fn parse_opts(args: &[String]) -> Result<(CliOpts, Vec<String>)> {
             ttl_secs,
             keep,
             limit,
+            no_vision_model,
         },
         rest,
     ))
@@ -109,24 +116,37 @@ fn parse_opts(args: &[String]) -> Result<(CliOpts, Vec<String>)> {
 /// Bootstrap the screen intelligence engine with config.
 async fn bootstrap_engine(
     verbose: bool,
+) -> Result<Arc<crate::openhuman::screen_intelligence::AccessibilityEngine>> {
+    bootstrap_engine_with_opts(verbose, false).await
+}
+
+/// Bootstrap with CLI overrides.
+async fn bootstrap_engine_with_opts(
+    verbose: bool,
+    no_vision_model: bool,
 ) -> Result<Arc<crate::openhuman::screen_intelligence::AccessibilityEngine>> {
     use crate::openhuman::config::Config;
     use crate::openhuman::screen_intelligence::global_engine;
 
-    let config = Config::load_or_init()
+    let mut config = Config::load_or_init()
         .await
         .map_err(|e| anyhow::anyhow!("config load failed: {e}"))?;
 
+    if no_vision_model {
+        config.screen_intelligence.use_vision_model = false;
+    }
+
     let engine = global_engine();
     let _ = engine
         .apply_config(config.screen_intelligence.clone())
         .await;
 
     if verbose {
         log::info!(
-            "[screen-intelligence-cli] engine initialized, enabled={}, vision={}, keep_screenshots={}, workspace={}",
+            "[screen-intelligence-cli] engine initialized, enabled={}, vision={}, use_vision_model={}, keep_screenshots={}, workspace={}",
             config.screen_intelligence.enabled,
             config.screen_intelligence.vision_enabled,
+            config.screen_intelligence.use_vision_model,
             config.screen_intelligence.keep_screenshots,
             config.workspace_dir.display(),
         );
@@ -148,15 +168,17 @@ fn run_server(args: &[String]) -> Result<()> {
     let (opts, rest) = parse_opts(args)?;
 
     if rest.iter().any(|a| is_help(a)) {
-        println!("Usage: openhuman screen-intelligence run [--ttl <secs>] [--keep] [-v]");
+        println!("Usage: openhuman screen-intelligence run [--ttl <secs>] [--keep] [--no-vision-model] [-v]");
         println!();
         println!("Start the screen intelligence capture + vision loop.");
-        println!("Captures screenshots at baseline FPS, sends to vision model,");
+        println!("Captures screenshots at baseline FPS, runs OCR and vision analysis,");
         println!("and logs summaries. Blocks until TTL expires or Ctrl+C.");
         println!();
-        println!("  --ttl <secs>     Session duration (default: 300)");
-        println!("  --keep           Keep screenshots on disk after vision processing");
-        println!("  -v, --verbose    Enable debug logging");
+        println!("  --ttl <secs>        Session duration (default: 300)");
+        println!("  --keep              Keep screenshots on disk after vision processing");
+        println!("  --no-vision-model   Skip the vision LLM — use OCR + text LLM only");
+        println!("  --ocr-only          Alias for --no-vision-model");
+        println!("  -v, --verbose       Enable debug logging");
         return Ok(());
     }
 
@@ -170,25 +192,31 @@ fn run_server(args: &[String]) -> Result<()> {
         .build()?;
 
     rt.block_on(async {
-        let config = crate::openhuman::config::Config::load_or_init()
+        let mut config = crate::openhuman::config::Config::load_or_init()
             .await
             .map_err(|e| anyhow::anyhow!("config load failed: {e}"))?;
 
+        if opts.no_vision_model {
+            config.screen_intelligence.use_vision_model = false;
+        }
+
         let server_config = crate::openhuman::screen_intelligence::server::SiServerConfig {
             ttl_secs: opts.ttl_secs,
             log_interval_secs: 5,
             keep_screenshots: opts.keep,
         };
 
+        let mode_label = if config.screen_intelligence.use_vision_model {
+            format!("vision LLM ({})", config.local_ai.vision_model_id)
+        } else {
+            "OCR + text LLM (no vision model)".to_string()
+        };
+
         eprintln!();
         eprintln!("  Screen Intelligence");
         eprintln!("  ───────────────────");
         eprintln!("  TTL:              {}s", opts.ttl_secs);
-        eprintln!(
-            "  Vision:           {}",
-            config.screen_intelligence.vision_enabled
-        );
-        eprintln!("  Vision model:     {}", config.local_ai.vision_model_id);
+        eprintln!("  Mode:             {}", mode_label);
         eprintln!(
             "  FPS:              {}",
             config.screen_intelligence.baseline_fps
@@ -334,13 +362,17 @@ fn run_capture(args: &[String]) -> Result<()> {
 /// `openhuman screen-intelligence start` — start a capture + vision session.
 fn run_start_session(args: &[String]) -> Result<()> {
     if args.iter().any(|a| is_help(a)) {
-        println!("Usage: openhuman screen-intelligence start [--ttl <secs>] [-v]");
+        println!(
+            "Usage: openhuman screen-intelligence start [--ttl <secs>] [--no-vision-model] [-v]"
+        );
         println!();
         println!("Start a screen intelligence capture session with vision analysis.");
         println!("The session runs until TTL expires or Ctrl+C is pressed.");
         println!();
-        println!("  --ttl <secs>     Session duration (default: 300, max: 3600)");
-        println!("  -v, --verbose    Enable debug logging");
+        println!("  --ttl <secs>        Session duration (default: 300, max: 3600)");
+        println!("  --no-vision-model   Skip the vision LLM — use OCR + text LLM only");
+        println!("  --ocr-only          Alias for --no-vision-model");
+        println!("  -v, --verbose       Enable debug logging");
         return Ok(());
     }
 
@@ -355,7 +387,7 @@ fn run_start_session(args: &[String]) -> Result<()> {
         .build()?;
 
     rt.block_on(async {
-        let engine = bootstrap_engine(opts.verbose).await?;
+        let engine = bootstrap_engine_with_opts(opts.verbose, opts.no_vision_model).await?;
 
         let params = crate::openhuman::screen_intelligence::StartSessionParams {
             consent: true,
@@ -501,6 +533,7 @@ fn run_doctor(args: &[String]) -> Result<()> {
             eprintln!("  Config:");
             eprintln!("    enabled:           {}", si.enabled);
             eprintln!("    vision_enabled:    {}", si.vision_enabled);
+            eprintln!("    use_vision_model:  {}", si.use_vision_model);
             eprintln!("    baseline_fps:      {}", si.baseline_fps);
             eprintln!("    keep_screenshots:  {}", si.keep_screenshots);
             eprintln!("    local_ai.enabled:  {}", la.enabled);
@@ -536,6 +569,7 @@ fn run_doctor(args: &[String]) -> Result<()> {
                 "config": config.as_ref().map(|c| serde_json::json!({
                     "enabled": c.screen_intelligence.enabled,
                     "vision_enabled": c.screen_intelligence.vision_enabled,
+                    "use_vision_model": c.screen_intelligence.use_vision_model,
                     "baseline_fps": c.screen_intelligence.baseline_fps,
                     "keep_screenshots": c.screen_intelligence.keep_screenshots,
                     "local_ai_enabled": c.local_ai.enabled,
@@ -640,10 +674,10 @@ fn print_help() {
     println!("Boots only the screen intelligence engine (accessibility capture + local-AI");
     println!("vision) without the full desktop app, Socket.IO, or skills runtime.\n");
     println!("Usage:");
-    println!("  openhuman screen-intelligence run       [--ttl <secs>] [-v]");
+    println!("  openhuman screen-intelligence run       [--ttl <secs>] [--no-vision-model] [-v]");
     println!("  openhuman screen-intelligence status     [-v]");
     println!("  openhuman screen-intelligence capture    [--keep] [-v]");
-    println!("  openhuman screen-intelligence start      [--ttl <secs>] [-v]");
+    println!("  openhuman screen-intelligence start      [--ttl <secs>] [--no-vision-model] [-v]");
     println!("  openhuman screen-intelligence stop       [-v]");
     println!("  openhuman screen-intelligence doctor     [-v]");
     println!("  openhuman screen-intelligence vision     [--limit <n>] [-v]");
@@ -658,8 +692,10 @@ fn print_help() {
     println!("  vision    Print recent vision summaries from the active session");
     println!();
     println!("Common options:");
-    println!("  --ttl <secs>     Session TTL (default: 300)");
-    println!("  --limit <n>      Max vision summaries for 'vision' (default: 10)");
-    println!("  --keep           Save screenshot to disk (for 'capture')");
-    println!("  -v, --verbose    Enable debug logging");
+    println!("  --ttl <secs>        Session TTL (default: 300)");
+    println!("  --limit <n>         Max vision summaries for 'vision' (default: 10)");
+    println!("  --keep              Save screenshot to disk (for 'capture')");
+    println!("  --no-vision-model   Skip vision LLM — use OCR + text LLM only");
+    println!("  --ocr-only          Alias for --no-vision-model");
+    println!("  -v, --verbose       Enable debug logging");
 }

src/openhuman/config/ops.rs

@@ -164,6 +164,7 @@ pub struct ScreenIntelligenceSettingsPatch {
     pub baseline_fps: Option<f32>,
     pub vision_enabled: Option<bool>,
     pub autocomplete_enabled: Option<bool>,
+    pub use_vision_model: Option<bool>,
     pub keep_screenshots: Option<bool>,
     pub allowlist: Option<Vec<String>>,
     pub denylist: Option<Vec<String>>,
@@ -282,6 +283,9 @@ pub async fn apply_screen_intelligence_settings(
     if let Some(autocomplete_enabled) = update.autocomplete_enabled {
         config.screen_intelligence.autocomplete_enabled = autocomplete_enabled;
     }
+    if let Some(use_vision_model) = update.use_vision_model {
+        config.screen_intelligence.use_vision_model = use_vision_model;
+    }
     if let Some(keep_screenshots) = update.keep_screenshots {
         config.screen_intelligence.keep_screenshots = keep_screenshots;
     }

src/openhuman/config/schema/accessibility.rs

@@ -19,6 +19,12 @@ pub struct ScreenIntelligenceConfig {
     pub panic_stop_hotkey: String,
     #[serde(default = "default_autocomplete_enabled")]
     pub autocomplete_enabled: bool,
+    /// When `true`, Pass 2 sends the screenshot to a vision-capable LLM for
+    /// visual context extraction.  When `false`, only Apple Vision OCR (Pass 1)
+    /// feeds into the text synthesis LLM (Pass 3) — no vision model required.
+    /// Default: `true`.
+    #[serde(default = "default_use_vision_model")]
+    pub use_vision_model: bool,
     /// When `true`, captured screenshots are saved to `{workspace_dir}/screenshots/`
     /// instead of being discarded after vision processing. Default: `false`.
     #[serde(default)]
@@ -61,6 +67,10 @@ fn default_autocomplete_enabled() -> bool {
     true
 }
 
+fn default_use_vision_model() -> bool {
+    true
+}
+
 impl Default for ScreenIntelligenceConfig {
     fn default() -> Self {
         Self {
@@ -72,6 +82,7 @@ impl Default for ScreenIntelligenceConfig {
             session_ttl_secs: default_session_ttl_secs(),
             panic_stop_hotkey: default_panic_stop_hotkey(),
             autocomplete_enabled: default_autocomplete_enabled(),
+            use_vision_model: default_use_vision_model(),
             keep_screenshots: false,
             allowlist: vec![],
             denylist: vec![

src/openhuman/config/schemas.rs

@@ -45,6 +45,7 @@ struct ScreenIntelligenceSettingsUpdate {
     baseline_fps: Option<f32>,
     vision_enabled: Option<bool>,
     autocomplete_enabled: Option<bool>,
+    use_vision_model: Option<bool>,
     keep_screenshots: Option<bool>,
     allowlist: Option<Vec<String>>,
     denylist: Option<Vec<String>>,
@@ -587,6 +588,7 @@ fn handle_update_screen_intelligence_settings(params: Map<String, Value>) -> Con
             baseline_fps: update.baseline_fps,
             vision_enabled: update.vision_enabled,
             autocomplete_enabled: update.autocomplete_enabled,
+            use_vision_model: update.use_vision_model,
             keep_screenshots: update.keep_screenshots,
             allowlist: update.allowlist,
             denylist: update.denylist,