Explain OpenResearch runs lost before the remote runner starts (OR-634)

src/commands/supervise.rs

@@ -1110,6 +1110,7 @@ async fn run_openresearch(
             Err(_) => break false,
         }
     };
+    let mut launch_err = None;
     if !already_launched {
         if bootstrap_heartbeat.is_none() {
             bootstrap_heartbeat = descriptor
@@ -1139,7 +1140,6 @@ async fn run_openresearch(
         }
         // sshd and the org key sync can lag a freshly-online box, so the
         // launch retries for ~2 minutes before giving up.
-        let mut launch_err = None;
         for backoff_secs in [0u64, 5, 10, 20, 30, 45] {
             if backoff_secs > 0 {
                 tokio::time::sleep(Duration::from_secs(backoff_secs)).await;
@@ -1182,7 +1182,7 @@ async fn run_openresearch(
                 }
             }
         }
-        if let Some(err) = launch_err.filter(|_| descriptor.rental_lease.is_none()) {
+        if let Some(err) = launch_err.take_if(|_| descriptor.rental_lease.is_none()) {
             if store.update_status(&run_id, RunStatus::Failed, Some(now_ms()), None)? {
                 store.set_result_markdown(
                     &run_id,
@@ -1216,6 +1216,17 @@ async fn run_openresearch(
     if descriptor.rental_lease.is_none() {
         teardown_box(&store, &lifecycle, &sandbox_id, &run_id).await;
     }
+    // Only a supervisor seen dying before its claim proves the run never started.
+    let never_started = launch_err.filter(|err| {
+        err.to_string()
+            .contains(crate::jobs::rental::SUPERVISOR_EXITED)
+    });
+    if let (Ok(RunStatus::Lost), Some(err)) = (&watch, never_started) {
+        store.set_result_markdown(
+            &run_id,
+            &format!("Outcome unavailable: the run never started on box {sandbox_id}: {err}"),
+        )?;
+    }
     watch?;
     Ok(())
 }

src/jobs/rental.rs

@@ -163,7 +163,7 @@ fn remote_supervisor_command(binary: &str, directory: &str) -> String {
     use super::ssh::sh_quote;
     // An asynchronous `cd && nohup` list keeps the SSH output pipes open.
     format!(
-        "cd {} || exit; nohup {} rental-supervise {} </dev/null > supervisor.log 2>&1 &",
+        "cd {} || exit; nohup {} rental-supervise {} </dev/null > supervisor.log 2>&1 & echo $! > supervisor.pid",
         sh_quote(directory),
         sh_quote(binary),
         sh_quote(&format!("{directory}/rental.json"))
@@ -238,9 +238,26 @@ pub async fn launch(
     .await?;
     let command = remote_supervisor_command(binary, &path);
     ssh_run(target, &command, None).await?;
+    ssh_run(target, &supervisor_started_script(&path), None).await?;
     Ok(())
 }
 
+pub const SUPERVISOR_EXITED: &str = "remote supervisor exited before starting the run:";
+
+/// Fails with the supervisor's output when it exits before claiming the launch.
+fn supervisor_started_script(directory: &str) -> String {
+    format!(
+        "d={}; i=0; while [ $i -lt 40 ]; do [ -e \"$d/launch_started\" ] && exit 0; \
+         if ! kill -0 \"$(cat \"$d/supervisor.pid\")\" 2>/dev/null; then \
+         [ -s \"$d/supervisor.log\" ] || exit 0; \
+         echo {} >&2; \
+         tail -c 2000 \"$d/supervisor.log\" >&2; exit 1; fi; \
+         i=$((i+1)); sleep 0.25; done",
+        super::ssh::sh_quote(directory),
+        super::ssh::sh_quote(SUPERVISOR_EXITED)
+    )
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -272,6 +289,34 @@ mod tests {
         .await
         .unwrap();
     }
+    #[cfg(unix)]
+    #[test]
+    fn startup_check_reports_a_supervisor_that_died_before_launch() {
+        let temp = crate::local::git::TemporaryDirectory::new("orx-rental-startup").unwrap();
+        let dir = temp.path();
+        let check = || {
+            std::process::Command::new("/bin/sh")
+                .args(["-c", &supervisor_started_script(dir.to_str().unwrap())])
+                .output()
+                .unwrap()
+        };
+        std::fs::write(dir.join("supervisor.pid"), "999999999").unwrap();
+        std::fs::write(
+            dir.join("supervisor.log"),
+            "Permission denied (os error 13)\n",
+        )
+        .unwrap();
+        let died = check();
+        assert!(!died.status.success());
+        let stderr = String::from_utf8_lossy(&died.stderr);
+        assert!(
+            stderr.contains(SUPERVISOR_EXITED)
+                && stderr.contains("Permission denied (os error 13)")
+        );
+        std::fs::write(dir.join("launch_started"), "").unwrap();
+        assert!(check().status.success());
+    }
+
     #[cfg(unix)]
     #[test]
     fn cancellation_waits_for_a_pid_or_terminal_acknowledgment() {

src/plane.rs

@@ -34,7 +34,7 @@ impl Run {
         if self.status == "cancelled" {
             return self.cancel_reason.as_ref().map(|r| format!("reason: {r}"));
         }
-        if self.status != "failed" {
+        if !matches!(self.status.as_str(), "failed" | "lost") {
             return None;
         }
         match self.result_markdown.as_deref().map(str::trim) {
@@ -166,6 +166,16 @@ mod tests {
         );
     }
 
+    #[test]
+    fn lost_run_reports_its_recorded_reason() {
+        let mut run = Run::from(&stored_run("lost", None));
+        run.result_markdown = Some("Outcome unavailable: box b1 was deleted".into());
+        assert_eq!(
+            run.failure_detail().as_deref(),
+            Some("reason: Outcome unavailable: box b1 was deleted")
+        );
+    }
+
     #[test]
     fn cancelled_run_reports_who_requested_the_cancel() {
         let mut run = Run::from(&stored_run("cancelled", None));