G15: fix stamp wrap and VM slot exhaustion

drivers/gpu/drm/asahi/g15/encode.rs

@@ -1,3 +1,3 @@
 // SPDX-License-Identifier: GPL-2.0-only OR MIT
 
 //! Work-image and microsequence encoders.
@@ -64,10 +64,9 @@
 }
 
 /// Firmware stamp value of ordinal `n` on the queue with this seed.
-/// Stamps step by 0x100 through 0xffffff values and never read 0.
+/// Stamps step by 0x100, including zero when the counter wraps.
 pub(crate) const fn stamp_seed(seed: u32, n: u64) -> u32 {
-    let base = (seed / 0x100) as u64;
-    ((((base - 1 + n % 0xffffff) % 0xffffff) + 1) * 0x100) as u32
+    seed.wrapping_add((n as u32).wrapping_mul(0x100))
 }
 
 /// Compute stamp of ordinal `n`.

drivers/gpu/drm/asahi/g15/engine.rs

@@ -1,3 +1,3 @@
 // SPDX-License-Identifier: GPL-2.0-only OR MIT
 
 //! The prefix/horizon/flight engine.
@@ -155,10 +155,10 @@
     pub(crate) fs_start: u32,
     /// Channel producers (TA, FS, compute) after the prefix's records.
     pub(crate) rec: [u32; 3],
-    /// Compute member j's FW stamp target, and the FS target it waits on directly (0: none):
+    /// Compute member j's FW stamp target, and the FS target it waits on directly:
     /// its predecessors in the prefix (for the compute-hang check).
     pub(crate) cstamp: [u32; sizes::COMPUTE_SLOTS],
-    pub(crate) cwait: [u32; sizes::COMPUTE_SLOTS],
+    pub(crate) cwait: [Option<u32>; sizes::COMPUTE_SLOTS],
     /// Compute slots, pass slots and clustering buffers it holds.
     pub(crate) claims: Claims,
     /// The prefix rebuilds the render allocators (priming); nothing is published behind it.
@@ -189,7 +189,7 @@
             fs_start: 0,
             rec: [0; 3],
             cstamp: [0; sizes::COMPUTE_SLOTS],
-            cwait: [0; sizes::COMPUTE_SLOTS],
+            cwait: [None; sizes::COMPUTE_SLOTS],
             claims: Claims::new(),
             rebuild: false,
             doomed: false,
@@ -385,9 +385,11 @@
     pub(crate) rebuild_live: bool,
     /// A ready prefix waits only for a record's consumption: repoll sooner.
     pub(crate) chan_wait: bool,
+    /// A ready job waits for an ASID held by a completing failed job.
+    pub(crate) slot_wait: bool,
     /// Progress tracking of the oldest prefix.
     pub(crate) progress_flight: u64,
-    pub(crate) progress: [u32; 3],
+    pub(crate) progress: [Option<u32>; 3],
     pub(crate) stats: Stats,
     /// Phase timers (debug_flags bit 49).
     pub(crate) prof: Prof,
@@ -424,8 +426,9 @@
             render_registered: false,
             rebuild_live: false,
             chan_wait: false,
+            slot_wait: false,
             progress_flight: 0,
-            progress: [0; 3],
+            progress: [None; 3],
             stats: Stats::default(),
             prof: Prof::default(),
             debugfs: None,
@@ -546,6 +549,7 @@
         let mut result = Ok(());
 
         self.engine.chan_wait = false;
+        self.engine.slot_wait = false;
         // Nothing is published while a hang request is armed.
         while self.recovery.failed().is_none()
             && !self.engine.frozen
@@ -592,7 +596,7 @@
                         dev_err!(
                             self.graph.dev.as_ref(),
                             "G15: publication of ctx {} failed after a queue WRITE ({:?})\n",
-                            job.vm.slot,
+                            job.slot(),
                             e
                         );
                         complete(done, &job, Err(EIO));
@@ -675,14 +679,27 @@
     /// yet (no free slot, or a channel record still unconsumed); EIO once a queue WRITE
     /// advanced; any other error means nothing was published.
     fn publish_prefix(&mut self, job: &Arc<Job>, done: &mut Completions) -> Result {
+        if let Err(error) = job.bind() {
+            if error == ENOSPC {
+                // A failed job can keep its binding until scheduler references drop after
+                // this pass. Leave healthy work pending and retry without the device lock.
+                self.engine.slot_wait = true;
+                return Err(EAGAIN);
+            }
+            return Err(error);
+        }
         let mut f = Flight::new(job.clone());
         let result = self.publish_prefix_into(job, &mut f);
         match result {
             Err(e) if e != EIO => {
                 // Never published: its storage is free again, and horizon jobs taken into it
                 // fail with it.
                 self.slots.release(&mut f.claims);
+                if job.progress.next() == job.progress.sent() {
+                    job.release_binding();
+                }
                 for j2 in f.group.iter() {
+                    j2.release_binding();
                     complete(done, j2, Err(e));
                 }
             }
@@ -899,11 +916,20 @@
                     hz_stop = Some(1);
                     break;
                 }
-                for k in 0..n2 {
-                    match command(&j2, k).ok_or(EINVAL)? {
-                        Command::Compute(c) => self.prefix_compute(&j2, c, &mut pf, f, false)?,
-                        Command::Render(_) => return Err(EINVAL),
+                j2.bind()?;
+                let encoded = (|| {
+                    for k in 0..n2 {
+                        match command(&j2, k).ok_or(EINVAL)? {
+                            Command::Compute(c) => self.prefix_compute(&j2, c, &mut pf, f, false)?,
+                            Command::Render(_) => return Err(EINVAL),
+                        }
                     }
+                    Ok(())
+                })();
+                if let Err(error) = encoded {
+                    // Not in the horizon yet and nothing of it reached the firmware.
+                    j2.release_binding();
+                    return Err(error);
                 }
                 computes += n2;
                 j2.progress.set_sent(n2);
@@ -1057,7 +1083,7 @@
             Queue,
             self.graph.dev,
             "G15: prefix ctx={} computes {} (n={} head {}) renders {} (draw {} ta/fs head {}/{}) behind {}: {:?}\n",
-            job.vm.slot,
+            job.slot(),
             computes,
             pf.n,
             pf.c_head,
@@ -1095,7 +1121,7 @@
     ) -> Result {
         let n = pf.n + 1;
         let t = encode::stamp(n);
-        let ctx = job.vm.slot;
+        let ctx = job.slot();
         let ts = &cmd.timestamps;
         let mut lap = Lap::start();
         // Nothing published yet on failure.
@@ -1191,11 +1217,7 @@
         }
         let j = pf.computes as usize;
         f.cstamp[j] = t;
-        f.cwait[j] = if pf.prev == Prev::Render {
-            pf.last_fs
-        } else {
-            0
-        };
+        f.cwait[j] = (pf.prev == Prev::Render).then_some(pf.last_fs);
         g.ring.w64(8 * pf.c_head as usize, s.work.va)?;
         pf.c_head = (pf.c_head + 1) % sizes::QUEUE_CAP;
         lap.lap(&mut self.engine.prof.write_ns);
@@ -1240,7 +1262,7 @@
         let k = f.claims.pass[pf.renders as usize] as usize;
         let n = pf.draw + 1;
         let usc = job.queue.usc_base;
-        let ctx = job.vm.slot;
+        let ctx = job.slot();
         let tta = encode::stamp_seed(ids::TA_SEED, n);
         let tfs = encode::stamp_seed(ids::FS_SEED, n);
         // Early tiling: a later render member after a render of the same prefix whose render

drivers/gpu/drm/asahi/g15/hw.rs

@@ -1,3 +1,3 @@
 // SPDX-License-Identifier: GPL-2.0-only OR MIT
 
 //! G15 hardware and firmware-ABI constants (layout values are shared by several modules; change
@@ -518,8 +518,6 @@
     pub(crate) const AHEAD_RECORDS: u32 = 64;
     /// Policy bound on logical queues reported in QUEUE_LIMITS (not a firmware limit).
     pub(crate) const MAX_QUEUES: u32 = 58;
-    /// TTBAT user slots: user VMs hold one each for their lifetime (ASIDs 1..63).
-    pub(crate) const MAX_VMS: u32 = 63;
 }
 
 /// Firmware channels, doorbells, seeds and identifiers.

drivers/gpu/drm/asahi/g15/job.rs

@@ -1,3 +1,3 @@
 // SPDX-License-Identifier: GPL-2.0-only OR MIT
 
 //! Jobs: one per SUBMIT, scheduled by a per-queue DRM scheduler and executed by the engine.
@@ -22,29 +22,35 @@
 //! exactly once with [`Job::complete`] (later calls are ignored), through [`Completions`]
 //! collected under the device lock and [`complete_all`] after it is dropped. A job whose storage
 //! the firmware may still reference stays in `Engine::retained` (failed jobs until reboot): its
-//! [`mmu::VmJobGuard`] keeps the VM's mappings and its VM state keeps the VM's TTBAT slot until
+//! [`mmu::VmJobGuard`] keeps the VM's mappings and its binding keeps the VM's TTBAT slot until
 //! the last `Arc<Job>` drops.
 
-use core::sync::atomic::{AtomicBool, Ordering};
+use core::sync::atomic::{AtomicBool, AtomicU32, Ordering};
 use kernel::{
     bindings, c_str, device,
     dma_fence::{Fence, RawDmaFence},
     drm::sched,
+    new_mutex,
     prelude::*,
-    sync::Arc,
+    sync::{Arc, Mutex},
 };
 
 use super::{command::Command, fence::JobFence, hw, queue::QueueState, vm::VmState, Shared};
 use crate::mmu;
 
 /// A job as the engine sees it.
+#[pin_data]
 pub(crate) struct Job {
     /// Submission ID (debugging).
     pub(crate) id: u64,
     /// The submitting queue (priority, service turn, horizon history).
     pub(crate) queue: Arc<QueueState>,
-    /// The job's VM (TTBAT slot, status).
+    /// The job's VM and status.
     pub(crate) vm: Arc<VmState>,
+    /// Hardware context, assigned immediately before the engine publishes the job.
+    context: AtomicU32,
+    #[pin]
+    binding: Mutex<Option<mmu::VmBind>>,
     /// Validated commands, in submission order (at most 64).
     pub(crate) commands: KVec<Command>,
     /// The job's hardware fence.
@@ -73,21 +79,43 @@
         guard: mmu::VmJobGuard,
     ) -> Result<Arc<Self>> {
         let progress = super::engine::Progress::new(commands.len());
-        Ok(Arc::new(
-            Self {
+        Arc::pin_init(
+            pin_init!(Self {
                 id,
                 queue,
                 vm,
+                context: AtomicU32::new(0),
+                binding <- new_mutex!(None, "g15::Job::binding"),
                 commands,
                 fence,
                 progress,
                 _timestamps: timestamps,
                 _guard: guard,
                 accepted: AtomicBool::new(false),
                 started: AtomicBool::new(false),
-            },
+            }),
             GFP_KERNEL,
-        )?)
+        )
+    }
+
+    pub(crate) fn slot(&self) -> u32 {
+        self.context.load(Ordering::Relaxed)
+    }
+
+    /// Called under the device lock immediately before publication.
+    pub(crate) fn bind(&self) -> Result {
+        let mut slot = self.binding.lock();
+        if slot.is_none() {
+            let binding = self.vm.try_bind()?;
+            self.context.store(binding.slot(), Ordering::Relaxed);
+            *slot = Some(binding);
+        }
+        Ok(())
+    }
+
+    /// All commands retired successfully, or nothing of the job was published.
+    pub(crate) fn release_binding(&self) {
+        drop(self.binding.lock().take());
     }
 
     /// Completes the job: latches the VM status on failure, then signals the fence (after every
@@ -120,6 +148,9 @@
                 self.vm.status.record(error);
             }
         }
+        if result.is_ok() || !engine {
+            self.release_binding();
+        }
         self.fence.signal(result);
     }
 

drivers/gpu/drm/asahi/g15/mod.rs

@@ -1,3 +1,3 @@
 // SPDX-License-Identifier: GPL-2.0-only OR MIT
 
 //! G15 GPU support (T6030, M3 Pro: J514S, J516S; firmware RTKit 2419, the macOS 14.7-14.8 stub).
@@ -375,7 +375,7 @@
     pub(crate) recovery: recovery::Recovery,
     /// Render UMA/PB allocators, their spare set and parameter-buffer growth.
     pub(crate) pools: pools::Pools,
-    /// Live VMs and their exclusive TTBAT slots, and the client aliases of every user VM.
+    /// Live VMs and the client aliases of every user VM.
     pub(crate) vms: Arc<vm::Registry>,
 }
 

drivers/gpu/drm/asahi/g15/poll.rs

@@ -1,3 +1,3 @@
 // SPDX-License-Identifier: GPL-2.0-only OR MIT
 
 //! The poll work: drain, retire, growth, hang checks, publication; and the repoll timers.
@@ -117,7 +117,7 @@
                 if let Err(error) = firmware.publish(&mut done) {
                     self.fail_stop(error);
                 }
-                if !firmware.engine.flights.is_empty() {
+                if !firmware.engine.flights.is_empty() || firmware.engine.slot_wait {
                     self.rearm(firmware.engine.chan_wait);
                 }
             }
@@ -189,10 +189,7 @@
     }
 }
 
-/// Stamp cell `obs` reached its own target, bounded by the newest published target `tail`.
-pub(crate) fn stamp_reached(obs: u32, own: u32, tail: u32) -> bool {
-    obs != 0 && obs & 0xff == 0 && obs.wrapping_sub(own) <= tail.wrapping_sub(own)
-}
+pub(crate) use super::recovery::logic::stamp_reached;
 
 /// Cursor `c` of a ring of `m` entries passed end `h` of entries from `b`, within tail `t`.
 pub(crate) fn ring_passed(c: u32, b: u32, h: u32, t: u32, m: u32) -> bool {
@@ -218,7 +215,7 @@
             // Idle bookkeeping of the PB: when the idle began, and the reclaim once it is due.
             self.note_idle();
             self.reclaim_check();
-            if self.recovery_idle() {
+            if self.recovery_idle() && self.engine.pending.is_empty() {
                 return Ok(false);
             }
         }
@@ -243,7 +240,10 @@
         // at the start of a pass that finds the flights empty.
         self.note_idle();
         published?;
-        Ok(!self.engine.flights.is_empty() || self.recovery.needs_repoll() || self.engine.frozen)
+        Ok(!self.engine.flights.is_empty()
+            || self.engine.slot_wait
+            || self.recovery.needs_repoll()
+            || self.engine.frozen)
     }
 
     /// Drains the rings and retires finished flights, with up to four trailing 5 us polls
@@ -462,6 +462,10 @@
                 p.count.store(p.sent(), Ordering::Relaxed);
             }
         }
+        if result.is_ok() && p.next() == p.sent() {
+            // All published prefixes retired; another VM may run before this job resumes.
+            job.release_binding();
+        }
         if p.next() < p.sent() || p.sent() < p.count() {
             return; // more of it published or still to publish
         }
@@ -479,6 +483,7 @@
             match result {
                 Err(error) => complete(done, j2, Err(error)),
                 Ok(()) => {
+                    j2.release_binding();
                     s.jobs += 1;
                     s.job_cmds += p2.count() as u64;
                     s.job_max = s.job_max.max(p2.count() as u64);
@@ -498,23 +503,25 @@
             return;
         };
         let (g, r, e) = (&self.graph.compute, &self.graph.render, &self.engine);
-        let (mut c, mut ta, mut fs) = (0, 0, 0);
+        let pending = super::recovery::logic::pending_stamp;
+        let (mut c, mut ta, mut fs) = (None, None, None);
 
         if f.computes != 0 {
-            c = g.fw_stamp.r32(0).unwrap_or(0);
-            if stamp_reached(c, f.target, e.target) {
-                c = 0; // its computes are done: later work does not count
-            }
+            c = g
+                .fw_stamp
+                .r32(0)
+                .ok()
+                .and_then(|v| pending(v, f.target, e.target));
         }
         if f.renders != 0 {
-            ta = r.fw_stamp[0].r32(0).unwrap_or(0);
-            fs = r.fw_stamp[1].r32(0).unwrap_or(0);
-            if stamp_reached(ta, f.ta_target, e.ta_target) {
-                ta = 0;
-            }
-            if stamp_reached(fs, f.fs_target, e.fs_target) {
-                fs = 0;
-            }
+            ta = r.fw_stamp[0]
+                .r32(0)
+                .ok()
+                .and_then(|v| pending(v, f.ta_target, e.ta_target));
+            fs = r.fw_stamp[1]
+                .r32(0)
+                .ok()
+                .and_then(|v| pending(v, f.fs_target, e.fs_target));
         }
         let moved = e.progress_flight == f.published_ns && [c, ta, fs] != e.progress;
         let published = f.published_ns;
@@ -607,7 +614,7 @@
             self.graph.dev.as_ref(),
             "G15: {}: ctx={} prefixes={} n={} target={:#x} user={:#x} fw={:#x} done={} read={} write={} head={} ch={}/{} events={}/{} errors={} busy={} fault={:#x} {}ms\n",
             what,
-            f.map_or(0, |f| f.job.vm.slot),
+            f.map_or(0, |f| f.job.slot()),
             e.flights.len(),
             e.ordinal,
             e.target,