A device lane short of stack moves it to the heap and back, so non-tail recursion under ! no longer faults at 1023 frames (#1393, #1394)

bend2/comp.ts

@@ -1416,6 +1416,7 @@ function file_new(book: Bend.Book, js: boolean) {
     lend: new Set<string>(),
     segs: [] as Seg[],
     spins: [] as Seg[],
+    room: 0,
     spun: new Map<string, string>(),
     clos: new Set<string>(),
     tabs: new Map<string, number>(),
@@ -2168,7 +2169,8 @@ function emit_task(sc: Scope, fid: string, rem: number, words: string[],
     `task_node(e, ${seg_ref(sc, fid)}, ${cont}, ${idx}, ${rem})`, words);
 }
 
-function emit_jump(sc: Scope, args: string[], k: Name, bang = false): void {
+function emit_jump(sc: Scope, args: string[], k: Name, bang = false,
+  push = false): void {
   const fid = seg_fid(k);
   sc.seg.fork ||= bang;
   if (bang || sc.seg.def !== k) {
@@ -2178,6 +2180,9 @@ function emit_jump(sc: Scope, args: string[], k: Name, bang = false): void {
         emit_task(sc, fid, 0, args)});`));
   }
   args.forEach((a, i) => file_push(sc, `r${i} = ${a};`));
+  if (push) {
+    file_push(sc, `WL_ROOM(${seg_ref(sc, fid)});`);
+  }
   if (sc.seg.def !== k) {
     return file_push(sc, `WL_JMP(${seg_ref(sc, fid)});`);
   }
@@ -2676,15 +2681,14 @@ function emit_fork(sc: Scope, x: Of<"Let">, ers: HTerm[]): void {
       (pos.set(p, depth), depth += b.val.ws.length, b.val.ws));
     emit_chain(sc, [["seq", () => {
       const fr = [...ws, seg_ref(sc, seg_fid(kn))];
-      file_push(sc, `WL_ROOM(${fr.length});`);
       fr.forEach((w, j) => file_push(sc, `STK(${j}) = ${w};`));
       file_push(sc, `WL_PUSHN(${fr.length});`);
     }], ...fork ? [] : [["", () => {
       file_push(sc, `WL_CONT = term_tsk(${seg_fid(kn)}, ${
         emit_task(sc, seg_fid(kn), 1, ws)});`);
       file_push(sc, `WL_IDX = ${ws.length};`);
     }] as [string, () => void]]]);
-    emit_jump(sc, cargs, c.k!, !fork && c.b);
+    emit_jump(sc, cargs, c.k!, !fork && c.b, true);
     const last = i === calls.length - 1;
     const held = [...sc.uses].filter(([p]) => pos.has(p));
     const at = held.flatMap(([p, b]) => b.val.ws.map((_, j) =>
@@ -2696,6 +2700,7 @@ function emit_fork(sc: Scope, x: Of<"Let">, ers: HTerm[]): void {
       rest);
     bind_uses(sc, o.ps[i], rs, rest, ty_ann(x.v[i])!);
   });
+  FL.room = Math.max(FL.room, depth + calls.length);
   if (fork) {
     const live = [...sc.uses];
     emit_jump(sc, live.flatMap(([, b]) => b.val.ws), name);
@@ -3469,11 +3474,6 @@ using namespace metal;
     STK(wi) = e.mem[(A) + wi]; \
   } \
   WL_PUSHN(N);
-#define WL_ROOM(N) \
-  if (DEVICE && sp + (N) * CUBE >= e.mem + STAT_OFF + CUBE) { \
-    err_post(e.mem, ERR_DEEP); \
-    return 0; \
-  }
 
 // Types
 // =====
@@ -4430,7 +4430,30 @@ ${spins}
 // ====
 
 // A host self-jump is a tail call: as a loop, clang hoists constants badly.
-#if !DEVICE
+// A device call that leaves less room than the longest chain of frames one
+// def pushes (#1393) jumps to F with bit 16 set, which moves the lane's stack
+// to the heap, under an exit frame, before F runs. Metal has no goto.
+#if DEVICE
+#define WL_ROOM(F) \
+  if (sp >= e.mem + STAT_OFF - ${FL.room} * CUBE) WL_JMP((F) | 1u << 16)
+#define WL_SPILT(F) ((F) == FID_EXIT)
+
+// A lane's stack starts as far past STAK_OFF as its allocator past ALC_OFF.
+FAR DEV Term* lane_spill(Env e, DEV Term* sp) {
+  DEV Term* sb = e.alc + (STAK_OFF - ALC_OFF);
+  u32 sn = (sp - sb) / CUBE;
+  u64 wa = heap_alloc(e, cls_fit(STAK_LEN));
+  for (u32 i = 0; i < sn; i += 1) {
+    e.mem[wa + i] = sb[i * CUBE];
+  }
+  sb[0]        = term_tsk(FID_EXIT, wa);
+  sb[CUBE]     = sn;
+  sb[2 * CUBE] = FID_EXIT;
+  return sb + 3 * CUBE;
+}
+#else
+#define WL_ROOM(F)
+#define WL_SPILT(F) 0
 #undef  WL_SPIN
 #undef  WL_SPUN
 #undef  WL_AGAIN
@@ -4523,21 +4546,31 @@ ${segs}
     Term cont = STK(0);
     u32  idx  = (u32)STK(1);
     u32  wf   = (u32)term_aux(cont);
-    if (cont != TERM_HOLE && fid_resw(wf)) {
+    if (cont != TERM_HOLE && (WL_SPILT(wf) || fid_resw(wf))) {
       u64 wa = term_loc(cont);
       u32 wn = fid_arity(wf);
-      WL_FRAME(cont)
-      seq = (seq & 1) | fid_nofk(wf) << 1;
-      WL_ARGS(wa, wn - n)
-      heap_free(e, cls_fit(wn + 2), wa);
+      if (!WL_SPILT(wf)) {
+        WL_FRAME(cont)
+        seq = (seq & 1) | fid_nofk(wf) << 1;
+      }
+      WL_ARGS(wa, WL_SPILT(wf) ? idx : wn - n)
+      heap_free(e, cls_fit(WL_SPILT(wf) ? STAK_LEN : wn + 2), wa);
       WL_TAKE(rv)
+      if (WL_SPILT(wf)) {
+        WL_RETN(n);
+      }
       WL_DYN(wf);
     }
     return task_deliver(e.mem, cont, idx, rv, n);
   }}
 
 #if DEVICE
   default: {
+    if (fid >> 16) {
+      fid &= 0xFFFF;
+      sp = lane_spill(e, sp);
+      break;
+    }
     err_post(e.mem, ERR_FIDS);
     return 0;
   }

tests/compile/bang_lane_spill.bend

@@ -0,0 +1,111 @@
+# A bang's root and a fork kid run on a lane's STAK_LEN-word stack on the
+# device. A call that finds it full moves the lane's stack words to a heap
+# block under an exit frame, whose return moves them back, so non-tail
+# recursion under ! goes as deep on the device as on the host (#1393, #1394).
+# Before, 1023 List.append frames faulted, and so did a 365-fork spine beside
+# a 3-fork one. walk's first spill comes at its second fork call, where the
+# def still holds two words on the stack, and they must come back with it.
+# tall's fork holds six 40-word records across its first call and hands one
+# to each later call, so its frames add up to 449 words. A spill at the first
+# call comes back near the top of the stack, and the later calls push onto
+# it with no check in between, so the lane keeps room for the whole chain.
+# With room for one frame instead, the second tall! faulted on CUDA.
+import Base
+
+def gen(n: Nat, +k: U32, acc: List<&2, U32>) -> List<&2, U32>:
+  match n:
+    case 0n:
+      acc
+    case 1n+p:
+      gen(p, (k + 1 : U32), k <> acc)
+
+def mix(xs: List<&2, U32>, +acc: U32) -> U32:
+  match xs:
+    case Nil{}:
+      acc
+    case Con{h, t}:
+      mix(t, ((acc * 31 : U32) + h : U32))
+
+def deep(+n: Nat) -> U32:
+  mix(List.append(&2, U32, gen(n, 0, []), [7]), 0)
+
+def spine(n: Nat, +i: U32) -> U32:
+  match n:
+    case 0n:
+      0
+    case 1n+p:
+      a b = spine(p, (i + 1 : U32)) mix(List.append(&2, U32, gen(800n, i, []), gen(64n, i, [])), i)
+      (a + b : U32)
+
+type Q is Data:
+  Q{a: U32, b: U32, c: U32, d: U32, e: U32}
+
+def leaf(+k: U32) -> Q:
+  Q{k, (k * 3 : U32), (k * 5 : U32), (k * 7 : U32), (k + 9 : U32)}
+
+def sum(q: Q) -> U32:
+  match q:
+    case Q{a, b, c, d, e}:
+      (((((a + b : U32) * 3 : U32) + c : U32) + ((d + e : U32) * 5 : U32) : U32) * 7 : U32)
+
+def walk(n: Nat, +k: U32) -> U32:
+  match n:
+    case 0n:
+      k
+    case 1n+p:
+      q b = leaf(k) walk(p, (k + 1 : U32))
+      (((sum(q) + b : U32) * 31 : U32) + k : U32)
+
+type P is Data:
+  P{a: Q, b: Q, c: Q, d: Q, e: Q, f: Q, g: Q, h: Q}
+
+def wide(+k: U32) -> P:
+  P{leaf(k), leaf((k + 1 : U32)), leaf((k + 2 : U32)), leaf((k + 3 : U32)),
+    leaf((k + 4 : U32)), leaf((k + 5 : U32)), leaf((k + 6 : U32)),
+    leaf((k + 7 : U32))}
+
+def flip(p: P, +k: U32) -> P:
+  match p:
+    case P{a, b, c, d, e, f, g, h}:
+      P{h, g, f, e, d, c, b, leaf((sum(a) + k : U32))}
+
+def fold(p: P) -> U32:
+  match p:
+    case P{a, b, c, d, e, f, g, h}:
+      (((sum(a) + sum(b) : U32) + (sum(c) + sum(d) : U32) : U32) + ((sum(e) + sum(f) : U32) + (sum(g) + sum(h) : U32) : U32) : U32)
+
+def tall(n: Nat, +k: U32) -> U32:
+  match n:
+    case 0n:
+      k
+    case 1n+p:
+      u = wide(k)
+      v = wide((k + 1 : U32))
+      w = wide((k + 2 : U32))
+      x = wide((k + 3 : U32))
+      y = wide((k + 4 : U32))
+      z = wide((k + 5 : U32))
+      a b c d e f g = tall(p, (k + 1 : U32)) flip(u, k) flip(v, k) flip(w, k) flip(x, k) flip(y, k) flip(z, k)
+      ((a * 31 : U32) + ((fold(b) + fold(c) : U32) + ((fold(d) + fold(e) : U32) + (fold(f) + fold(g) : U32) : U32) : U32) : U32)
+
+def two(+n: Nat) -> U32:
+  a b = spine(n, 0) spine(3n, 0)
+  (a + b : U32)
+
+def pair(+n: Nat) -> U32:
+  a b = walk(n, 1) walk(3n, 0)
+  (a + b : U32)
+
+def main() -> IO(Unit):
+  do IO<Unit>:
+    IO.print(U32.show(deep!(5000n)))
+    IO.print(U32.show(two!(365n)))
+    IO.print(U32.show(pair!(600n)))
+    IO.print(U32.show(tall!(40n, 0)))
+    IO.print(U32.show(tall!(40n, 1)))
+
+#|1543298507
+#|2063974529
+#|1878402805
+#|2730820976
+#|2131216241

tests/io/stack_fault_trap.bend

@@ -19,9 +19,8 @@
 # the C wall's own error 9 line, thrown as the one plain string
 # both printers spell verbatim - the interpreter's catch on stdout,
 # io_exit's on stderr - exit 1 either way, the compiled-JS deep-cut
-# limit now loud AND numbered. The device half of the law (WL_ROOM posting
-# ERR_DEEP on the strided device stacks) rides on no host drive, so
-# a csrc pin holds it verbatim beside the four host trap sites.
+# limit now loud AND numbered. A device lane's stack never faults: a full
+# one moves to the heap (tests/compile/bang_lane_spill.bend).
 # The print BEFORE the deep call pins the trap's stdout flush: stdout is
 # buffered when it goes to a pipe, so a trap that skips the flush loses
 # what the program already printed.