DLM rung-2: real cross-node $ENQ GRANT (vms-e8f1 / vms-17c, H4)

.github/workflows/ci.yml

@@ -377,6 +377,15 @@ jobs:
               - 'tests/qemu/Dockerfile.dlm-harness-h3'
               - 'tests/qemu/init_dlm_h3.sh'
               - 'tests/qemu/run_dlm_harness_h3.sh'
+              # H4 (rd vms-e8f1 / vms-17c, THE CROWN): after the join, node A
+              # issues a cross-node $ENQ for a resource mastered on node B; node
+              # B DISPATCHES it to its REAL executive, which GRANTS it (rung-2
+              # foundation grant, vms_lock_dlm_xnode_dispatch). Proves the GRANT
+              # (SS$_NORMAL, the 2296->1 flip) AND that B genuinely holds the lock
+              # for A's CSID (SCSD-I-DLMHELD held_for_csid).
+              - 'tests/qemu/Dockerfile.dlm-harness-h4'
+              - 'tests/qemu/init_dlm_h4.sh'
+              - 'tests/qemu/run_dlm_harness_h4.sh'
               - 'tests/cluster/two-ovmx/mk_sysgen_scratch.py'
               - 'tests/cluster/two-ovmx/verdict.sh'
 
@@ -2456,6 +2465,84 @@ jobs:
           path: h3out/
           if-no-files-found: ignore
 
+  # -----------------------------------------------------------------------
+  # DLM Harness H4 (rd vms-e8f1 / vms-17c) -- THE CROWN: the real cross-node GRANT.
+  # tests/qemu/Dockerfile.dlm-harness-h4 builds the SAME real-executive ingredients
+  # as H3 (vms.ko -- now with the rung-2 grant in vms_lock_dlm_xnode_dispatch --
+  # SCSD.EXE static musl + sca_l2probe + the two OVMXA/OVMXB SYSGEN stores) and
+  # boots the SAME TWO QEMU nodes, each with a real /dev/vms, on one shared L2 via a
+  # QEMU `socket` (mcast) netdev (NO host bridge, NO privilege). It runs the SAME
+  # join sequencer so both nodes reach VMS$VAXcluster membership, then arms node A's
+  # cross-node $ENQ (OVMX_DLM_ENQ=RESONE) for a resource mastered on node B. Node B
+  # DISPATCHES it to its REAL executive, which GRANTS it. H4 PASS iff (a) B's
+  # dispatch returns 0x00000001 (SS$_NORMAL -- GRANTED; the 2296->1 flip), NOT 2296
+  # and NOT 2680, AND (b) B's OWN resource DB (GET_RESMASTER, printed as
+  # SCSD-I-DLMHELD) shows the lock GENUINELY HELD for A's CSID (found=1,
+  # is_local_master=1, n_granted>=1, held_for_csid=A). INV-6: the verdict READS B's
+  # grant + held-lock state from B's own log; a fabricated grant cannot pass.
+  # -----------------------------------------------------------------------
+  dlm-harness-h4:
+    name: DLM Harness H4 (cross-node $ENQ GRANTED -> SS$_NORMAL, B holds lock for A, QEMU)
+    needs: changes
+    if: ${{ github.event_name != 'pull_request' || needs.changes.outputs.dlm_harness == 'true' }}
+    runs-on: ubuntu-latest
+    # vms.ko + SCSD.EXE build + TWO QEMU boots, each running the join sequencer +
+    # the one-shot $ENQ for ~90s. The inner H4_WALL_TIMEOUT (600s) is the real
+    # budget; this is the OUTER bound so a genuine hang surfaces.
+    timeout-minutes: 50
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Build DLM harness H4 image (gha layer cache)
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: tests/qemu/Dockerfile.dlm-harness-h4
+          push: false
+          load: true
+          tags: ovmx-dlm-h4:latest
+          cache-from: type=gha,scope=dlm-harness-h4
+          cache-to: type=gha,mode=max,scope=dlm-harness-h4
+
+      - name: Boot TWO QEMU nodes, drive node A's cross-node $ENQ and grant it on B
+        run: |
+          set +e
+          # GitHub Linux runners expose /dev/kvm; pass it through so the two
+          # guests boot under KVM (much faster than TCG). Fall back cleanly if
+          # a runner lacks it -- the harness auto-detects and uses TCG.
+          KVM=""
+          [ -e /dev/kvm ] && KVM="--device /dev/kvm"
+          mkdir -p "$PWD/h4out"
+          RAW=$(docker run --rm $KVM -v "$PWD/h4out:/out" ovmx-dlm-h4:latest 2>&1)
+          RC=$?
+          set -e
+          # QEMU serial emits CRLF; strip \r so the line-anchored match holds.
+          OUTPUT=$(printf '%s\n' "$RAW" | tr -d '\r')
+          echo "$OUTPUT"
+          if [ "$RC" -ne 0 ]; then
+            echo "FAIL: DLM harness H4 exited nonzero"
+            exit 1
+          fi
+          if printf '%s\n' "$OUTPUT" | grep -qE '^  DLM HARNESS H4 PASSED'; then
+            echo "PASS: node A's cross-node \$ENQ was GRANTED by node B's real executive (rc=SS\$_NORMAL), and B holds the lock for A's CSID."
+          else
+            echo "FAIL: H4 verdict 'DLM HARNESS H4 PASSED' not found."
+            printf '%s\n' "$OUTPUT" | grep -E 'verdict inputs|B_DISPATCH_STATUS|B_HELD|held_for_csid|highest join rung|VAXCLMEMBER|DLMENQ|DLMRX|DLMHELD|H4-NODE|MISS:|FAIL:' || true
+            exit 1
+          fi
+
+      - name: Upload H4 pcap + node logs
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: dlm-harness-h4-artifacts
+          path: h4out/
+          if-no-files-found: ignore
+
   # -----------------------------------------------------------------------
   # Job 3b: Kernel Executive (vms-e4d, epic vms-6b8 PHASE 0 HARD BARRIER)
   #

src/kernel-core/vms_lock.c

@@ -902,6 +902,17 @@ static long vms_enq_core(struct vms_proc *proc, struct vms_enq_args *io)
     lock->waiting = 0;
     exec_cv_init(&lock->wait_wq);
     lock->grant_state = 0;
+    /*
+     * The cluster identity this lock is held FOR. 0 for an ordinary $ENQ (a
+     * local process on this node owns the lock -- args.owner_csid is 0, the
+     * memset default every userspace caller leaves in place). Non-zero only on
+     * the cross-node DLM grant path (vms_lock_dlm_xnode_dispatch, vms-e8f1),
+     * which sets owner_csid to the REMOTE requester's CSID so the master's lock
+     * record carries the identity it is held on behalf of. Read back through
+     * GET_RESMASTER.remote_holder_csid -- a genuine held-lock proof, not a
+     * fabricated status (INV-6).
+     */
+    lock->req_csid = args.owner_csid;
 
     /*
      * Record the access mode $ENQ was issued from, so image rundown can tell
@@ -1381,8 +1392,19 @@ long vms_ioctl_get_resmaster(struct vms_proc *proc, unsigned long arg)
         args.master_csid = res->master_csid;
         args.is_local_master =
             (res->master_csid != 0 && res->master_csid == vms_local_csid) ? 1 : 0;
-        exec_list_for_each_entry(granted, &res->granted, res_granted)
+        exec_list_for_each_entry(granted, &res->granted, res_granted) {
             n++;
+            /*
+             * Report the identity a REMOTE-held grant is held for (the first
+             * one, if several). A local grant carries req_csid==0 and is
+             * skipped, so remote_holder_csid stays 0 when every holder is
+             * local. This is the held-lock proof the cross-node grant
+             * (vms-e8f1) is verified by: after a peer's $ENQ, the master's DB
+             * genuinely shows a lock held FOR that peer's CSID.
+             */
+            if (granted->req_csid != 0 && args.remote_holder_csid == 0)
+                args.remote_holder_csid = granted->req_csid;
+        }
         args.n_granted = n;
         exec_unlock(&res->lock);
     }
@@ -1405,45 +1427,89 @@ long vms_ioctl_get_resmaster(struct vms_proc *proc, unsigned long arg)
  * HERE -- the point at which the kernel lock manager would act on a peer's
  * behalf, as the resource's master or directory node.
  *
- * RUNG 1 IS THE TRANSPORT ONLY. This handler exists, the message reaches it
- * DECODED, and it returns SS$_UNSUPPORTED. It does NOT grant, queue, dequeue, or
- * deliver a blocking AST. INV-6: no fabricated cross-node grant -- a cross-node
- * lock op honestly fails with SS$_UNSUPPORTED, exactly as dlm_resolve_master()
- * already does for the SEND side, rather than fabricating a remote answer.
+ * RUNG 2 -- THE FOUNDATION GRANT (vms-e8f1). VMS_DLM_OP_ENQ now acts on the real
+ * single-node lock manager on the mastering node: the decoded cross-node $ENQ is
+ * run through vms_enq_core() with the requesting proc bound to the REMOTE
+ * requester's cluster identity (owner_csid = req->req_csid), so a COMPATIBLE
+ * request is GRANTED (SS$_NORMAL) and the master's resource DB genuinely holds a
+ * lock record FOR the peer's CSID -- verifiable via GET_RESMASTER
+ * (remote_holder_csid). Membership here is a stub-of-one, so this node is the
+ * directory + master for the name and vms_enq_core grants locally; that is the
+ * mastering node's real grant, not a fabrication.
  *
- * RUNG 2 (vms-7fa) wires each op into the real single-node lock manager on the
- * mastering node: VMS_DLM_OP_ENQ -> vms_enq_core() with proc bound to the
- * remote requester's cluster identity, DEQ -> the release path, GRANT/BLKAST ->
- * completing/notifying the ORIGINATING node's pending request. That is where
- * this switch stops returning SS$_UNSUPPORTED; nothing above this line changes.
+ * SCOPE FENCE (INV-6 -- everything past the foundation FAILS HONESTLY):
+ *   - The ENQ runs with LCK_M_NOQUEUE and without LCK_M_SYNC, so an INCOMPATIBLE
+ *     cross-node request declines with SS$_NOTQUEUED rather than building a
+ *     cross-node wait queue. Cross-node CONTENTION/blocking is a later rung
+ *     (vms-904c) -- this handler never queues or blocks the delivery thread.
+ *   - LVB replication (vms-d81) and cross-node blocking-AST delivery are later
+ *     rungs, so no LCK_M_VALBLK / AST is carried into the grant here.
+ *   - VMS_DLM_OP_DEQ (release), GRANT, and BLKAST are NOT the foundation: they
+ *     still return SS$_UNSUPPORTED -- an honest decline, never a faked receipt.
  *
  * The request is VALIDATED so a malformed message is rejected (SS$_BADPARAM)
  * rather than silently dropped -- the same discipline vms_enq_core applies.
  */
 uint32_t vms_lock_dlm_xnode_dispatch(struct vms_proc *proc,
                                      struct vms_dlm_xnode_args *req)
 {
-    (void)proc;
-
     if (!req)
         return SS__BADPARAM;
+    if (!proc)
+        return SS__BADPARAM;
     if (req->lkmode > LCK_K_EXMODE)
         return SS__BADPARAM;
     req->resnam[sizeof(req->resnam) - 1] = '\0';
 
     switch (req->op) {
-    case VMS_DLM_OP_ENQ:
+    case VMS_DLM_OP_ENQ: {
+        struct vms_enq_args a;
+
+        /* A request that names a resource must actually name one. */
+        if (req->resnam[0] == '\0')
+            return SS__BADPARAM;
+
+        /*
+         * THE FOUNDATION GRANT. Marshal the decoded cross-node $ENQ into the
+         * single-node lock manager on this (the mastering) node and grant it.
+         * The lock is held FOR the remote requester's cluster identity
+         * (owner_csid), NOT the local delivery daemon -- vms_enq_core stamps
+         * lock->req_csid = a.owner_csid, so GET_RESMASTER later reports the
+         * grant as remote-held for the peer's CSID.
+         *
+         * LCK_M_NOQUEUE fences out cross-node contention (a later rung): a
+         * compatible request grants immediately (SS$_NORMAL); an incompatible
+         * one declines with SS$_NOTQUEUED -- honest, no wait queue. No
+         * LCK_M_SYNC: this must never block the daemon thread waiting on a
+         * remote-driven grant. No VALBLK/AST: LVB replication and blocking-AST
+         * delivery are later rungs.
+         */
+        memset(&a, 0, sizeof(a));
+        a.lkmode = req->lkmode;
+        a.flags = LCK_M_NOQUEUE;
+        memcpy(a.resnam, req->resnam, sizeof(a.resnam));
+        a.resnam[sizeof(a.resnam) - 1] = '\0';
+        a.owner_csid = req->req_csid;   /* held FOR the remote requester */
+
+        vms_enq_core(proc, &a);
+
+        /* Hand the master's lock id back to the requester (the GRANT reply's
+         * master_lkid). On an incompatible/declined request a.lkid is 0. */
+        req->master_lkid = a.lkid;
+        req->master_csid = vms_local_csid; /* this node mastered the grant */
+        return a.status;
+    }
     case VMS_DLM_OP_DEQ:
         /* A request that names a resource must actually name one. */
         if (req->resnam[0] == '\0')
             return SS__BADPARAM;
-        /* Rung 2 acts on the decoded request here (vms_enq_core on the master,
-         * the release path for DEQ). Rung 1 delivers it and honestly declines. */
+        /* Cross-node release is NOT the foundation grant -- a later rung wires
+         * the master's release path. Decline honestly (never a faked receipt). */
         return SS__UNSUPPORTED;
     case VMS_DLM_OP_GRANT:
     case VMS_DLM_OP_BLKAST:
-        /* Responses carry no resource name. Rung 2 completes/notifies the
-         * originating node's pending request here; rung 1 declines. */
+        /* Responses carry no resource name. Completing/notifying the
+         * originating node's pending request is a later rung; decline honestly. */
         return SS__UNSUPPORTED;
     default:
         return SS__BADPARAM;

src/kernel-netbsd/vms_internal.h

@@ -389,6 +389,11 @@ struct vms_lock_entry {
 	                                     *            SS__DEADLOCK=cycle detected */
 	uint8_t             acmode;         /* access mode $ENQ was issued from (0-3),
 	                                     * for image rundown -- NOT a lock mode */
+	uint32_t            req_csid;       /* cluster CSID this lock is held FOR;
+	                                     * 0 = a local process owns it, non-zero =
+	                                     * a cross-node grant on behalf of a remote
+	                                     * node (vms-e8f1). Set from
+	                                     * vms_enq_args.owner_csid at creation. */
 };
 
 /* Lock resource -- a named resource in the lock database. */

src/kernel-netbsd/vms_lock_nb.h

@@ -129,7 +129,10 @@ struct vms_enq_args {
 	uint32_t lk_status;         /* return: lock status (granted mode in LKSB) */
 	uint8_t  valblk[LCK_VALBLK_SIZE]; /* in/out: lock value block */
 	uint32_t status;            /* return: SS$_ status */
-	uint32_t pad;
+	uint32_t owner_csid;        /* in: cluster CSID that OWNS this lock; 0 = the
+	                             * local node. Cross-node DLM dispatch sets it to
+	                             * the remote requester's CSID (vms-e8f1). Was a
+	                             * reserved pad -- same size, no ABI change. */
 };
 
 struct vms_deq_args {
@@ -159,7 +162,9 @@ struct vms_resmaster_args {
 	uint32_t is_local_master;   /* return: 1 if mastered by this node */
 	uint32_t n_granted;         /* return: granted locks on the resource */
 	uint32_t status;            /* return: SS$_ status */
-	uint32_t pad;
+	uint32_t remote_holder_csid;/* return: CSID a remote-held granted lock is held
+	                             * FOR; 0 if all grants are local (vms-e8f1). Was a
+	                             * reserved pad -- same size, no ABI change. */
 };
 
 /*

src/kernel/vms_internal.h

@@ -451,6 +451,15 @@ struct vms_lock_entry {
                                          * USER mode; inner-mode locks are
                                          * process-permanent. See
                                          * vms_proc_rundown_locks(). */
+    uint32_t            req_csid;       /* cluster CSID this lock is held FOR.
+                                         * 0 = a local process on THIS node owns
+                                         * it (the common case). Non-zero = a
+                                         * cross-node grant: the master holds this
+                                         * lock on behalf of a REMOTE node whose
+                                         * CSID this is (DLM epic vms-7fa rung 2,
+                                         * vms-e8f1). Set from vms_enq_args
+                                         * .owner_csid at creation; surfaced by
+                                         * GET_RESMASTER.remote_holder_csid. */
 };
 
 /* Lock resource (named resource in the lock database) */

src/kernel/vms_ioctl.h

@@ -407,7 +407,15 @@ struct vms_enq_args {
     uint32_t lk_status;         /* return: lock status (granted mode in LKSB) */
     uint8_t  valblk[LCK_VALBLK_SIZE]; /* lock value block */
     uint32_t status;            /* return: SS$_ status */
-    uint32_t pad;
+    uint32_t owner_csid;        /* in: cluster CSID that OWNS this lock; 0 = the
+                                 * local node (the calling process's own node).
+                                 * A userspace $ENQ leaves this 0 (local hold);
+                                 * the cross-node DLM dispatch
+                                 * (vms_lock_dlm_xnode_dispatch) sets it to the
+                                 * REMOTE requester's CSID so the master's lock
+                                 * record is stamped with the identity it is held
+                                 * FOR (DLM epic vms-7fa rung 2, vms-e8f1). Was a
+                                 * reserved pad; same size, no ABI change. */
 };
 
 struct vms_deq_args {
@@ -452,7 +460,13 @@ struct vms_resmaster_args {
     uint32_t is_local_master;   /* return: 1 if mastered by this node */
     uint32_t n_granted;         /* return: granted locks on the resource */
     uint32_t status;            /* return: SS$_ status */
-    uint32_t pad;
+    uint32_t remote_holder_csid;/* return: the CSID a REMOTE-held granted lock on
+                                 * this resource is held FOR (the req_csid stamped
+                                 * by the cross-node DLM grant); 0 if every grant
+                                 * is local. Lets a test PROVE the master genuinely
+                                 * holds a lock for a peer's cluster identity, not
+                                 * just that n_granted rose (DLM epic vms-7fa rung
+                                 * 2, vms-e8f1). Was a reserved pad; same size. */
 };
 
 #define VMS_IOCTL_ENQ       _IOWR(VMS_IOC_MAGIC, 0x30, struct vms_enq_args)

src/libvmssys/vms_kif.c

@@ -720,7 +720,8 @@ uint32_t vms_kif_getlki(uint32_t lkid, uint32_t *granted_mode,
 uint32_t vms_kif_get_resmaster(const char *resnam, uint32_t *found,
                                uint32_t *local_csid, uint32_t *dir_csid,
                                uint32_t *master_csid, uint32_t *is_local_master,
-                               uint32_t *n_granted)
+                               uint32_t *n_granted,
+                               uint32_t *remote_holder_csid)
 {
     struct vms_resmaster_args args;
 
@@ -739,6 +740,7 @@ uint32_t vms_kif_get_resmaster(const char *resnam, uint32_t *found,
     if (master_csid) *master_csid = args.master_csid;
     if (is_local_master) *is_local_master = args.is_local_master;
     if (n_granted) *n_granted = args.n_granted;
+    if (remote_holder_csid) *remote_holder_csid = args.remote_holder_csid;
 
     return args.status;
 }

src/libvmssys/vms_kif.h

@@ -288,7 +288,8 @@ uint32_t vms_kif_getlki(uint32_t lkid, uint32_t *granted_mode,
 uint32_t vms_kif_get_resmaster(const char *resnam, uint32_t *found,
                                uint32_t *local_csid, uint32_t *dir_csid,
                                uint32_t *master_csid, uint32_t *is_local_master,
-                               uint32_t *n_granted);
+                               uint32_t *n_granted,
+                               uint32_t *remote_holder_csid);
 
 /* Dispatch a decoded cross-node DLM request to the kernel lock manager's
  * cross-node handler (vms-94c, DLM epic vms-7fa rung 1).