From 84eb36a805b86773a2ad5af0817c629b5dbb2008 Mon Sep 17 00:00:00 2001 From: alice Date: Mon, 10 Aug 2026 14:13:25 +0000 Subject: [PATCH 1/2] release: cut-release.sh builds a versioned, checksummed release bundle from a clean tree, byte-reproducibly (vms-d73) Foundation of the Release Engineering pillar (epic vms-a84): replaces the hand-rolled per-cut process (build, copy artifacts, hand-write checksums) with tools/cut-release.sh, which git-archives HEAD to a clean /tmp tree (never builds into the repo), drives the existing containerized distro/Dockerfile.bootable build via `docker buildx build --output=local`, and writes dist/release-/ with vmlinuz, initramfs-ovmx-slim.cpio.gz, ovmx-distrib.img, ovmx-os.kit, SHA256SUMS, and an OVMX-defined (Rule 8) release-manifest.json. Version is single-sourced from the archived src/libvms/include/ovmx_identity.h -- no literal in the script. Byte-reproducibility required normalizing every wall-clock the build stamps into a shipped artifact: SOURCE_DATE_EPOCH now threads through Dockerfile.bootable (new build-arg, default 0) into cpio/gzip packing (mtime-normalized + sorted file lists + gzip -n) and into tools/ovmx_kit_pack.c / tools/vmsfs_master.c, which previously stamped time(NULL) into the OS kit header and the mastered VMSFS image's file timestamps. .github/workflows/ci.yml adds cut-release-reproducible: cuts the same commit twice with --no-cache, diffs all four artifacts + SHA256SUMS byte-for-byte, recomputes checksums against the emitted manifest, and reuses tests/qemu/test_distrib_boot.sh (unmodified) to boot the freshly-cut bundle to a real SYSTEM login. Gated behind a new `release` path filter. CMakeLists.txt adds an optional `make release` convenience target wrapping the script. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/ci.yml | 120 ++++++++++++++++++ CMakeLists.txt | 16 +++ distro/Dockerfile.bootable | 46 ++++++- tools/cut-release.sh | 252 +++++++++++++++++++++++++++++++++++++ tools/ovmx_kit_pack.c | 24 +++- tools/vmsfs_master.c | 26 +++- 6 files changed, 476 insertions(+), 8 deletions(-) create mode 100755 tools/cut-release.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e1304298c..75711157c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -51,6 +51,7 @@ jobs: toolchain: ${{ steps.decide.outputs.toolchain }} kernel: ${{ steps.decide.outputs.kernel }} boot: ${{ steps.decide.outputs.boot }} + release: ${{ steps.decide.outputs.release }} steps: - name: Checkout uses: actions/checkout@v4 @@ -99,6 +100,15 @@ jobs: - 'tests/qemu/test_executive_integral.sh' - 'tests/qemu/test_distrib_boot.sh' - 'tests/uat/**' + release: + - '.github/workflows/**' + - 'tools/cut-release.sh' + - 'distro/Dockerfile.bootable' + - 'tools/ovmx_kit_pack.c' + - 'tools/vmsfs_master.c' + - 'src/libvms/include/ovmx_identity.h' + - 'src/libvms/include/ovmx_kit_format.h' + - 'src/kernel/vmsfs/vmsfs_ondisk.h' - name: Decide (force every output true on non-PR events) id: decide @@ -109,6 +119,7 @@ jobs: echo "toolchain=${{ steps.filter.outputs.toolchain }}" echo "kernel=${{ steps.filter.outputs.kernel }}" echo "boot=${{ steps.filter.outputs.boot }}" + echo "release=${{ steps.filter.outputs.release }}" } >> "$GITHUB_OUTPUT" else { @@ -116,6 +127,7 @@ jobs: echo "toolchain=true" echo "kernel=true" echo "boot=true" + echo "release=true" } >> "$GITHUB_OUTPUT" fi # ----------------------------------------------------------------------- @@ -349,6 +361,114 @@ jobs: ovmx-boot:latest \ /test.sh + # ----------------------------------------------------------------------- + # cut-release-reproducible (vms-d73, epic vms-a84 RELEASE ENGINEERING) + # + # tools/cut-release.sh is the FIRST piece of release-engineering machinery: + # a repeatable "cut a release" command replacing the previous hand-rolled + # per-cut process (build the image, copy artifacts out, hand-write + # checksums). This job is that command's ground-source proof: + # + # 1. Run cut-release.sh TWICE, from the SAME commit, each with + # --no-cache so the two builds share nothing (a cache hit would prove + # cache reuse, not reproducibility) — and diff the four release + # artifacts (vmlinuz, initramfs-ovmx-slim.cpio.gz, ovmx-distrib.img, + # ovmx-os.kit) byte-for-byte. + # 2. Recompute SHA256SUMS against the actual bundle bytes and compare + # against what the script wrote, so a bug in the script's own + # checksum step cannot self-certify. + # 3. Reuse the EXISTING boot-smoke harness (tests/qemu/test_distrib_boot.sh + # — the same one persistent-boot above runs against a normal build) to + # boot the freshly-cut bundle's own vmlinuz/initramfs/distrib.img to a + # real SYSTEM login prompt, rather than reinventing that proof. + # + # Two full containerized builds (~25-30 min cold each per boot.sh) make this + # the heaviest job in the workflow; it is gated behind the `release` path + # filter so ordinary PRs that do not touch release-engineering surfaces + # don't pay for it, and still runs in full on push/merge_group/schedule. + # ----------------------------------------------------------------------- + cut-release-reproducible: + name: cut-release is byte-reproducible + boots (vms-d73) + needs: changes + if: ${{ github.event_name != 'pull_request' || needs.changes.outputs.release == 'true' }} + runs-on: ubuntu-latest + timeout-minutes: 150 + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + # First independent cut. --no-cache on both cuts is the point: this + # proves the BUILD is reproducible, not that BuildKit's cache is. + - name: Cut release A (clean tree, no cache) + run: | + tools/cut-release.sh --no-cache --out-dir "${{ github.workspace }}/dist/release-a" + + - name: Cut release B (independent clean tree, no cache) + run: | + tools/cut-release.sh --no-cache --out-dir "${{ github.workspace }}/dist/release-b" + + # The manifest's own "cut_at_utc" field legitimately differs between + # the two invocations (it is wall-clock metadata ABOUT the cut, not a + # build input) -- the reproducibility claim is scoped to the four real + # release artifacts and their checksums, which is exactly what this + # step compares. SHA256SUMS also carries no such field, so it is + # compared byte-for-byte too, not just semantically. + - name: Assert the two cuts are byte-identical + run: | + set -euo pipefail + A="dist/release-a" + B="dist/release-b" + FAIL=0 + for f in vmlinuz initramfs-ovmx-slim.cpio.gz ovmx-distrib.img ovmx-os.kit ovmx-os.kit.manifest.txt SHA256SUMS; do + if cmp -s "$A/$f" "$B/$f"; then + echo "OK: $f is byte-identical across both cuts" + else + echo "FAIL: $f DIFFERS between the two independent cuts" + cmp "$A/$f" "$B/$f" || true + FAIL=1 + fi + done + [ "$FAIL" -eq 0 ] || { echo "Reproducibility check FAILED"; exit 1; } + echo "All release artifacts + SHA256SUMS are byte-identical across two independent --no-cache cuts." + + # Defense in depth against a bug in the script's own checksum step: + # recompute sha256 directly against the shipped bytes and compare + # against what release-manifest.json / SHA256SUMS claim. + - name: Recompute checksums and compare against the emitted manifest + run: | + set -euo pipefail + cd dist/release-a + sha256sum -c SHA256SUMS + for f in vmlinuz initramfs-ovmx-slim.cpio.gz ovmx-distrib.img ovmx-os.kit; do + REAL=$(sha256sum "$f" | awk '{print $1}') + # Each artifact is emitted as a single JSON line (cut-release.sh + # printf's one object per line, no embedded newlines) -- a plain + # grep+sed extraction is exact and needs no JSON library. + CLAIMED=$(grep "\"component\": \"$f\"" release-manifest.json | sed -n 's/.*"sha256": "\([a-f0-9]*\)".*/\1/p') + [ -n "$CLAIMED" ] || { echo "FAIL: $f not found in release-manifest.json"; exit 1; } + [ "$REAL" = "$CLAIMED" ] || { echo "FAIL: $f real=$REAL manifest=$CLAIMED"; exit 1; } + echo "OK: release-manifest.json sha256 for $f matches the actual artifact bytes" + done + + # THE BOOT PROOF. Reuses tests/qemu/test_distrib_boot.sh verbatim + # against the CUT bundle's own artifacts (bind-mounted at the fixed + # /boot/ paths the script expects) instead of reinventing it — the same + # QEMU boot-to-login assertions persistent-boot already runs against a + # normal (non-cut) build. + - name: Boot the freshly-cut bundle to a login prompt + run: | + docker run --rm \ + -v ${{ github.workspace }}/dist/release-a/vmlinuz:/boot/vmlinuz:ro \ + -v ${{ github.workspace }}/dist/release-a/initramfs-ovmx-slim.cpio.gz:/boot/initramfs-ovmx-slim.cpio.gz:ro \ + -v ${{ github.workspace }}/dist/release-a/ovmx-distrib.img:/boot/ovmx-distrib.img:ro \ + -v ${{ github.workspace }}/tests/qemu/test_distrib_boot.sh:/test.sh:ro \ + ubuntu:24.04 \ + bash -c "apt-get update -qq && apt-get install -y --no-install-recommends qemu-system-x86 qemu-system-arm qemu-efi-aarch64 cpio >/dev/null 2>&1 && bash /test.sh" + # ----------------------------------------------------------------------- # Job 2b: PARTS 0.2 Demo E2E (vms-1c8, epic vms-2579/vms-5dd) # diff --git a/CMakeLists.txt b/CMakeLists.txt index 825c72ed1..45028376b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -634,3 +634,19 @@ install(DIRECTORY distro/rootfs/vms/ DESTINATION /vms DIRECTORY_PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE) + +# `make release` — convenience wrapper for tools/cut-release.sh (vms-d73, +# epic vms-a84 RELEASE ENGINEERING). The script is the actual machinery +# (git-archives HEAD to a clean tree, runs the containerized build, emits +# dist/release-/ with SHA256SUMS + release-manifest.json) -- this +# target only makes it reachable as `make release` / `ninja release` from an +# already-configured build dir. Not part of ALL; run it explicitly. For +# options (--ref, --out-dir, --no-cache, ...) call tools/cut-release.sh +# directly -- see its --help. +add_custom_target(release + COMMAND ${CMAKE_SOURCE_DIR}/tools/cut-release.sh + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} + COMMENT "Cutting an OVMX release bundle via tools/cut-release.sh" + VERBATIM + USES_TERMINAL +) diff --git a/distro/Dockerfile.bootable b/distro/Dockerfile.bootable index a5078da9f..0f8a529d9 100644 --- a/distro/Dockerfile.bootable +++ b/distro/Dockerfile.bootable @@ -109,6 +109,19 @@ FROM ubuntu:24.04 AS builder ENV DEBIAN_FRONTEND=noninteractive +# Reproducible-build clock (vms-d73, tools/cut-release.sh). Every wall-clock +# timestamp this stage bakes into a shipped artifact -- the OVMX kit header's +# kh_build_time (tools/ovmx_kit_pack.c) and the mastered VMSFS image's +# fh_created/fh_modified/fh_accessed fields (tools/vmsfs_master.c) -- reads +# this instead of time(NULL) when it is set, and every cpio/initramfs pack +# step below normalizes its input file timestamps to it. Default 0 (the Unix +# epoch) so ordinary builds (boot.sh, other CI jobs) that never pass the +# build-arg still produce a deterministic, harmless value; cut-release.sh +# passes the real HEAD commit time so a release's artifacts carry an honest, +# reproducible date instead of 1970. +ARG SOURCE_DATE_EPOCH=0 +ENV SOURCE_DATE_EPOCH=${SOURCE_DATE_EPOCH} + RUN apt-get update && apt-get install -y --no-install-recommends \ cmake gcc make musl-tools linux-libc-dev \ linux-headers-generic linux-image-generic \ @@ -301,8 +314,14 @@ RUN mkdir -p /initramfs-fat/dev /initramfs-fat/proc /initramfs-fat/sys \ echo "OK: shipped PARTS.EXE is VMS-native (EM_X86_64, zero DT_NEEDED/DT_HASH)" && \ [ -f "/initramfs-fat/vms/SYS0/SYSCOMMON/SYSUPD/PARTS_SETUP.COM" ] || { echo "FAIL: PARTS_SETUP.COM missing from SYS\$UPDATE: (SYSUPD)"; exit 1; } && \ echo "OK: PARTS_SETUP.COM present under SYS\$UPDATE: (SYSUPD)" && \ - # Pack the fat initramfs - cd /initramfs-fat && find . | cpio -o -H newc 2>/dev/null | gzip > /boot/initramfs-ovmx.cpio.gz && \ + # Pack the fat initramfs. Reproducibility (vms-d73): normalize every file's + # mtime to SOURCE_DATE_EPOCH (cp resets mtime to build wall-clock, which + # the cpio "newc" format embeds per entry) and sort the cpio member order + # -- find(1)'s raw readdir order is not a byte-reproducibility guarantee + # even when the create sequence is identical -- and drop gzip's embedded + # mtime (-n) so the compressed stream's bytes depend only on content. + find /initramfs-fat -exec touch -h -d "@${SOURCE_DATE_EPOCH}" {} + && \ + cd /initramfs-fat && find . | sort | cpio -o -H newc 2>/dev/null | gzip -n > /boot/initramfs-ovmx.cpio.gz && \ echo "FAT initramfs: $(ls -lh /boot/initramfs-ovmx.cpio.gz | awk '{print $5}')" # -- OVMX OS kit (vms-0b6) ------------------------------------------------------ @@ -334,6 +353,12 @@ RUN mkdir -p /kit-stage/SYSEXE /kit-stage/SYSLIB /kit-stage/SYSMGR /kit-stage/SY echo "$LISTING" | grep -q "$name" || { echo "FAIL: ovmx-os.kit manifest missing $name"; exit 1; }; \ echo "OK: ovmx-os.kit manifest names $name"; \ done && \ + # Persist the listing as a build artifact (vms-d73, tools/cut-release.sh): + # the release script folds this into release-manifest.json and re-checks + # DCL.EXE/LOGINOUT.EXE/STARTUP.COM against the SHIPPED kit's own listing, + # not just this build-time variable. SOURCE_DATE_EPOCH normalization above + # makes the embedded "Built:" line (kh_build_time) reproducible too. + echo "$LISTING" > /boot/ovmx-os.kit.manifest.txt && \ # Round-trip the kit itself here too (not just in ctest): extract and # byte-compare every payload file against the staged tree that was # packed. A flag file (not a loop exit code) carries failure out of the @@ -448,8 +473,9 @@ RUN mkdir -p /initramfs-slim/dev /initramfs-slim/proc /initramfs-slim/sys \ # Minimal config — SYSUAF + manager files (STARTUP.EXE reads these at boot) cp distro/rootfs/vms/SYS0/SYSCOMMON/SYSEXE/SYSUAF.DAT /initramfs-slim/vms/SYS0/SYSCOMMON/SYSEXE/ && \ cp -r distro/rootfs/vms/SYS0/SYSCOMMON/SYSMGR/* /initramfs-slim/vms/SYS0/SYSCOMMON/SYSMGR/ && \ - # Pack the slim initramfs - cd /initramfs-slim && find . | cpio -o -H newc 2>/dev/null | gzip > /boot/initramfs-ovmx-slim.cpio.gz && \ + # Pack the slim initramfs (reproducibility normalization, see FAT step above) + find /initramfs-slim -exec touch -h -d "@${SOURCE_DATE_EPOCH}" {} + && \ + cd /initramfs-slim && find . | sort | cpio -o -H newc 2>/dev/null | gzip -n > /boot/initramfs-ovmx-slim.cpio.gz && \ echo "SLIM initramfs: $(ls -lh /boot/initramfs-ovmx-slim.cpio.gz | awk '{print $5}')" # -- NOEXEC initramfs (negative control) --------------------------------------- @@ -464,7 +490,8 @@ RUN mkdir -p /initramfs-slim/dev /initramfs-slim/proc /initramfs-slim/sys \ # precisely the state OVMX exists to make unreachable (CLAUDE.md Rule 9). RUN cp -a /initramfs-fat /initramfs-noexec && \ rm -f /initramfs-noexec/lib/modules/vms.ko && \ - cd /initramfs-noexec && find . | cpio -o -H newc 2>/dev/null | gzip > /boot/initramfs-ovmx-noexec.cpio.gz && \ + find /initramfs-noexec -exec touch -h -d "@${SOURCE_DATE_EPOCH}" {} + && \ + cd /initramfs-noexec && find . | sort | cpio -o -H newc 2>/dev/null | gzip -n > /boot/initramfs-ovmx-noexec.cpio.gz && \ echo "NOEXEC initramfs: $(ls -lh /boot/initramfs-ovmx-noexec.cpio.gz | awk '{print $5}')" # -- NODEV initramfs (negative control) ----------------------------------------- @@ -480,7 +507,8 @@ RUN cp -a /initramfs-fat /initramfs-noexec && \ # Test artifact, not a runtime -- same rule as NOEXEC above. RUN cp -a /initramfs-fat /initramfs-nodev && \ cp src/kernel/vmsfs/vmsfs.ko /initramfs-nodev/lib/modules/vms.ko && \ - cd /initramfs-nodev && find . | cpio -o -H newc 2>/dev/null | gzip > /boot/initramfs-ovmx-nodev.cpio.gz && \ + find /initramfs-nodev -exec touch -h -d "@${SOURCE_DATE_EPOCH}" {} + && \ + cd /initramfs-nodev && find . | sort | cpio -o -H newc 2>/dev/null | gzip -n > /boot/initramfs-ovmx-nodev.cpio.gz && \ echo "NODEV initramfs: $(ls -lh /boot/initramfs-ovmx-nodev.cpio.gz | awk '{print $5}')" # Ensure vmlinuz symlink exists @@ -510,6 +538,12 @@ COPY --from=builder /boot/ovmx-distrib.img /boot/ovmx-distrib.img # carried here so the build's kit artifact is retrievable like the other # `-o dist` outputs, not buried inside the builder stage. COPY --from=builder /boot/ovmx-os.kit /boot/ovmx-os.kit +# The kit's own internal file listing (vms-d73, tools/cut-release.sh) -- +# carried out alongside the kit itself so a release cut can fold "the kit's +# internal manifest names DCL.EXE/LOGINOUT.EXE/STARTUP.COM" into +# release-manifest.json from the actual shipped kit, without needing +# ovmx_kit_pack on the release host. +COPY --from=builder /boot/ovmx-os.kit.manifest.txt /boot/ovmx-os.kit.manifest.txt # Negative controls for test_executive_integral.sh only — never a runtime. COPY --from=builder /boot/initramfs-ovmx-noexec.cpio.gz /boot/initramfs-ovmx-noexec.cpio.gz COPY --from=builder /boot/initramfs-ovmx-nodev.cpio.gz /boot/initramfs-ovmx-nodev.cpio.gz diff --git a/tools/cut-release.sh b/tools/cut-release.sh new file mode 100755 index 000000000..f8821578b --- /dev/null +++ b/tools/cut-release.sh @@ -0,0 +1,252 @@ +#!/bin/bash +# cut-release.sh - build a versioned, checksummed OVMX release bundle from a +# CLEAN tree (vms-d73, epic vms-a84 RELEASE ENGINEERING). +# +# THE PROBLEM THIS REPLACES: every OVMX release before this was hand-rolled -- +# a conductor ran the container build, copied artifacts out, and hand-wrote +# checksums and release notes. There was no repeatable "cut a release" +# command. This is that command. +# +# WHAT IT DOES: +# 1. git-archives the requested ref to a scratch tree UNDER /tmp and builds +# THERE -- never the working repo (CLAUDE.md "never rw-mount + build +# into the repo"; the standing containerized-build rule, Project Rule 9). +# 2. Runs the existing containerized build (distro/Dockerfile.bootable, +# unchanged pipeline -- this script does not reinvent the build, it +# drives it) via `docker buildx build --output=local`, extracting +# vmlinuz / initramfs-ovmx-slim.cpio.gz / ovmx-distrib.img / ovmx-os.kit +# to the host. +# 3. Reads OVMX_PRODUCT_NAME / OVMX_PRODUCT_VERSION out of the ARCHIVED +# copy of src/libvms/include/ovmx_identity.h -- the identity SSOT +# (INV-1) -- so the version is single-sourced from the exact commit +# being cut, never a literal in this script. +# 4. Emits SHA256SUMS and a machine-readable release-manifest.json (an +# OVMX-defined format, Rule 8: not a VSI/PCSI artifact, labeled as ours) +# mapping component -> version -> sha256, including the OS kit's own +# internal file manifest (tools/ovmx_kit_pack.c) so DCL.EXE/LOGINOUT.EXE/ +# STARTUP.COM's presence in the shipped kit is asserted from the CUT +# BUNDLE itself, not merely from the build's internal log. +# 5. Writes the bundle to dist/release-/ (or --out-dir). +# +# Byte-reproducibility (the vms-d73 DONE criterion) depends on +# SOURCE_DATE_EPOCH: two cuts of the SAME commit must embed the SAME clock +# into every wall-clock field the build would otherwise stamp (the OS kit's +# kh_build_time, the mastered VMSFS image's per-file timestamps, and every +# packed initramfs's cpio/gzip metadata -- see distro/Dockerfile.bootable and +# tools/ovmx_kit_pack.c / tools/vmsfs_master.c). This script defaults that +# clock to the CUT COMMIT'S timestamp, so two independent cuts of the same +# ref are deterministic without the caller doing anything -- see +# .github/workflows/ci.yml's cut-release-reproducible job, which builds +# twice from a clean tree and diffs the artifact bytes. +# +# Usage: +# tools/cut-release.sh [options] +# +# Options: +# --ref REF Git ref/commit to cut (default: HEAD) +# --out-dir DIR Bundle output directory +# (default: /dist/release-) +# --work-dir DIR Scratch dir for the git-archive tree +# (default: a fresh mktemp -d under /tmp) +# --keep-work Do not delete the scratch dir on exit +# --no-cache Pass --no-cache to the container build (a true +# from-scratch build; needed to PROVE +# reproducibility rather than measure cache reuse) +# --source-date-epoch N Override the reproducible-build clock +# (default: the commit time of --ref) +# -h, --help Show this help and exit +# +# Exit 0 on a complete bundle; nonzero and a diagnostic on any failure. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(git -C "$SCRIPT_DIR" rev-parse --show-toplevel)" + +GIT_REF="HEAD" +OUT_DIR="" +WORK_DIR="" +KEEP_WORK=0 +NO_CACHE=0 +SOURCE_DATE_EPOCH_OVERRIDE="" + +usage() { + sed -n '2,45p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//' +} + +while [ $# -gt 0 ]; do + case "$1" in + --ref) GIT_REF="$2"; shift 2 ;; + --ref=*) GIT_REF="${1#--ref=}"; shift ;; + --out-dir) OUT_DIR="$2"; shift 2 ;; + --out-dir=*) OUT_DIR="${1#--out-dir=}"; shift ;; + --work-dir) WORK_DIR="$2"; shift 2 ;; + --work-dir=*) WORK_DIR="${1#--work-dir=}"; shift ;; + --keep-work) KEEP_WORK=1; shift ;; + --no-cache) NO_CACHE=1; shift ;; + --source-date-epoch) SOURCE_DATE_EPOCH_OVERRIDE="$2"; shift 2 ;; + --source-date-epoch=*) SOURCE_DATE_EPOCH_OVERRIDE="${1#--source-date-epoch=}"; shift ;; + -h|--help) usage; exit 0 ;; + *) echo "cut-release.sh: unknown option: $1" >&2; usage >&2; exit 1 ;; + esac +done + +log() { echo "cut-release: $*"; } +fail() { echo "cut-release: FATAL: $*" >&2; exit 1; } + +command -v docker >/dev/null 2>&1 || fail "docker not found" +docker buildx version >/dev/null 2>&1 || fail "docker buildx not found (needed for --output=local)" + +# --- Resolve the commit being cut ------------------------------------------ +COMMIT="$(git -C "$REPO_ROOT" rev-parse "$GIT_REF")" \ + || fail "cannot resolve git ref '$GIT_REF' in $REPO_ROOT" + +if [ -n "$(git -C "$REPO_ROOT" status --porcelain 2>/dev/null)" ]; then + log "NOTE: working tree has uncommitted changes -- git archive only cuts" \ + "committed content at $COMMIT ($GIT_REF); uncommitted edits are NOT in this bundle." +fi + +SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH_OVERRIDE:-$(git -C "$REPO_ROOT" log -1 --format=%ct "$COMMIT")}" +case "$SOURCE_DATE_EPOCH" in + ''|*[!0-9]*) fail "SOURCE_DATE_EPOCH must be a nonnegative integer, got: $SOURCE_DATE_EPOCH" ;; +esac + +# --- Scratch tree: git archive the commit, NEVER build in the working repo - +CLEANUP_WORK_DIR=0 +if [ -z "$WORK_DIR" ]; then + WORK_DIR="$(mktemp -d /tmp/ovmx-cut-release.XXXXXX)" + CLEANUP_WORK_DIR=1 +else + mkdir -p "$WORK_DIR" +fi +SRC_DIR="$WORK_DIR/src" +DOCKER_OUT_DIR="$WORK_DIR/dockerout" +mkdir -p "$SRC_DIR" + +cleanup() { + if [ "$KEEP_WORK" -eq 0 ] && [ "$CLEANUP_WORK_DIR" -eq 1 ]; then + rm -rf "$WORK_DIR" + fi +} +trap cleanup EXIT + +log "cutting $GIT_REF ($COMMIT) into clean tree: $SRC_DIR" +git -C "$REPO_ROOT" archive --format=tar "$COMMIT" | tar -x -C "$SRC_DIR" + +# --- Version: single-sourced from the ARCHIVED ovmx_identity.h, not a literal +# in this script (vms-d73 constraint). Reads the exact commit being cut, not +# the possibly-ahead working tree. +IDENTITY_HEADER="$SRC_DIR/src/libvms/include/ovmx_identity.h" +[ -f "$IDENTITY_HEADER" ] || fail "identity header missing from the cut tree: $IDENTITY_HEADER" + +PRODUCT_NAME="$(sed -n 's/^#define[[:space:]]\+OVMX_PRODUCT_NAME[[:space:]]\+"\([^"]*\)".*/\1/p' "$IDENTITY_HEADER" | head -1)" +PRODUCT_VERSION="$(sed -n 's/^#define[[:space:]]\+OVMX_PRODUCT_VERSION[[:space:]]\+"\([^"]*\)".*/\1/p' "$IDENTITY_HEADER" | head -1)" +[ -n "$PRODUCT_NAME" ] || fail "could not read OVMX_PRODUCT_NAME from $IDENTITY_HEADER" +[ -n "$PRODUCT_VERSION" ] || fail "could not read OVMX_PRODUCT_VERSION from $IDENTITY_HEADER" +log "product: $PRODUCT_NAME $PRODUCT_VERSION (single-sourced from ovmx_identity.h)" + +OUT_DIR="${OUT_DIR:-$REPO_ROOT/dist/release-$PRODUCT_VERSION}" +mkdir -p "$OUT_DIR" + +# --- The containerized build (unchanged pipeline; this script drives it) --- +docker buildx inspect >/dev/null 2>&1 || \ + docker buildx create --name ovmx-cut-release --use >/dev/null + +BUILDX_ARGS=( + build + -f "$SRC_DIR/distro/Dockerfile.bootable" + --build-arg "SOURCE_DATE_EPOCH=$SOURCE_DATE_EPOCH" + --output "type=local,dest=$DOCKER_OUT_DIR" +) +[ "$NO_CACHE" -eq 1 ] && BUILDX_ARGS+=(--no-cache) +BUILDX_ARGS+=("$SRC_DIR") + +log "building (SOURCE_DATE_EPOCH=$SOURCE_DATE_EPOCH, no-cache=$NO_CACHE) -- this runs the full" \ + "kernel + VMS-native toolchain + QEMU build; expect ~25-30 minutes cold" +docker buildx "${BUILDX_ARGS[@]}" + +# --- Collect the release artifact set -------------------------------------- +# Exactly the four vms-d73 OUTCOME artifacts. Symlinks (docker local export +# can preserve them, e.g. /boot/vmlinuz -> vmlinuz-) are dereferenced +# with cp -L so the shipped bundle is self-contained real files, not a +# dangling reference into a container that no longer exists. +declare -A ARTIFACT_SRC=( + [vmlinuz]="$DOCKER_OUT_DIR/boot/vmlinuz" + [initramfs-ovmx-slim.cpio.gz]="$DOCKER_OUT_DIR/boot/initramfs-ovmx-slim.cpio.gz" + [ovmx-distrib.img]="$DOCKER_OUT_DIR/boot/ovmx-distrib.img" + [ovmx-os.kit]="$DOCKER_OUT_DIR/boot/ovmx-os.kit" +) +ARTIFACT_ORDER=(vmlinuz initramfs-ovmx-slim.cpio.gz ovmx-distrib.img ovmx-os.kit) + +for name in "${ARTIFACT_ORDER[@]}"; do + src="${ARTIFACT_SRC[$name]}" + [ -e "$src" ] || fail "expected build artifact missing: $src" + cp -L "$src" "$OUT_DIR/$name" +done + +# The OS kit's own internal manifest (tools/ovmx_kit_pack.c list output), +# captured at build time from the exact ovmx-os.kit this bundle ships +# (distro/Dockerfile.bootable's kit-packing stage writes it) -- carried +# alongside so the release manifest can name the kit's real contents without +# needing ovmx_kit_pack on this host. +KIT_LISTING_SRC="$DOCKER_OUT_DIR/boot/ovmx-os.kit.manifest.txt" +[ -f "$KIT_LISTING_SRC" ] || fail "OS kit internal manifest missing from build output: $KIT_LISTING_SRC" +cp "$KIT_LISTING_SRC" "$OUT_DIR/ovmx-os.kit.manifest.txt" + +# Ground-source check (vms-d73 DONE criterion), re-verified against the +# SHIPPED bundle file, not just the build's internal gate: the OS kit must +# actually name the images the login chain depends on. +for name in "DCL.EXE" "LOGINOUT.EXE" "STARTUP.COM"; do + grep -qF "$name" "$OUT_DIR/ovmx-os.kit.manifest.txt" || \ + fail "ovmx-os.kit.manifest.txt (shipped kit's own listing) does not name $name" +done +log "OS kit manifest names DCL.EXE, LOGINOUT.EXE, STARTUP.COM (ground-source check on the shipped kit)" + +# --- Checksums --------------------------------------------------------------- +( + cd "$OUT_DIR" + sha256sum "${ARTIFACT_ORDER[@]}" ovmx-os.kit.manifest.txt > SHA256SUMS +) +log "wrote $OUT_DIR/SHA256SUMS" + +# --- Machine-readable release manifest -------------------------------------- +# OVMX-defined format (Project Rule 8): this is NOT a VSI/PCSI artifact and is +# not presented as one -- OpenVMS/PCSI have no equivalent machine-readable +# release manifest this mirrors. It is OVMX release-engineering tooling +# output, JSON by convenience, labeled as ours right here. +MANIFEST="$OUT_DIR/release-manifest.json" +CUT_AT_UTC="$(date -u +%Y-%m-%dT%H:%M:%SZ)" + +sha256_of() { sha256sum "$OUT_DIR/$1" | awk '{print $1}'; } +bytes_of() { stat -c%s "$OUT_DIR/$1"; } + +{ + printf '{\n' + printf ' "_format": "ovmx-release-manifest",\n' + printf ' "_format_note": "OVMX-defined release manifest (Project Rule 8) -- not a VSI/PCSI artifact; no VMS-authentic equivalent is claimed.",\n' + printf ' "format_version": 1,\n' + printf ' "product_name": "%s",\n' "$PRODUCT_NAME" + printf ' "product_version": "%s",\n' "$PRODUCT_VERSION" + printf ' "git_ref": "%s",\n' "$GIT_REF" + printf ' "git_commit": "%s",\n' "$COMMIT" + printf ' "source_date_epoch": %s,\n' "$SOURCE_DATE_EPOCH" + printf ' "cut_at_utc": "%s",\n' "$CUT_AT_UTC" + printf ' "artifacts": [\n' + first=1 + for name in "${ARTIFACT_ORDER[@]}"; do + [ "$first" -eq 1 ] && first=0 || printf ',\n' + printf ' {"component": "%s", "product_version": "%s", "sha256": "%s", "bytes": %s}' \ + "$name" "$PRODUCT_VERSION" "$(sha256_of "$name")" "$(bytes_of "$name")" + done + printf '\n ],\n' + printf ' "ovmx_os_kit_internal_manifest": {\n' + printf ' "file": "ovmx-os.kit.manifest.txt",\n' + printf ' "sha256": "%s",\n' "$(sha256_of "ovmx-os.kit.manifest.txt")" + printf ' "names_verified_present": ["DCL.EXE", "LOGINOUT.EXE", "STARTUP.COM"]\n' + printf ' }\n' + printf '}\n' +} > "$MANIFEST" +log "wrote $MANIFEST" + +log "release bundle complete: $OUT_DIR" +ls -la "$OUT_DIR" diff --git a/tools/ovmx_kit_pack.c b/tools/ovmx_kit_pack.c index 04e41219b..c4bef6c28 100644 --- a/tools/ovmx_kit_pack.c +++ b/tools/ovmx_kit_pack.c @@ -192,6 +192,28 @@ static void walk_stage(const char *root, const char *reldir, struct pack_list *l closedir(d); } +/* + * kitpack_build_time - the timestamp stamped into kh_build_time. + * + * Reproducibility (vms-d73, tools/cut-release.sh): a plain time(NULL) makes + * kh_build_time the one wall-clock byte difference between two otherwise + * byte-identical kit builds of the same tree. When SOURCE_DATE_EPOCH is set + * (Dockerfile.bootable exports it from its own build-arg of the same name, + * default 0) it is authoritative; a real build passes the release commit's + * timestamp, so the field stays honest instead of freezing at the epoch. + */ +static time_t kitpack_build_time(void) +{ + const char *sde = getenv("SOURCE_DATE_EPOCH"); + if (sde && sde[0] != '\0') { + char *end = NULL; + long v = strtol(sde, &end, 10); + if (end != sde && *end == '\0' && v >= 0) + return (time_t)v; + } + return time(NULL); +} + static int entry_cmp(const void *a, const void *b) { const struct pack_entry *ea = a; @@ -236,7 +258,7 @@ static int do_pack(const char *kitfile, const char *stagedir, snprintf(hdr.kh_producer, sizeof(hdr.kh_producer), "%s", producer); snprintf(hdr.kh_product_version, sizeof(hdr.kh_product_version), "%s", OVMX_PRODUCT_VERSION); - hdr.kh_build_time = (uint64_t)time(NULL); + hdr.kh_build_time = (uint64_t)kitpack_build_time(); hdr.kh_file_count = (uint32_t)l.count; hdr.kh_index_offset = sizeof(struct ovmx_kit_header); hdr.kh_payload_offset = hdr.kh_index_offset + diff --git a/tools/vmsfs_master.c b/tools/vmsfs_master.c index 072e5cf04..ca8fd1e6c 100644 --- a/tools/vmsfs_master.c +++ b/tools/vmsfs_master.c @@ -313,6 +313,30 @@ static void assign_data(struct node *dir, uint32_t *next_lbn) * Block writer * ================================================================ */ +/* + * master_build_time - the timestamp stamped into every mastered file's + * fh_created/fh_modified/fh_accessed (write_header() applies one value to + * every entry, not per-file real mtimes -- see build_tree() above, which + * only carries st_size out of lstat(), never st_mtime). + * + * Reproducibility (vms-d73, tools/cut-release.sh): time(NULL) would make this + * the one wall-clock difference between two otherwise byte-identical masters + * of the same staged tree. SOURCE_DATE_EPOCH (Dockerfile.bootable exports it + * from its own build-arg of the same name, default 0) is authoritative when + * set; a real release build passes the release commit's timestamp. + */ +static time_t master_build_time(void) +{ + const char *sde = getenv("SOURCE_DATE_EPOCH"); + if (sde && sde[0] != '\0') { + char *end = NULL; + long v = strtol(sde, &end, 10); + if (end != sde && *end == '\0' && v >= 0) + return (time_t)v; + } + return time(NULL); +} + static int wr_block(int fd, uint32_t lbn, const void *buf) { off_t off = (off_t)lbn * VMSFS_BLOCK_SIZE; @@ -582,7 +606,7 @@ static int do_master(const char *image, const char *label, return 1; } - time_t now = time(NULL); + time_t now = master_build_time(); uint8_t zero[VMSFS_BLOCK_SIZE]; memset(zero, 0, sizeof(zero)); From 677df2a446db4a2617512839d8159f7c7c682b7a Mon Sep 17 00:00:00 2001 From: alice Date: Mon, 10 Aug 2026 14:53:36 +0000 Subject: [PATCH 2/2] release: fix cpio inode non-determinism in initramfs packing (vms-d73) CI's cut-release-reproducible job caught a real byte diff in initramfs-ovmx-slim.cpio.gz between two independent --no-cache cuts of the same commit (vmlinuz, ovmx-distrib.img, ovmx-os.kit were all already byte-identical). Root-caused by direct measurement, not guessing: isolated reproducers for both the kernel modules (vms.ko/vmsfs.ko via kbuild) and the static musl binaries (STARTUP.EXE/INITIALIZE.EXE/vms_mount_helper) proved BYTE-IDENTICAL across two independent --no-cache container builds -- ruling out the "kernel modules embed build metadata" hypothesis. The actual cause: GNU cpio's "newc" format embeds the REAL FILESYSTEM INODE NUMBER of every entry, which is not deterministic across independent container builds even when every file's content and mtime are identical (verified directly with two byte-identical directory trees built independently on the same host). Fix: add --reproducible to every `cpio -o -H newc` invocation in distro/Dockerfile.bootable (fat/slim/noexec/nodev initramfs packing) -- GNU cpio's own name for --ignore-devno --ignore-dirnlink --renumber-inodes. Verified locally: two independently-created, byte-identical directory trees now produce byte-identical cpio archives with this flag; without it they differ at the same header offset (the inode field) the CI failure pointed to. Co-Authored-By: Claude Opus 4.8 (1M context) --- distro/Dockerfile.bootable | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/distro/Dockerfile.bootable b/distro/Dockerfile.bootable index 0f8a529d9..ac0e6621b 100644 --- a/distro/Dockerfile.bootable +++ b/distro/Dockerfile.bootable @@ -316,12 +316,19 @@ RUN mkdir -p /initramfs-fat/dev /initramfs-fat/proc /initramfs-fat/sys \ echo "OK: PARTS_SETUP.COM present under SYS\$UPDATE: (SYSUPD)" && \ # Pack the fat initramfs. Reproducibility (vms-d73): normalize every file's # mtime to SOURCE_DATE_EPOCH (cp resets mtime to build wall-clock, which - # the cpio "newc" format embeds per entry) and sort the cpio member order - # -- find(1)'s raw readdir order is not a byte-reproducibility guarantee - # even when the create sequence is identical -- and drop gzip's embedded - # mtime (-n) so the compressed stream's bytes depend only on content. + # the cpio "newc" format embeds per entry); sort the cpio member order -- + # find(1)'s raw readdir order is not a byte-reproducibility guarantee even + # when the create sequence is identical; drop gzip's embedded mtime (-n); + # and --reproducible (measured necessary, not theoretical -- vms-d73 CI + # caught a real diff without it), which is GNU cpio's own name for + # --ignore-devno --ignore-dirnlink --renumber-inodes. Without it, cpio's + # "newc" format embeds the REAL filesystem inode number of every entry + # (verified directly: two byte-identical directory trees, created + # independently with identical mtimes, still produced different cpio + # archives purely from inode allocation differing across builds) -- file + # CONTENT and mtimes being reproducible is not sufficient on its own. find /initramfs-fat -exec touch -h -d "@${SOURCE_DATE_EPOCH}" {} + && \ - cd /initramfs-fat && find . | sort | cpio -o -H newc 2>/dev/null | gzip -n > /boot/initramfs-ovmx.cpio.gz && \ + cd /initramfs-fat && find . | sort | cpio -o -H newc --reproducible 2>/dev/null | gzip -n > /boot/initramfs-ovmx.cpio.gz && \ echo "FAT initramfs: $(ls -lh /boot/initramfs-ovmx.cpio.gz | awk '{print $5}')" # -- OVMX OS kit (vms-0b6) ------------------------------------------------------ @@ -475,7 +482,7 @@ RUN mkdir -p /initramfs-slim/dev /initramfs-slim/proc /initramfs-slim/sys \ cp -r distro/rootfs/vms/SYS0/SYSCOMMON/SYSMGR/* /initramfs-slim/vms/SYS0/SYSCOMMON/SYSMGR/ && \ # Pack the slim initramfs (reproducibility normalization, see FAT step above) find /initramfs-slim -exec touch -h -d "@${SOURCE_DATE_EPOCH}" {} + && \ - cd /initramfs-slim && find . | sort | cpio -o -H newc 2>/dev/null | gzip -n > /boot/initramfs-ovmx-slim.cpio.gz && \ + cd /initramfs-slim && find . | sort | cpio -o -H newc --reproducible 2>/dev/null | gzip -n > /boot/initramfs-ovmx-slim.cpio.gz && \ echo "SLIM initramfs: $(ls -lh /boot/initramfs-ovmx-slim.cpio.gz | awk '{print $5}')" # -- NOEXEC initramfs (negative control) --------------------------------------- @@ -491,7 +498,7 @@ RUN mkdir -p /initramfs-slim/dev /initramfs-slim/proc /initramfs-slim/sys \ RUN cp -a /initramfs-fat /initramfs-noexec && \ rm -f /initramfs-noexec/lib/modules/vms.ko && \ find /initramfs-noexec -exec touch -h -d "@${SOURCE_DATE_EPOCH}" {} + && \ - cd /initramfs-noexec && find . | sort | cpio -o -H newc 2>/dev/null | gzip -n > /boot/initramfs-ovmx-noexec.cpio.gz && \ + cd /initramfs-noexec && find . | sort | cpio -o -H newc --reproducible 2>/dev/null | gzip -n > /boot/initramfs-ovmx-noexec.cpio.gz && \ echo "NOEXEC initramfs: $(ls -lh /boot/initramfs-ovmx-noexec.cpio.gz | awk '{print $5}')" # -- NODEV initramfs (negative control) ----------------------------------------- @@ -508,7 +515,7 @@ RUN cp -a /initramfs-fat /initramfs-noexec && \ RUN cp -a /initramfs-fat /initramfs-nodev && \ cp src/kernel/vmsfs/vmsfs.ko /initramfs-nodev/lib/modules/vms.ko && \ find /initramfs-nodev -exec touch -h -d "@${SOURCE_DATE_EPOCH}" {} + && \ - cd /initramfs-nodev && find . | sort | cpio -o -H newc 2>/dev/null | gzip -n > /boot/initramfs-ovmx-nodev.cpio.gz && \ + cd /initramfs-nodev && find . | sort | cpio -o -H newc --reproducible 2>/dev/null | gzip -n > /boot/initramfs-ovmx-nodev.cpio.gz && \ echo "NODEV initramfs: $(ls -lh /boot/initramfs-ovmx-nodev.cpio.gz | awk '{print $5}')" # Ensure vmlinuz symlink exists