hyperloom custom: keep each measurement's raw logs (they overwrote each other)

tests/hyperloom_benchmark_test.py

@@ -79,6 +79,21 @@ def main():
     texts["prose"] = "B"
     assert [k for k in bench.gated_text_keys(ref_texts) if texts.get(k) != ref_texts[k]] == ["prose"]
 
+    # Raw logs are kept per measurement. They used to be fixed names inside RESULT_DIR, so a second
+    # measurement in the same RESULT_DIR overwrote the first one's serve.log / llama-bench.log — measured
+    # on 2026-11-02, the 19:01 arm destroyed the 18:45 baseline's raw logs, and a pair's A and B halves
+    # would have collided the same way. This is that reproducer: two measurement names, one RESULT_DIR.
+    import tempfile
+
+    with tempfile.TemporaryDirectory() as td:
+        root = pathlib.Path(td)
+        for fname, payload in (("arm-A", "FIRST"), ("arm-B", "SECOND")):
+            bench.run(["sh", "-c", f"echo {payload}"], "llama-bench", 10, bench.log_dir_for(root, fname))
+        assert (root / "logs-arm-A/llama-bench.log").read_text().strip() == "FIRST"
+        assert (root / "logs-arm-B/llama-bench.log").read_text().strip() == "SECOND"
+        # A repeat of the same measurement overwrites only itself, and the result file is untouched by it.
+        assert bench.log_dir_for(root, "arm-A") == root / "logs-arm-A"
+
     # Box rule 2026-11-01 12:13: the drafter at long context is the arm that faulted this box, so a
     # candidate that asks for both must be refused (scores zero) and not load a drafter.
     import os

tools/hyperloom/custom/benchmark.py

@@ -346,6 +346,19 @@ def run(cmd, name, timeout, out_dir):
     return rc, text
 
 
+def log_dir_for(out_dir, name):
+    """The directory one measurement's raw logs go in.
+
+    They used to share fixed names with the result directory itself, so a second measurement in the same
+    RESULT_DIR silently overwrote the first one's serve.log / llama-bench.log / kld.log: measured on
+    2026-11-02, the 19:01 arm destroyed the 18:45 baseline's raw logs, and a pair's two halves (A, B) would
+    have collided the same way. One directory per RESULT_FILENAME makes the raw evidence survive the run.
+    """
+    log_dir = out_dir / f"logs-{name}"
+    log_dir.mkdir(parents=True, exist_ok=True)
+    return log_dir
+
+
 def bench_csv(text):
     """Parse `llama-bench -o csv` into {(n_prompt, n_gen): tok/s}, keyed by header names."""
     lines = [ln for ln in text.splitlines() if ln.strip()]
@@ -413,7 +426,7 @@ def long_prompt(n_tokens):
     return LONG_PROMPT_UNIT * max(1, int(n_tokens * LONG_CHARS_PER_TOKEN / len(LONG_PROMPT_UNIT)) + 1)
 
 
-def serve_measure(onebit, model, args, reps, tokens, timeout, out_dir, longctx=0, longtok=128):
+def serve_measure(onebit, model, args, reps, tokens, timeout, log_dir, longctx=0, longtok=128):
     """Start `1bit serve` once, run the three prompts (+ the long-context leg), return the numbers."""
     port = free_port()
     cmd = [onebit, "serve", "-m", str(model), "--port", str(port)] + args
@@ -423,14 +436,14 @@ def serve_measure(onebit, model, args, reps, tokens, timeout, out_dir, longctx=0
     # ggml-hrx for the counts; `1bit serve` exposes no `-lv` passthrough, so they only surface when
     # its own verbosity reaches the library's INFO level, and `placement` stays null otherwise.
     run_env = dict(os.environ, GGML_HRX_STRICT="0", GGML_HRX_REPORT="2")
-    logtext = (out_dir / "serve.log").open("w", encoding="utf-8")
+    logtext = (log_dir / "serve.log").open("w", encoding="utf-8")
     proc = subprocess.Popen(cmd, env=run_env, stdout=logtext, stderr=subprocess.STDOUT,
                             stdin=subprocess.DEVNULL)
     tps, texts = {}, {}
     long_info = {}
     try:
         if not wait_healthy(port, proc, time.time() + timeout / 2):
-            raise RuntimeError(f"`1bit serve` did not become healthy (see {out_dir / 'serve.log'})")
+            raise RuntimeError(f"`1bit serve` did not become healthy (see {log_dir / 'serve.log'})")
         ask(port, "hi", 4, timeout)  # warm-up: pays the first-token compile, not the measurement
         for name, prompt in DECODE_PROMPTS.items():
             samples = []
@@ -456,7 +469,7 @@ def serve_measure(onebit, model, args, reps, tokens, timeout, out_dir, longctx=0
             proc.kill()
             proc.wait()
         logtext.close()
-    return tps, texts, placement_summary(out_dir / "serve.log"), long_info
+    return tps, texts, placement_summary(log_dir / "serve.log"), long_info
 
 
 PLACEMENT_RE = re.compile(r"placement: (\d+) nodes, (\d+) off HRX")
@@ -492,13 +505,13 @@ def gate_reference(ref, base_kld, texts_path):
             "base_kld_sha256": sha256_head(base_kld), "texts_sha256": sha256_head(texts_path)}
 
 
-def kld(ppl, model, args, base, wiki, timeout, out_dir):
+def kld(ppl, model, args, base, wiki, timeout, log_dir):
     """Prefill-path KLD against the stored base logits; records the base when it is missing."""
     cmd = [ppl, "-m", str(model), "-f", str(wiki), "-c", "512", "--chunks", "8",
            "--device", "HRX0", "-ngl", "99"] + (args or ["-b", "512", "-ub", "512"])
     recording = not base.is_file()
     cmd += ["--kl-divergence-base", str(base)] + ([] if recording else ["--kl-divergence"])
-    rc, text = run(cmd, "kld-record" if recording else "kld", timeout, out_dir)
+    rc, text = run(cmd, "kld-record" if recording else "kld", timeout, log_dir)
     if recording:
         return {"recorded": rc == 0 and base.is_file(), "mean": None}
     match = KLD_MEAN_RE.search(text)
@@ -523,6 +536,7 @@ def main():
     out_dir = pathlib.Path(env("RESULT_DIR") or ".").expanduser()
     out_dir.mkdir(parents=True, exist_ok=True)
     name = env("RESULT_FILENAME") or "inferencex_result"
+    log_dir = log_dir_for(out_dir, name)
     model = model_file()
 
     # Out-of-tree build (README.md): `1bit` at the build root, the HRX llama.cpp tools built by
@@ -582,7 +596,7 @@ def finish(passed, reason, rc=0):
     # 1. llama-bench: prefill and decode throughput from one model load.
     rc, text = run([str(bench), "-m", str(model), "-dev", "HRX0"] + bench_args()
                    + ["-p", "512", "-n", "128", "-r", str(reps), "-o", "csv"],
-                   "llama-bench", min(600, deadline - time.time()), out_dir)
+                   "llama-bench", min(600, deadline - time.time()), log_dir)
     rows = bench_csv(text)
     pp512, tg128 = rows.get((512, 0)), rows.get((0, 128))
     if rc != 0 or not pp512 or not tg128:
@@ -593,7 +607,7 @@ def finish(passed, reason, rc=0):
     try:
         tps, texts, placement, long_info = serve_measure(
             onebit, model, serve_lever_args(), reps, tokens,
-            min(1200 if longctx else 600, deadline - time.time()), out_dir,
+            min(1200 if longctx else 600, deadline - time.time()), log_dir,
             longctx=longctx, longtok=longtok)
     except (RuntimeError, OSError) as exc:
         return finish(False, f"serve leg failed: {exc}", rc=1)
@@ -627,7 +641,7 @@ def finish(passed, reason, rc=0):
             ref_texts = {}
     establishing = not ref or not ref_texts or not base_kld.is_file()
 
-    kld_stat = kld(ppl, model, bench_args(), base_kld, wiki, min(900, deadline - time.time()), out_dir)
+    kld_stat = kld(ppl, model, bench_args(), base_kld, wiki, min(900, deadline - time.time()), log_dir)
     result["quality_gate"].update({"kld_max_mean": kld_max, "kld_mean": kld_stat["mean"],
                                    "kld_recorded": kld_stat["recorded"]})
     if establishing and kld_stat["recorded"] and kld_stat["mean"] is None: