test: RED contract for extract_trace (thinking reveal)

tests/test_grind_trace.py

@@ -0,0 +1,141 @@
+"""Spec for extract_trace: the grinder's live reasoning-progress lines are
+logged to stderr but discarded by _remote_grind (capture_output=True, only
+p.stdout returned).  This test asserts a pure function that extracts the
+useful bracketed trace lines from raw stderr — the [iter …], [EMPTY REPLY …],
+[tier-escalation …] and [regression …] lines — and drops everything else.
+
+extract_trace MUST be a pure function: stderr: str -> str.  No side effects,
+no I/O, no model calls.  Feed it the raw stderr from a grind session; it
+returns only the trace lines, joined with newlines.
+
+Run: python3 -m pytest tests/test_grind_trace.py -q
+"""
+import importlib.machinery
+import importlib.util
+import os
+import sys
+
+import pytest
+
+GRINDER = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
+                       "bin", "bullpen-grinder")
+loader = importlib.machinery.SourceFileLoader("grinder_under_test", GRINDER)
+spec = importlib.util.spec_from_loader(loader.name, loader)
+grinder = importlib.util.module_from_spec(spec)
+sys.modules[loader.name] = grinder
+loader.exec_module(grinder)
+
+extract_trace = grinder.extract_trace
+
+
+# ---- sample stderr blobs ------------------------------------------------
+
+SAMPLE_NORMAL = """\
+[iter 0 tier=deepseek-v4-flash-dspark wrote=1 failed=42 target=whole-spec]
+[iter 1 tier=deepseek-v4-flash-dspark wrote=1 failed=38 target=whole-spec]
+[iter 2 tier=deepseek-v4-flash-dspark wrote=1 failed=35 target=whole-spec]
+[iter 3 tier=deepseek-v4-flash-dspark wrote=1 failed=30 target=whole-spec]
+"""
+
+SAMPLE_EMPTY_REPLY = """\
+[iter 4 tier=deepseek-v4-flash-dspark EMPTY REPLY finish_reason=length reasoning_tokens=3997 budget=4000]
+[reasoning starved: budget 4000 -> 32000 (ceiling), retrying]
+[iter 5 tier=deepseek-v4-flash-dspark wrote=1 failed=28 target=whole-spec]
+"""
+
+SAMPLE_ESCALATION = """\
+[iter 6 tier=deepseek-v4-flash-dspark wrote=0 failed=28 target=whole-spec REGRESSED]
+[regression ['test_foo'] -> escalate to gpt-oss-120b]
+[iter 7 tier=gpt-oss-120b wrote=1 failed=25 target=whole-spec]
+"""
+
+SAMPLE_STALL_ESCALATE = """\
+[iter 8 tier=gpt-oss-120b wrote=0 failed=25 target=whole-spec]
+[stall -> escalate to kimi-k3]
+[iter 9 tier=kimi-k3 wrote=1 failed=20 target=whole-spec]
+"""
+
+SAMPLE_NOISE = """\
+WARN: bwrap missing — tests running UNSANDBOXED
+/usr/lib/python3.11/subprocess.py:968: DeprecationWarning: something unrelated
+..F..F..
+2 failed, 3 passed in 1.23s
+"""
+
+SAMPLE_FULL_SESSION = SAMPLE_NORMAL + SAMPLE_NOISE + SAMPLE_EMPTY_REPLY + SAMPLE_ESCALATION + SAMPLE_STALL_ESCALATE
+
+
+# ---- basic behaviour ----------------------------------------------------
+
+def test_extract_trace_returns_empty_on_empty_input():
+    assert extract_trace("") == ""
+
+
+def test_extract_trace_returns_empty_when_no_trace_lines():
+    noise = "WARN: bwrap missing\n..F..F..\n2 failed, 3 passed\n"
+    assert extract_trace(noise) == ""
+
+
+def test_extract_trace_pulls_iter_lines():
+    result = extract_trace(SAMPLE_NORMAL)
+    lines = result.strip().splitlines()
+    assert len(lines) == 4
+    assert all("[iter " in line for line in lines)
+
+
+def test_extract_trace_pulls_empty_reply_lines():
+    result = extract_trace(SAMPLE_EMPTY_REPLY)
+    lines = result.strip().splitlines()
+    # must include the EMPTY REPLY line and the reasoning starved line
+    assert any("EMPTY REPLY" in line for line in lines)
+    assert any("reasoning starved" in line for line in lines)
+
+
+def test_extract_trace_pulls_escalation_lines():
+    result = extract_trace(SAMPLE_ESCALATION)
+    lines = result.strip().splitlines()
+    assert any("escalate to" in line for line in lines)
+    assert any("REGRESSED" in line for line in lines)
+
+
+def test_extract_trace_pulls_stall_escalation():
+    result = extract_trace(SAMPLE_STALL_ESCALATE)
+    lines = result.strip().splitlines()
+    assert any("stall -> escalate" in line for line in lines)
+
+
+def test_extract_trace_drops_noise():
+    result = extract_trace(SAMPLE_NOISE)
+    assert result.strip() == ""
+
+
+def test_extract_trace_full_session():
+    result = extract_trace(SAMPLE_FULL_SESSION)
+    lines = result.strip().splitlines()
+    # every returned line must be a bracketed trace line
+    for line in lines:
+        assert line.startswith("["), f"non-trace line leaked through: {line!r}"
+    # must have iter, empty-reply, escalation, stall lines
+    assert any("[iter " in l for l in lines)
+    assert any("EMPTY REPLY" in l for l in lines)
+    assert any("escalate to" in l for l in lines)
+    assert any("stall ->" in l for l in lines)
+
+
+def test_extract_trace_preserves_line_order():
+    result = extract_trace(SAMPLE_FULL_SESSION)
+    lines = result.strip().splitlines()
+    # iter numbers should be monotonically non-decreasing
+    iters = []
+    for line in lines:
+        m = __import__("re").search(r"\[iter (\d+)", line)
+        if m:
+            iters.append(int(m.group(1)))
+    assert iters == sorted(iters), "trace lines lost their original order"
+
+
+def test_extract_trace_joins_with_newlines():
+    result = extract_trace(SAMPLE_NORMAL)
+    assert "\n" in result, "multiple lines should be newline-joined"
+    # no trailing newline
+    assert not result.endswith("\n") or result.count("\n") == len(result.strip().splitlines()) - 1