From 8a938439dc922489c35cc80fcb34e0ad080aed27 Mon Sep 17 00:00:00 2001 From: Markus Fritsche Date: Thu, 23 Jul 2026 17:36:01 +0200 Subject: [PATCH] test: RED contract for extract_trace (thinking reveal) --- tests/test_grind_trace.py | 141 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 141 insertions(+) create mode 100644 tests/test_grind_trace.py diff --git a/tests/test_grind_trace.py b/tests/test_grind_trace.py new file mode 100644 index 0000000..63f0672 --- /dev/null +++ b/tests/test_grind_trace.py @@ -0,0 +1,141 @@ +"""Spec for extract_trace: the grinder's live reasoning-progress lines are +logged to stderr but discarded by _remote_grind (capture_output=True, only +p.stdout returned). This test asserts a pure function that extracts the +useful bracketed trace lines from raw stderr — the [iter …], [EMPTY REPLY …], +[tier-escalation …] and [regression …] lines — and drops everything else. + +extract_trace MUST be a pure function: stderr: str -> str. No side effects, +no I/O, no model calls. Feed it the raw stderr from a grind session; it +returns only the trace lines, joined with newlines. + +Run: python3 -m pytest tests/test_grind_trace.py -q +""" +import importlib.machinery +import importlib.util +import os +import sys + +import pytest + +GRINDER = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), + "bin", "bullpen-grinder") +loader = importlib.machinery.SourceFileLoader("grinder_under_test", GRINDER) +spec = importlib.util.spec_from_loader(loader.name, loader) +grinder = importlib.util.module_from_spec(spec) +sys.modules[loader.name] = grinder +loader.exec_module(grinder) + +extract_trace = grinder.extract_trace + + +# ---- sample stderr blobs ------------------------------------------------ + +SAMPLE_NORMAL = """\ +[iter 0 tier=deepseek-v4-flash-dspark wrote=1 failed=42 target=whole-spec] +[iter 1 tier=deepseek-v4-flash-dspark wrote=1 failed=38 target=whole-spec] +[iter 2 tier=deepseek-v4-flash-dspark wrote=1 failed=35 target=whole-spec] +[iter 3 tier=deepseek-v4-flash-dspark wrote=1 failed=30 target=whole-spec] +""" + +SAMPLE_EMPTY_REPLY = """\ +[iter 4 tier=deepseek-v4-flash-dspark EMPTY REPLY finish_reason=length reasoning_tokens=3997 budget=4000] +[reasoning starved: budget 4000 -> 32000 (ceiling), retrying] +[iter 5 tier=deepseek-v4-flash-dspark wrote=1 failed=28 target=whole-spec] +""" + +SAMPLE_ESCALATION = """\ +[iter 6 tier=deepseek-v4-flash-dspark wrote=0 failed=28 target=whole-spec REGRESSED] +[regression ['test_foo'] -> escalate to gpt-oss-120b] +[iter 7 tier=gpt-oss-120b wrote=1 failed=25 target=whole-spec] +""" + +SAMPLE_STALL_ESCALATE = """\ +[iter 8 tier=gpt-oss-120b wrote=0 failed=25 target=whole-spec] +[stall -> escalate to kimi-k3] +[iter 9 tier=kimi-k3 wrote=1 failed=20 target=whole-spec] +""" + +SAMPLE_NOISE = """\ +WARN: bwrap missing — tests running UNSANDBOXED +/usr/lib/python3.11/subprocess.py:968: DeprecationWarning: something unrelated +..F..F.. +2 failed, 3 passed in 1.23s +""" + +SAMPLE_FULL_SESSION = SAMPLE_NORMAL + SAMPLE_NOISE + SAMPLE_EMPTY_REPLY + SAMPLE_ESCALATION + SAMPLE_STALL_ESCALATE + + +# ---- basic behaviour ---------------------------------------------------- + +def test_extract_trace_returns_empty_on_empty_input(): + assert extract_trace("") == "" + + +def test_extract_trace_returns_empty_when_no_trace_lines(): + noise = "WARN: bwrap missing\n..F..F..\n2 failed, 3 passed\n" + assert extract_trace(noise) == "" + + +def test_extract_trace_pulls_iter_lines(): + result = extract_trace(SAMPLE_NORMAL) + lines = result.strip().splitlines() + assert len(lines) == 4 + assert all("[iter " in line for line in lines) + + +def test_extract_trace_pulls_empty_reply_lines(): + result = extract_trace(SAMPLE_EMPTY_REPLY) + lines = result.strip().splitlines() + # must include the EMPTY REPLY line and the reasoning starved line + assert any("EMPTY REPLY" in line for line in lines) + assert any("reasoning starved" in line for line in lines) + + +def test_extract_trace_pulls_escalation_lines(): + result = extract_trace(SAMPLE_ESCALATION) + lines = result.strip().splitlines() + assert any("escalate to" in line for line in lines) + assert any("REGRESSED" in line for line in lines) + + +def test_extract_trace_pulls_stall_escalation(): + result = extract_trace(SAMPLE_STALL_ESCALATE) + lines = result.strip().splitlines() + assert any("stall -> escalate" in line for line in lines) + + +def test_extract_trace_drops_noise(): + result = extract_trace(SAMPLE_NOISE) + assert result.strip() == "" + + +def test_extract_trace_full_session(): + result = extract_trace(SAMPLE_FULL_SESSION) + lines = result.strip().splitlines() + # every returned line must be a bracketed trace line + for line in lines: + assert line.startswith("["), f"non-trace line leaked through: {line!r}" + # must have iter, empty-reply, escalation, stall lines + assert any("[iter " in l for l in lines) + assert any("EMPTY REPLY" in l for l in lines) + assert any("escalate to" in l for l in lines) + assert any("stall ->" in l for l in lines) + + +def test_extract_trace_preserves_line_order(): + result = extract_trace(SAMPLE_FULL_SESSION) + lines = result.strip().splitlines() + # iter numbers should be monotonically non-decreasing + iters = [] + for line in lines: + m = __import__("re").search(r"\[iter (\d+)", line) + if m: + iters.append(int(m.group(1))) + assert iters == sorted(iters), "trace lines lost their original order" + + +def test_extract_trace_joins_with_newlines(): + result = extract_trace(SAMPLE_NORMAL) + assert "\n" in result, "multiple lines should be newline-joined" + # no trailing newline + assert not result.endswith("\n") or result.count("\n") == len(result.strip().splitlines()) - 1