Markus Fritsche
142 lines
5.1 KiB
Python
142 lines
5.1 KiB
Python
"""Spec for extract_trace: the grinder's live reasoning-progress lines are
|
|
logged to stderr but discarded by _remote_grind (capture_output=True, only
|
|
p.stdout returned). This test asserts a pure function that extracts the
|
|
useful bracketed trace lines from raw stderr — the [iter …], [EMPTY REPLY …],
|
|
[tier-escalation …] and [regression …] lines — and drops everything else.
|
|
|
|
extract_trace MUST be a pure function: stderr: str -> str. No side effects,
|
|
no I/O, no model calls. Feed it the raw stderr from a grind session; it
|
|
returns only the trace lines, joined with newlines.
|
|
|
|
Run: python3 -m pytest tests/test_grind_trace.py -q
|
|
"""
|
|
import importlib.machinery
|
|
import importlib.util
|
|
import os
|
|
import sys
|
|
|
|
import pytest
|
|
|
|
GRINDER = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
|
|
"bin", "bullpen-grinder")
|
|
loader = importlib.machinery.SourceFileLoader("grinder_under_test", GRINDER)
|
|
spec = importlib.util.spec_from_loader(loader.name, loader)
|
|
grinder = importlib.util.module_from_spec(spec)
|
|
sys.modules[loader.name] = grinder
|
|
loader.exec_module(grinder)
|
|
|
|
extract_trace = grinder.extract_trace
|
|
|
|
|
|
# ---- sample stderr blobs ------------------------------------------------
|
|
|
|
SAMPLE_NORMAL = """\
|
|
[iter 0 tier=deepseek-v4-flash-dspark wrote=1 failed=42 target=whole-spec]
|
|
[iter 1 tier=deepseek-v4-flash-dspark wrote=1 failed=38 target=whole-spec]
|
|
[iter 2 tier=deepseek-v4-flash-dspark wrote=1 failed=35 target=whole-spec]
|
|
[iter 3 tier=deepseek-v4-flash-dspark wrote=1 failed=30 target=whole-spec]
|
|
"""
|
|
|
|
SAMPLE_EMPTY_REPLY = """\
|
|
[iter 4 tier=deepseek-v4-flash-dspark EMPTY REPLY finish_reason=length reasoning_tokens=3997 budget=4000]
|
|
[reasoning starved: budget 4000 -> 32000 (ceiling), retrying]
|
|
[iter 5 tier=deepseek-v4-flash-dspark wrote=1 failed=28 target=whole-spec]
|
|
"""
|
|
|
|
SAMPLE_ESCALATION = """\
|
|
[iter 6 tier=deepseek-v4-flash-dspark wrote=0 failed=28 target=whole-spec REGRESSED]
|
|
[regression ['test_foo'] -> escalate to gpt-oss-120b]
|
|
[iter 7 tier=gpt-oss-120b wrote=1 failed=25 target=whole-spec]
|
|
"""
|
|
|
|
SAMPLE_STALL_ESCALATE = """\
|
|
[iter 8 tier=gpt-oss-120b wrote=0 failed=25 target=whole-spec]
|
|
[stall -> escalate to kimi-k3]
|
|
[iter 9 tier=kimi-k3 wrote=1 failed=20 target=whole-spec]
|
|
"""
|
|
|
|
SAMPLE_NOISE = """\
|
|
WARN: bwrap missing — tests running UNSANDBOXED
|
|
/usr/lib/python3.11/subprocess.py:968: DeprecationWarning: something unrelated
|
|
..F..F..
|
|
2 failed, 3 passed in 1.23s
|
|
"""
|
|
|
|
SAMPLE_FULL_SESSION = SAMPLE_NORMAL + SAMPLE_NOISE + SAMPLE_EMPTY_REPLY + SAMPLE_ESCALATION + SAMPLE_STALL_ESCALATE
|
|
|
|
|
|
# ---- basic behaviour ----------------------------------------------------
|
|
|
|
def test_extract_trace_returns_empty_on_empty_input():
|
|
assert extract_trace("") == ""
|
|
|
|
|
|
def test_extract_trace_returns_empty_when_no_trace_lines():
|
|
noise = "WARN: bwrap missing\n..F..F..\n2 failed, 3 passed\n"
|
|
assert extract_trace(noise) == ""
|
|
|
|
|
|
def test_extract_trace_pulls_iter_lines():
|
|
result = extract_trace(SAMPLE_NORMAL)
|
|
lines = result.strip().splitlines()
|
|
assert len(lines) == 4
|
|
assert all("[iter " in line for line in lines)
|
|
|
|
|
|
def test_extract_trace_pulls_empty_reply_lines():
|
|
result = extract_trace(SAMPLE_EMPTY_REPLY)
|
|
lines = result.strip().splitlines()
|
|
# must include the EMPTY REPLY line and the reasoning starved line
|
|
assert any("EMPTY REPLY" in line for line in lines)
|
|
assert any("reasoning starved" in line for line in lines)
|
|
|
|
|
|
def test_extract_trace_pulls_escalation_lines():
|
|
result = extract_trace(SAMPLE_ESCALATION)
|
|
lines = result.strip().splitlines()
|
|
assert any("escalate to" in line for line in lines)
|
|
assert any("REGRESSED" in line for line in lines)
|
|
|
|
|
|
def test_extract_trace_pulls_stall_escalation():
|
|
result = extract_trace(SAMPLE_STALL_ESCALATE)
|
|
lines = result.strip().splitlines()
|
|
assert any("stall -> escalate" in line for line in lines)
|
|
|
|
|
|
def test_extract_trace_drops_noise():
|
|
result = extract_trace(SAMPLE_NOISE)
|
|
assert result.strip() == ""
|
|
|
|
|
|
def test_extract_trace_full_session():
|
|
result = extract_trace(SAMPLE_FULL_SESSION)
|
|
lines = result.strip().splitlines()
|
|
# every returned line must be a bracketed trace line
|
|
for line in lines:
|
|
assert line.startswith("["), f"non-trace line leaked through: {line!r}"
|
|
# must have iter, empty-reply, escalation, stall lines
|
|
assert any("[iter " in l for l in lines)
|
|
assert any("EMPTY REPLY" in l for l in lines)
|
|
assert any("escalate to" in l for l in lines)
|
|
assert any("stall ->" in l for l in lines)
|
|
|
|
|
|
def test_extract_trace_preserves_line_order():
|
|
result = extract_trace(SAMPLE_FULL_SESSION)
|
|
lines = result.strip().splitlines()
|
|
# iter numbers should be monotonically non-decreasing
|
|
iters = []
|
|
for line in lines:
|
|
m = __import__("re").search(r"\[iter (\d+)", line)
|
|
if m:
|
|
iters.append(int(m.group(1)))
|
|
assert iters == sorted(iters), "trace lines lost their original order"
|
|
|
|
|
|
def test_extract_trace_joins_with_newlines():
|
|
result = extract_trace(SAMPLE_NORMAL)
|
|
assert "\n" in result, "multiple lines should be newline-joined"
|
|
# no trailing newline
|
|
assert not result.endswith("\n") or result.count("\n") == len(result.strip().splitlines()) - 1
|