grinder: auto-dispatch @reviewer on escalation/regression/large diff

W31 self-improvement gap #2 (reviewer-confirmed): 125 GREENs shipped, only
41 ever reviewed. Most grinds are low-risk and fine to skip, but a GREEN
that escalated past the floor tier, had a regression along the way, or grew
past a minimal-fix size is exactly the kind of result likely to hide a real
problem in code nobody looks at again. grind()'s report() now emits a
machine-parseable "AUTO-REVIEW: yes (reasons)" marker on those GREENs;
serve() parses it and fires the same @reviewer ask a human would send,
reusing the existing wake-on-address mechanism (reviewer_lurker/CLAUDE.md)
rather than adding a new dispatch path.
This commit is contained in:
2026-07-27 22:27:36 +02:00
parent 1d80c82ba3
commit c6d5e72452
+32 -1
View File
@@ -31,6 +31,7 @@ TIERS = os.environ.get("GRIND_TIERS", "deepseek-v4-flash-dspark").split(",") #
PY_TIERS = os.environ.get("PY_GRIND_TIERS", "deepseek-v4-flash-dspark,[free] qwen/qwen3-coder")
MAX_ITERS = 10 # total model attempts across all tiers
ESCALATE_AT = 4 # attempts on a tier without progress before escalating
LARGE_DIFF_LINES = 40 # insertions+deletions above which a GREEN still gets auto-flagged for review
COLLECTION_PENALTY = 500 # a module that won't import scores worse than any assertion-fail count
PYTEST_TIMEOUT = 120
GO_TEST_TIMEOUT = 600 # compile + test is slower than pytest, especially cold
@@ -286,6 +287,13 @@ def _eval_escalation_discipline(rid, escalations, passed, final_model):
f"final_model={final_model} passed={passed} note=\"{note}\" ts={int(time.time())}",
ns="/trajectory/grinder")
def _diffstat_lines(stat):
"""Sum insertions+deletions out of a `git diff --stat` summary line
('N files changed, A insertions(+), D deletions(-)'). 0 if unparsable."""
ins = re.search(r"(\d+) insertion", stat)
dels = re.search(r"(\d+) deletion", stat)
return (int(ins.group(1)) if ins else 0) + (int(dels.group(1)) if dels else 0)
def grind(task, test_rel, targets):
rid = str(int(time.time()))
work = f"/tmp/grind-{rid}"
@@ -381,8 +389,22 @@ def grind(task, test_rel, targets):
diff = sh(["git", "-C", work, "diff", baseline, b["sha"], "--", ".", *NOISE],
timeout=60).stdout
if b["passed"]:
# #2 of the W31 self-improvement findings: 125 GREENs shipped, only 41 ever
# reviewed. Flag the risky subset for an automatic @reviewer dispatch (serve()
# below parses this line) instead of leaving every GREEN's review to chance —
# escalated past the floor tier, a regression happened along the way, or the
# diff grew past a "minimal fix" size are exactly the cases most likely to hide
# a real problem in code nobody looks at again.
escalated = b["model"] != TIERS[0]
n_regressions = sum(1 for e in escalations if e.get("reason") == "regression")
difflines = _diffstat_lines(stat)
reasons = []
if escalated: reasons.append(f"escalated to {b['model']}")
if n_regressions: reasons.append(f"{n_regressions} regression(s)")
if difflines > LARGE_DIFF_LINES: reasons.append(f"large diff ({difflines} lines)")
auto_review = ("AUTO-REVIEW: yes (" + ", ".join(reasons) + ")") if reasons else "AUTO-REVIEW: no"
return (f"GREEN ✅ — {test_rel} passes ({_model_tag(b)}).\n{stat}\n"
f"worktree: {work} (diff below, apply after review)\n\n{diff[:3500]}")
f"worktree: {work} (diff below, apply after review)\n{auto_review}\n\n{diff[:3500]}")
return (f"RED ❌ — best-so-far {b['failed']} failing ({_model_tag(b)}); could not reach "
f"green in {MAX_ITERS} attempts. worktree left at {work} for a human.\n{stat}\n\n"
f"{diff[:3000]}\nlast pytest:\n{b['tail']}")
@@ -678,6 +700,11 @@ def _room_read(since):
except Exception: pass
return out
def _auto_review_reason(reply):
"""Pulls the reason string out of grind()'s 'AUTO-REVIEW: yes (...)' marker line, or None."""
m = re.search(r"^AUTO-REVIEW: yes \((.*)\)$", reply, re.M)
return m.group(1) if m else None
def serve(nick):
state = os.path.expanduser(f"~/.grinder-{nick}.since")
def persist(v):
@@ -732,12 +759,16 @@ def serve(nick):
targets = [t for t in re.findall(r"\b[\w-]+(?:/[\w.-]+)*/[\w.-]+\.\w+\b", body)
if not t.startswith("tests/") and t != test_rel]
reply = _remote_grind(body, test_rel, targets)
review_reason = _auto_review_reason(reply)
posted = False
for _ in range(3):
if say(reply, to=f"@{asker}", typ="reply", rid=mid): posted = True; break
time.sleep(2)
if posted: since = mid; persist(since)
else: sys.stderr.write(f"@{nick}: reply failed id {mid}; retry\n"); break
if review_reason:
say(f"🔍 auto-review — @{nick}'s grind for @{asker} flagged itself ({review_reason}); "
f"please review before it ships.\n\n{reply}", to="@reviewer", typ="ask")
except Exception as e:
sys.stderr.write(f"serve loop error: {e}\n")
time.sleep(3)