grinder: auto-dispatch @reviewer on escalation/regression/large diff
W31 self-improvement gap #2 (reviewer-confirmed): 125 GREENs shipped, only 41 ever reviewed. Most grinds are low-risk and fine to skip, but a GREEN that escalated past the floor tier, had a regression along the way, or grew past a minimal-fix size is exactly the kind of result likely to hide a real problem in code nobody looks at again. grind()'s report() now emits a machine-parseable "AUTO-REVIEW: yes (reasons)" marker on those GREENs; serve() parses it and fires the same @reviewer ask a human would send, reusing the existing wake-on-address mechanism (reviewer_lurker/CLAUDE.md) rather than adding a new dispatch path.
This commit is contained in:
+32
-1
@@ -31,6 +31,7 @@ TIERS = os.environ.get("GRIND_TIERS", "deepseek-v4-flash-dspark").split(",") #
|
||||
PY_TIERS = os.environ.get("PY_GRIND_TIERS", "deepseek-v4-flash-dspark,[free] qwen/qwen3-coder")
|
||||
MAX_ITERS = 10 # total model attempts across all tiers
|
||||
ESCALATE_AT = 4 # attempts on a tier without progress before escalating
|
||||
LARGE_DIFF_LINES = 40 # insertions+deletions above which a GREEN still gets auto-flagged for review
|
||||
COLLECTION_PENALTY = 500 # a module that won't import scores worse than any assertion-fail count
|
||||
PYTEST_TIMEOUT = 120
|
||||
GO_TEST_TIMEOUT = 600 # compile + test is slower than pytest, especially cold
|
||||
@@ -286,6 +287,13 @@ def _eval_escalation_discipline(rid, escalations, passed, final_model):
|
||||
f"final_model={final_model} passed={passed} note=\"{note}\" ts={int(time.time())}",
|
||||
ns="/trajectory/grinder")
|
||||
|
||||
def _diffstat_lines(stat):
|
||||
"""Sum insertions+deletions out of a `git diff --stat` summary line
|
||||
('N files changed, A insertions(+), D deletions(-)'). 0 if unparsable."""
|
||||
ins = re.search(r"(\d+) insertion", stat)
|
||||
dels = re.search(r"(\d+) deletion", stat)
|
||||
return (int(ins.group(1)) if ins else 0) + (int(dels.group(1)) if dels else 0)
|
||||
|
||||
def grind(task, test_rel, targets):
|
||||
rid = str(int(time.time()))
|
||||
work = f"/tmp/grind-{rid}"
|
||||
@@ -381,8 +389,22 @@ def grind(task, test_rel, targets):
|
||||
diff = sh(["git", "-C", work, "diff", baseline, b["sha"], "--", ".", *NOISE],
|
||||
timeout=60).stdout
|
||||
if b["passed"]:
|
||||
# #2 of the W31 self-improvement findings: 125 GREENs shipped, only 41 ever
|
||||
# reviewed. Flag the risky subset for an automatic @reviewer dispatch (serve()
|
||||
# below parses this line) instead of leaving every GREEN's review to chance —
|
||||
# escalated past the floor tier, a regression happened along the way, or the
|
||||
# diff grew past a "minimal fix" size are exactly the cases most likely to hide
|
||||
# a real problem in code nobody looks at again.
|
||||
escalated = b["model"] != TIERS[0]
|
||||
n_regressions = sum(1 for e in escalations if e.get("reason") == "regression")
|
||||
difflines = _diffstat_lines(stat)
|
||||
reasons = []
|
||||
if escalated: reasons.append(f"escalated to {b['model']}")
|
||||
if n_regressions: reasons.append(f"{n_regressions} regression(s)")
|
||||
if difflines > LARGE_DIFF_LINES: reasons.append(f"large diff ({difflines} lines)")
|
||||
auto_review = ("AUTO-REVIEW: yes (" + ", ".join(reasons) + ")") if reasons else "AUTO-REVIEW: no"
|
||||
return (f"GREEN ✅ — {test_rel} passes ({_model_tag(b)}).\n{stat}\n"
|
||||
f"worktree: {work} (diff below, apply after review)\n\n{diff[:3500]}")
|
||||
f"worktree: {work} (diff below, apply after review)\n{auto_review}\n\n{diff[:3500]}")
|
||||
return (f"RED ❌ — best-so-far {b['failed']} failing ({_model_tag(b)}); could not reach "
|
||||
f"green in {MAX_ITERS} attempts. worktree left at {work} for a human.\n{stat}\n\n"
|
||||
f"{diff[:3000]}\nlast pytest:\n{b['tail']}")
|
||||
@@ -678,6 +700,11 @@ def _room_read(since):
|
||||
except Exception: pass
|
||||
return out
|
||||
|
||||
def _auto_review_reason(reply):
|
||||
"""Pulls the reason string out of grind()'s 'AUTO-REVIEW: yes (...)' marker line, or None."""
|
||||
m = re.search(r"^AUTO-REVIEW: yes \((.*)\)$", reply, re.M)
|
||||
return m.group(1) if m else None
|
||||
|
||||
def serve(nick):
|
||||
state = os.path.expanduser(f"~/.grinder-{nick}.since")
|
||||
def persist(v):
|
||||
@@ -732,12 +759,16 @@ def serve(nick):
|
||||
targets = [t for t in re.findall(r"\b[\w-]+(?:/[\w.-]+)*/[\w.-]+\.\w+\b", body)
|
||||
if not t.startswith("tests/") and t != test_rel]
|
||||
reply = _remote_grind(body, test_rel, targets)
|
||||
review_reason = _auto_review_reason(reply)
|
||||
posted = False
|
||||
for _ in range(3):
|
||||
if say(reply, to=f"@{asker}", typ="reply", rid=mid): posted = True; break
|
||||
time.sleep(2)
|
||||
if posted: since = mid; persist(since)
|
||||
else: sys.stderr.write(f"@{nick}: reply failed id {mid}; retry\n"); break
|
||||
if review_reason:
|
||||
say(f"🔍 auto-review — @{nick}'s grind for @{asker} flagged itself ({review_reason}); "
|
||||
f"please review before it ships.\n\n{reply}", to="@reviewer", typ="ask")
|
||||
except Exception as e:
|
||||
sys.stderr.write(f"serve loop error: {e}\n")
|
||||
time.sleep(3)
|
||||
|
||||
Reference in New Issue
Block a user