diff --git a/bin/bullpen-grinder b/bin/bullpen-grinder index e9b5ff9..750cf69 100755 --- a/bin/bullpen-grinder +++ b/bin/bullpen-grinder @@ -31,6 +31,7 @@ TIERS = os.environ.get("GRIND_TIERS", "deepseek-v4-flash-dspark").split(",") # PY_TIERS = os.environ.get("PY_GRIND_TIERS", "deepseek-v4-flash-dspark,[free] qwen/qwen3-coder") MAX_ITERS = 10 # total model attempts across all tiers ESCALATE_AT = 4 # attempts on a tier without progress before escalating +LARGE_DIFF_LINES = 40 # insertions+deletions above which a GREEN still gets auto-flagged for review COLLECTION_PENALTY = 500 # a module that won't import scores worse than any assertion-fail count PYTEST_TIMEOUT = 120 GO_TEST_TIMEOUT = 600 # compile + test is slower than pytest, especially cold @@ -286,6 +287,13 @@ def _eval_escalation_discipline(rid, escalations, passed, final_model): f"final_model={final_model} passed={passed} note=\"{note}\" ts={int(time.time())}", ns="/trajectory/grinder") +def _diffstat_lines(stat): + """Sum insertions+deletions out of a `git diff --stat` summary line + ('N files changed, A insertions(+), D deletions(-)'). 0 if unparsable.""" + ins = re.search(r"(\d+) insertion", stat) + dels = re.search(r"(\d+) deletion", stat) + return (int(ins.group(1)) if ins else 0) + (int(dels.group(1)) if dels else 0) + def grind(task, test_rel, targets): rid = str(int(time.time())) work = f"/tmp/grind-{rid}" @@ -381,8 +389,22 @@ def grind(task, test_rel, targets): diff = sh(["git", "-C", work, "diff", baseline, b["sha"], "--", ".", *NOISE], timeout=60).stdout if b["passed"]: + # #2 of the W31 self-improvement findings: 125 GREENs shipped, only 41 ever + # reviewed. Flag the risky subset for an automatic @reviewer dispatch (serve() + # below parses this line) instead of leaving every GREEN's review to chance — + # escalated past the floor tier, a regression happened along the way, or the + # diff grew past a "minimal fix" size are exactly the cases most likely to hide + # a real problem in code nobody looks at again. + escalated = b["model"] != TIERS[0] + n_regressions = sum(1 for e in escalations if e.get("reason") == "regression") + difflines = _diffstat_lines(stat) + reasons = [] + if escalated: reasons.append(f"escalated to {b['model']}") + if n_regressions: reasons.append(f"{n_regressions} regression(s)") + if difflines > LARGE_DIFF_LINES: reasons.append(f"large diff ({difflines} lines)") + auto_review = ("AUTO-REVIEW: yes (" + ", ".join(reasons) + ")") if reasons else "AUTO-REVIEW: no" return (f"GREEN ✅ — {test_rel} passes ({_model_tag(b)}).\n{stat}\n" - f"worktree: {work} (diff below, apply after review)\n\n{diff[:3500]}") + f"worktree: {work} (diff below, apply after review)\n{auto_review}\n\n{diff[:3500]}") return (f"RED ❌ — best-so-far {b['failed']} failing ({_model_tag(b)}); could not reach " f"green in {MAX_ITERS} attempts. worktree left at {work} for a human.\n{stat}\n\n" f"{diff[:3000]}\nlast pytest:\n{b['tail']}") @@ -678,6 +700,11 @@ def _room_read(since): except Exception: pass return out +def _auto_review_reason(reply): + """Pulls the reason string out of grind()'s 'AUTO-REVIEW: yes (...)' marker line, or None.""" + m = re.search(r"^AUTO-REVIEW: yes \((.*)\)$", reply, re.M) + return m.group(1) if m else None + def serve(nick): state = os.path.expanduser(f"~/.grinder-{nick}.since") def persist(v): @@ -732,12 +759,16 @@ def serve(nick): targets = [t for t in re.findall(r"\b[\w-]+(?:/[\w.-]+)*/[\w.-]+\.\w+\b", body) if not t.startswith("tests/") and t != test_rel] reply = _remote_grind(body, test_rel, targets) + review_reason = _auto_review_reason(reply) posted = False for _ in range(3): if say(reply, to=f"@{asker}", typ="reply", rid=mid): posted = True; break time.sleep(2) if posted: since = mid; persist(since) else: sys.stderr.write(f"@{nick}: reply failed id {mid}; retry\n"); break + if review_reason: + say(f"🔍 auto-review — @{nick}'s grind for @{asker} flagged itself ({review_reason}); " + f"please review before it ships.\n\n{reply}", to="@reviewer", typ="ask") except Exception as e: sys.stderr.write(f"serve loop error: {e}\n") time.sleep(3)