#!/usr/bin/env python3
"""bullpen-triage — the failure supervisor (task #75).

A dumb room watcher (no heavy LLM of its own). When an agent posts a FAILURE reply
(timed out / error rc / RED / turn-cap / could not reach green / refused / unparsable),
it asks a SMALL local model to classify WHY and propose ONE concrete fix, then surfaces
that to the human (@markus) for approval — instead of a failure silently scrolling past.

Runs on noether (sic reaches the hertz room; the small model is via the proxy). It never
acts on the proposal itself — a human approves and re-dispatches. Read-only + advisory.
"""
import json, os, re, subprocess, sys, time, urllib.request

SIC   = "/usr/local/bin/sic"
import os, sys
sys.path[:0] = [p for p in (os.path.join(os.path.dirname(os.path.realpath(__file__)), "..", "lib"), "/usr/local/lib/bullpen") if os.path.isdir(p)]
import bullpen_config as cfg
PROXY = cfg.PROXY
MODEL = os.environ.get("TRIAGE_MODEL", "deepseek-v4-flash-dspark")  # local deepseek floor (DGX); qwen3.6-coding backend went dead 2026-07-23
NICK  = "triage"
STATE = os.path.expanduser("~/.triage.since")
FAIL_RE = re.compile(r"timed out|error rc\d|no result|RED ❌|could not reach green|"
                     r"turn.?cap|\brefused\b|unparsable|grinder .*error|no reply from", re.I)

def hz(*a, timeout=30):
    return subprocess.run([SIC, cfg.ROOM_HOST, "lmcp-tool", *a], capture_output=True, text=True, timeout=timeout)

def room_read(since):
    try: r = hz("room_read", f"since={since}")
    except Exception: return None
    if r.returncode != 0: return None
    out = []
    for l in r.stdout.splitlines():
        l = l.strip()
        if l:
            try: out.append(json.loads(l))
            except Exception: pass
    return out

def say(body, to="", typ="chat", rid=None):
    a = ["room_say", f"from={NICK}", f"type={typ}", f"body={body}"]
    if to: a.append(f"to={to}")
    if rid is not None: a.append(f"in_reply_to={rid}")
    try: return hz(*a).returncode == 0
    except Exception: return False

def find_ask(allmsgs, reply):
    rid = reply.get("in_reply_to")
    for m in allmsgs:
        if m.get("id") == rid:
            return m.get("body", "")
    return "(original ask not found)"

def triage(ask, fail):
    """Returns (verdict_text, served_model) — the proxy may fail over from MODEL."""
    sysp = ("You are a triage supervisor for a multi-agent coding/ops system. An agent run FAILED. "
            "Classify the failure as EXACTLY one of: turns-exhausted, looping, model-refused, "
            "tool-or-connection-error, model-incapable, ambiguous-task. Then propose ONE concrete "
            "fix a human can approve (examples: 'escalate to a stronger tier', 'raise max-turns to N', "
            "'clarify/narrow the task', 'a backend is down — retry when up', 'hand to @testdesigner "
            "for a tighter spec'). Output EXACTLY two lines:\nREASON: <class>\nFIX: <one short line>")
    user = f"The agent was asked:\n{ask[:1200]}\n\nIt replied (a failure):\n{fail[:1200]}"
    payload = json.dumps({"model": MODEL, "temperature": 0.1, "max_tokens": 120,
        "messages": [{"role": "system", "content": sysp}, {"role": "user", "content": user}]}).encode()
    req = urllib.request.Request(PROXY, payload, {"Content-Type": "application/json"})
    with urllib.request.urlopen(req, timeout=90) as r:
        d = json.load(r)
    return d["choices"][0]["message"]["content"].strip(), d.get("model")

# Room-history mining (2026-07-28) found 13 near-identical "triage model unreachable" posts to
# @markus spread across 2026-07-21..23 — triage's OWN backend was down/misconfigured for a real
# stretch, and every single occurrence spammed an individual room message with no escalation or
# dedup. The failure supervisor has no supervisor of its own. Fix: after ESCALATE_AT consecutive
# own-backend failures, post ONE consolidated warning and go quiet until it recovers, instead of
# repeating the same noise every ~3s poll for two days straight.
BACKEND_FAIL_ESCALATE_AT = 3

def main():
    def persist(v):
        try: open(STATE, "w").write(str(v))
        except Exception: pass
    try:
        since = int(open(STATE).read().strip())
    except Exception:
        m = room_read(0); since = max([x.get("id", 0) for x in m], default=0) if m else 0; persist(since)
    sys.stderr.write(f"bullpen-triage watching for failures from id {since}\n")
    own_backend_fails = 0
    while True:
        try:
            batch = room_read(since)
            if batch is None:
                time.sleep(3); continue
            allmsgs = None
            for m in batch:
                mid = m.get("id", since)
                if mid <= since: continue
                since = mid; persist(since)
                if m.get("type") != "reply" or m.get("from") == NICK:
                    continue
                body = str(m.get("body", ""))
                if not FAIL_RE.search(body):
                    continue
                if allmsgs is None:
                    allmsgs = room_read(0) or []
                ask = find_ask(allmsgs, m)
                try:
                    verdict, served = triage(ask, body)
                except Exception as e:
                    own_backend_fails += 1
                    if own_backend_fails > BACKEND_FAIL_ESCALATE_AT:
                        # already escalated once; stay quiet in the room, keep the evidence in
                        # the journal only, so a stuck backend doesn't repeat the same room
                        # message every poll for hours/days.
                        sys.stderr.write(f"triage backend still down (fail #{own_backend_fails}): {e}\n")
                        continue
                    verdict, served = f"REASON: tool-or-connection-error\nFIX: triage model unreachable ({e})", None
                    if own_backend_fails == BACKEND_FAIL_ESCALATE_AT:
                        say(f"🔎 triage — my OWN backend ({MODEL}) has failed {own_backend_fails} times in a "
                            f"row (latest: {e}). This looks like the model/proxy is down, not the agents I'm "
                            f"watching — suppressing further individual triage-failure posts until it "
                            f"recovers.", to="@markus")
                        continue
                    who = m.get("from", "?")
                    model_tag = f"model {MODEL}"
                    say(f"🔎 triage — @{who}'s run failed. ({model_tag})\n{verdict}\n"
                        f"(approve → re-dispatch with that fix; ignore to drop.)", to="@markus")
                    continue
                if own_backend_fails >= BACKEND_FAIL_ESCALATE_AT:
                    say(f"🔎 triage — backend recovered after {own_backend_fails} consecutive failures; "
                        f"resuming normal triage.", to="@markus")
                own_backend_fails = 0
                who = m.get("from", "?")
                model_tag = f"asked {MODEL}, served by {served}" if served and served != MODEL else f"model {served or MODEL}"
                say(f"🔎 triage — @{who}'s run failed. ({model_tag})\n{verdict}\n"
                    f"(approve → re-dispatch with that fix; ignore to drop.)", to="@markus")
        except Exception as e:
            sys.stderr.write(f"triage loop error: {e}\n")
        time.sleep(3)

if __name__ == "__main__":
    main()
