#!/usr/bin/env python3
"""bullpen-doctor — the herd's health monitor. Answers "busy or passed out?" from the room.

Deterministic (no LLM): reads host vitals (load/temp/mem) + the herd's active jobs (grinder
runs on boltzmann, claude sessions on noether) with their AGE and PROGRESS, and returns a verdict
per job: HEALTHY-BUSY / STALLED (running long, no progress) / near-timeout / idle. Complements
@triage (which fires AFTER a failure) — the doctor watches DURING the run.

  bullpen-doctor --once            # print a checkup
  bullpen-doctor --serve doctor    # room worker: replies to `bp doctor "..."` with a checkup
"""
import json, os, re, subprocess, sys, time

import os, sys
sys.path[:0] = [p for p in (os.path.join(os.path.dirname(os.path.realpath(__file__)), "..", "lib"), "/usr/local/lib/bullpen") if os.path.isdir(p)]
import bullpen_config as cfg
SIC = "/usr/local/bin/sic"
STALL_SECS = 1500         # a job older than this → flag (near the lurker RUN_TIMEOUT of 1800s)
HOT_C      = 80           # thermal warning

def sh(host, cmd, timeout=25):
    argv = ["sh", "-c", cmd] if host == cfg.COORD_HOST else [SIC, host, "sh", "-c", cmd]
    try:
        return subprocess.run(argv, capture_output=True, text=True, timeout=timeout)
    except Exception as e:
        class R: returncode = 1; stdout = ""; stderr = str(e)
        return R()

def vitals(host):
    cmd = ("printf 'load=%s ' \"$(cut -d' ' -f1-3 /proc/loadavg)\"; "
           "t=$(cat /sys/class/thermal/thermal_zone0/temp 2>/dev/null); "
           "[ -n \"$t\" ] && printf 'temp=%dC ' $((t/1000)); "
           "free -m | awk '/Mem:/{printf \"mem=%d/%dMB\", $3, $2}'")
    r = sh(host, cmd)
    return r.stdout.strip() if r.returncode == 0 and r.stdout.strip() else f"UNREACHABLE ({r.stderr.strip()[:50]})"

def grinds_on_grindhost():
    # active grinder runs + age. [b]ullpen bracket-trick so the pgrep doesn't match its own cmdline.
    cmd = (r"""for p in $(pgrep -f '[b]ullpen-grinder --once' 2>/dev/null); do """
           r"""a=$(ps -o etimes= -p $p 2>/dev/null | tr -d ' '); echo "grind pid=$p age=${a}s"; done""")
    r = sh(cfg.GRIND_HOST, cmd)
    return r.stdout.strip()

def claude_on_coordhost():
    # the HERD's agent sessions = `claude` whose cwd is a ~/<nick>_lurker dir, tagged with the
    # MODEL that instance runs (its .model file). The persistent primary session is excluded.
    cmd = (r"""for p in $(pgrep -x claude 2>/dev/null); do """
           r"""cw=$(readlink /proc/$p/cwd 2>/dev/null); d=$(basename "$cw" 2>/dev/null); """
           r"""case "$d" in *_lurker) a=$(ps -o etimes= -p $p 2>/dev/null | tr -d ' '); """
           r"""m=$(cat "$cw/.model" 2>/dev/null); """
           r"""echo "agent=${d%_lurker} model=${m:-default} pid=$p age=${a}s";; esac; done""")
    r = sh(cfg.COORD_HOST, cmd)
    return r.stdout.strip()

def _parse_lines(text, fields):
    out = []
    for ln in (text or "").splitlines():
        row = {f: (re.search(rf"\b{f}=([^\s]+)", ln) or [None, None])[1] for f in fields}
        if any(row.values()):
            out.append({k: v for k, v in row.items() if v is not None})
    return out

def _ages(text):
    return [int(a) for a in re.findall(r"age=(\d+)s", text)]

def _temp(v):
    m = re.search(r"temp=(\d+)C", v);  return int(m.group(1)) if m else None

DUP_WINDOW    = 400     # recent room messages to scan for the stuck-retry pattern
DUP_THRESHOLD = 3       # this many byte-identical (from,to,body) posts = flag a loop
DUP_WINDOW_SECS = 3600  # …but only inside one hour — see stuck_dispatch_check for why

def _recent_room(limit=DUP_WINDOW):
    try:
        r = subprocess.run([SIC, cfg.ROOM_HOST, "lmcp-tool", "room_read", "since=0", f"limit={limit}"],
                           capture_output=True, text=True, timeout=30)
    except Exception:
        return None
    if r.returncode != 0:
        return None
    out = []
    for l in r.stdout.splitlines():
        l = l.strip()
        if l:
            try: out.append(json.loads(l))
            except Exception: pass
    return out

def orchestrator_wake_check():
    """An orchestrator that cannot be woken by a worker's REPLY deadlocks the moment it hands
    off — it ends its turn correctly and waits forever for an event the lurker discards.
    docs/ORCHESTRATION.md §2 states the invariant ("any orchestrator MUST wake on replies")
    and it cost a stalled campaign on 2026-07-24, but nothing checked it: the opt-in is a bare
    flag file in a per-nick lurker dir, and deploy/install.sh deliberately does not touch those.
    So a fresh fleet, a restored home, or a renamed nick brings the herd up looking healthy and
    silently unable to finish a pipeline.

    Cheap and read-only: one `test -e` per orchestrator on the coordinator host."""
    flags = []
    for nick in cfg.ORCHESTRATORS:
        r = sh(cfg.COORD_HOST, f'test -e ~/{nick}_lurker/.wakereplies && echo yes || echo no')
        out = (r.stdout or "").strip()
        state = out.splitlines()[-1] if out else ""
        if state == "no":
            flags.append(f"⚠ @{nick} is an orchestrator but its lurker is REPLY-DEAF "
                         f"(~/{nick}_lurker/.wakereplies missing) — it will deadlock on handoff")
        elif state != "yes":
            flags.append(f"⚠ could not check @{nick}'s wake set on {cfg.COORD_HOST}")
    return flags


def stuck_dispatch_check():
    """Deterministic loop-detector, no LLM: >=DUP_THRESHOLD byte-identical (from,to,body) posts
    among the last DUP_WINDOW room messages is the fingerprint of a stuck retry/replay, NOT
    fresh reasoning — a model re-deciding the same dispatch rarely reproduces prose verbatim
    even at temperature 0, since surrounding context differs each call. Found via the
    2026-07-25 asteroids incident: 15 byte-identical "Write a webgame-smoke test..." dispatches
    from @foreman to @testdesigner over ~9h, with ZERO opencode session activity in that window
    (checked directly in orca's opencode.db) — the room kept accepting posts while nothing was
    actually reasoning, and nobody was watching for it. This check closes that blind spot;
    doctor previously covered boltzmann/noether process vitals only, never orca/opencode."""
    msgs = _recent_room()
    if not msgs:
        return []
    seen = {}
    for m in msgs:
        if m.get("type") not in ("ask", "chat") or not m.get("body"):
            continue
        key = (m.get("from"), m.get("to"), m.get("body"))
        seen.setdefault(key, []).append((m.get("id"), m.get("ts") or 0))
    flags = []
    for (frm, to, body), posts in seen.items():
        if len(posts) < DUP_THRESHOLD:
            continue
        # A stuck retry is TIGHT IN TIME — that is what makes it a loop rather than a habit.
        # Some legitimate posts are byte-identical by construction: bullpen-selfimprove
        # composes a deterministic brief per ISO week, so re-running it days apart produces
        # the same bytes and tripped this check on 2026-08-02. Requiring the duplicates to
        # fall inside one window separates "the same message again and again right now" from
        # "the same scheduled message, weeks apart".
        stamps = sorted(t for _, t in posts if t)
        if stamps and (stamps[-1] - stamps[0]) > DUP_WINDOW_SECS:
            continue
        ids = [i for i, _ in posts]
        flags.append(f"⚠ {frm} -> {to}: {len(posts)}x byte-identical post (ids {min(ids)}..{max(ids)}) "
                     f"— stuck retry, not fresh reasoning: {body[:80]!r}")
    return flags

def checkup():
    bz, ne = vitals(cfg.GRIND_HOST), vitals(cfg.COORD_HOST)
    grinds, sessions = grinds_on_grindhost(), claude_on_coordhost()
    lines = ["🩺 herd checkup",
             f"  {cfg.GRIND_HOST}: {bz}",
             f"  {cfg.COORD_HOST}:   {ne}"]
    lines.append(f"  active grinds:   {grinds if grinds else 'none'}")
    lines.append(f"  active sessions: {sessions if sessions else 'none'}")

    flags = []
    for host, v in ((cfg.GRIND_HOST, bz), (cfg.COORD_HOST, ne)):
        t = _temp(v)
        if t and t >= HOT_C: flags.append(f"⚠ {host} HOT ({t}C)")
    long_jobs = [a for a in (_ages(grinds) + _ages(sessions)) if a >= STALL_SECS]
    busy = bool(grinds) or bool(sessions)
    if long_jobs:
        flags.append(f"⚠ {len(long_jobs)} job(s) running >{STALL_SECS//60}min — check for a stall/loop")
    flags.extend(stuck_dispatch_check())
    flags.extend(orchestrator_wake_check())

    if flags:
        verdict = "NEEDS A LOOK — " + "; ".join(flags)
    elif busy:
        verdict = "BUSY & HEALTHY — jobs running within normal time, temps fine (working, not wedged)"
    else:
        verdict = "IDLE — nothing grinding right now, all quiet"
    lines.append(f"  → {verdict}")
    return "\n".join(lines)

def checkup_json():
    """Structured checkup for bullseye's status panel (per-agent model/host/liveness + vitals)."""
    bz, ne = vitals(cfg.GRIND_HOST), vitals(cfg.COORD_HOST)
    def pv(v):
        tm = re.search(r"temp=(\d+)C", v)
        return {"load": (re.search(r"load=([\d.]+)", v) or [None, None])[1],
                "temp_c": int(tm.group(1)) if tm else None,
                "mem": (re.search(r"mem=(\S+)", v) or [None, None])[1],
                "reachable": "UNREACHABLE" not in v}
    agents = _parse_lines(claude_on_coordhost(), ["agent", "model", "pid", "age"])
    grinds = _parse_lines(grinds_on_grindhost(), ["pid", "age"])
    long_jobs = [x for x in (agents + grinds) if x.get("age", "0").isdigit() and int(x["age"]) >= STALL_SECS]
    hot = [h for h, v in ((cfg.GRIND_HOST, bz), (cfg.COORD_HOST, ne))
           if (re.search(r"temp=(\d+)C", v) and int(re.search(r"temp=(\d+)C", v).group(1)) >= HOT_C)]
    stuck = stuck_dispatch_check()
    verdict = "needs-a-look" if (hot or long_jobs or stuck) else ("busy-healthy" if (agents or grinds) else "idle")
    return {"ts": int(time.time()), "hosts": {cfg.GRIND_HOST: pv(bz), cfg.COORD_HOST: pv(ne)},
            "agents": agents, "grinds": grinds, "verdict": verdict,
            "flags": {"hot": hot, "long_jobs": len(long_jobs), "stuck_dispatches": stuck}}

# ---- room-worker mode (noether; sic hertz for the room) ----
TRUST = cfg.TRUST

def _hz(*a, timeout=30):
    return subprocess.run([SIC, cfg.ROOM_HOST, "lmcp-tool", *a], capture_output=True, text=True, timeout=timeout)

def _room_read(since):
    try: r = _hz("room_read", f"since={since}")
    except Exception: return None
    if r.returncode != 0: return None
    out = []
    for l in r.stdout.splitlines():
        l = l.strip()
        if l:
            try: out.append(json.loads(l))
            except Exception: pass
    return out

def serve(nick):
    state = os.path.expanduser(f"~/.{nick}.since")
    def persist(v):
        try: open(state, "w").write(str(v))
        except Exception: pass
    def say(body, to="", typ="chat", rid=None):
        a = ["room_say", f"from={nick}", f"type={typ}", f"body={body}"]
        if to: a.append(f"to={to}")
        if rid is not None: a.append(f"in_reply_to={rid}")
        try: return _hz(*a).returncode == 0
        except Exception: return False
    try: since = int(open(state).read().strip())
    except Exception:
        m = _room_read(0); since = max([x.get("id", 0) for x in m], default=0) if m else 0; persist(since)
    sys.stderr.write(f"bullpen-doctor serving @{nick} from id {since}\n")
    while True:
        try:
            batch = _room_read(since)
            if batch is None:
                time.sleep(3); continue
            for m in batch:
                mid = m.get("id", since)
                if mid <= since: continue
                asker = (m.get("from") or "").lower()
                if (m.get("from") == nick or m.get("type") not in ("ask", "chat")
                        or (m.get("to") or "").lstrip("@").lower() != nick or asker not in TRUST):
                    since = mid; persist(since); continue
                report = checkup()
                posted = False
                for _ in range(3):
                    if say(report, to=f"@{asker}", typ="reply", rid=mid): posted = True; break
                    time.sleep(2)
                if posted: since = mid; persist(since)
                else: break
        except Exception as e:
            sys.stderr.write(f"doctor loop error: {e}\n")
        time.sleep(3)

if __name__ == "__main__":
    if len(sys.argv) >= 2 and sys.argv[1] == "--serve":
        serve(sys.argv[2].strip() if len(sys.argv) > 2 else "doctor")
    elif len(sys.argv) >= 2 and sys.argv[1] == "--json":
        print(json.dumps(checkup_json()))
    else:
        print(checkup())
