#!/usr/bin/env python3
"""bullpen-trajectory-dispatcher-audit — mechanical trajectory eval for @dispatcher (§4b).

Sweeps room history for @dispatcher replies, re-runs the CURRENT keyword-scoring
function (`_score()` — the exact one live dispatch() calls) against each historical
ask, and flags any case where the deterministic keyword path would route differently
today than what was actually recommended at the time. This catches ROSTER regex
drift (a pattern tightened/loosened later silently reclassifying old asks), not
per-call bugs — dispatch() and this audit call the identical function, so a fresh
ask always self-agrees; the value is comparing PAST recommendations against
PRESENT rules.

Two kinds of historical replies are marked N-A, not FAIL — they were never rule-
checkable in the first place: an ambiguous keyword score routed through the LLM
tie-break, or the debate route (@architect/@skeptic, not in ROSTER at all).

Not run automatically yet — invoke by hand or wire into a timer once useful.
Per ~/claude/trajectory-evals-design.md §4b/§5.

NOTE: /etc/bullpen/mneme-token is root-only (0600), unlike the group-readable
post-secret. The room-facing services write fine because their systemd units
have no User= (root by default); a manual/ad-hoc run needs `sudo` or the
mem.save() call fails silently (fail-soft by design) and nothing persists —
use --dry-run first to sanity-check output before assuming a bare run wrote.

  bullpen-trajectory-dispatcher-audit [--since=N] [--dry-run]   # needs sudo for a real (non-dry-run) write
"""
import importlib.machinery, importlib.util, json, os, re, sys, time

sys.path[:0] = [p for p in (os.path.join(os.path.dirname(os.path.realpath(__file__)), "..", "lib"), "/usr/local/lib/bullpen") if os.path.isdir(p)]
import bullpen_worker as bw
import bullpen_mem as mem

# Load the LIVE bullpen-dispatcher script as a module so this audit calls the exact
# same _score()/ROSTER dispatch() uses — no copy-pasted logic to drift out of sync.
# bullpen-dispatcher has no .py suffix, so spec_from_file_location's suffix-based loader
# inference can't find one (returns None) — an explicit SourceFileLoader is required.
_DISP_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), "bullpen-dispatcher")
_loader = importlib.machinery.SourceFileLoader("bullpen_dispatcher_live", _DISP_PATH)
_spec = importlib.util.spec_from_loader(_loader.name, _loader)
_disp = importlib.util.module_from_spec(_spec)
_loader.exec_module(_disp)

VALID_NICKS = {n for n, _, _ in _disp.ROSTER}
NICK_RE = re.compile(r"→\s*@(\w+)")

def _read_all(since=0):
    out = []
    for line in bw.lmcp("room_read", since=since).stdout.splitlines():
        line = line.strip()
        if not line:
            continue
        try:
            out.append(json.loads(line))
        except Exception:
            pass
    return out

def audit(since=0, dry_run=False):
    msgs = _read_all(since)
    by_id = {m["id"]: m for m in msgs if "id" in m}
    n_pass = n_fail = n_na = 0
    for m in msgs:
        if m.get("from") != "dispatcher" or m.get("type") != "reply":
            continue
        ask = by_id.get(m.get("in_reply_to"))
        if not ask:
            continue                      # ask fell outside this sweep's --since window
        mm = NICK_RE.search(m.get("body", ""))
        if not mm:
            continue                      # roster-dump reply — nothing was recommended, nothing to check
        picked = mm.group(1)
        body = ask.get("body", "")
        if picked not in VALID_NICKS:
            verdict, note = "N-A", f"'{picked}' is a conversant/debate route, not in ROSTER — not audited"
        else:
            scores = _disp._score(body)
            top_s, top_nick, _ = scores[0]
            second_s = scores[1][0]
            deterministic = top_s >= 1 and top_s > second_s
            if not deterministic:
                verdict = "N-A"
                note = f"ambiguous keyword score (top={top_s} second={second_s}) — LLM tie-break, not rule-checkable"
            elif top_nick == picked:
                verdict = "PASS"
                note = f"keyword winner '{top_nick}' still matches what was recommended"
            else:
                verdict = "FAIL"
                note = (f"keyword winner is now '{top_nick}' (score {top_s}) but dispatcher "
                        f"recommended '{picked}' at ask time — ROSTER pattern drift since then")
        if verdict == "PASS": n_pass += 1
        elif verdict == "FAIL": n_fail += 1
        else: n_na += 1
        text = (f"EVAL agent=dispatcher msg_id={m['id']} ask_id={ask['id']} rubric=roster-consistency "
                f"verdict={verdict} note=\"{note}\" ts={int(time.time())}")
        if dry_run:
            print(text)
        else:
            mem.save(text, ns="/trajectory/dispatcher")
    print(f"audited: {n_pass} pass, {n_fail} flagged, {n_na} n-a (llm-arbitrated/debate)", file=sys.stderr)

if __name__ == "__main__":
    since, dry = 0, "--dry-run" in sys.argv
    for a in sys.argv[1:]:
        if a.startswith("--since="):
            since = int(a.split("=", 1)[1])
    audit(since, dry)
