#!/usr/bin/env python3
"""bullpen-evals — read back the trajectory-eval records nobody was reading (#97).

Three workers score their own trajectories into mneme `/trajectory/*`: the grinder rates
escalation discipline (was a tier bump EARNED, or fired before the stall count?), the
researcher rates citation integrity, the dispatcher rates roster consistency. All three
wrote; nothing ever read. W31 called this the free feedback loop for the other gaps —
a config change gets before/after measurement for nothing, if somebody looks.

  bullpen-evals                     # everything, grouped
  bullpen-evals --agent grinder     # one agent
  bullpen-evals --since 2026-07-28  # only records at or after that date
  bullpen-evals --detail            # one line per record instead of the summary

Reads only (mneme reads need no token), so this runs anywhere on the fleet.
"""
import argparse
import os
import re
import sys
import time

sys.path[:0] = [p for p in (os.path.join(os.path.dirname(os.path.realpath(__file__)), "..", "lib"),
                            "/usr/local/lib/bullpen") if os.path.isdir(p)]
import bullpen_mem as mem

# Every eval record contains `rubric=`, and — measured 2026-08-02 — that token hits all of
# them with ZERO false positives, while `EVAL` also matches prose about EvalPlus in other
# namespaces. mneme's FTS5 ANDs its terms and does not stem, so a query is only as good as
# its rarest token; picking this one deliberately rather than by feel.
ANCHOR = "rubric"
FIELD_RE = re.compile(r'(\w+)=(?:"([^"]*)"|(\S+))')


def parse(text):
    """The records are flat `key=value` with one quoted free-text `note`.

    finditer, not findall: findall hands back '' for a group that did NOT participate, so
    `quoted if quoted is not None else bare` always took the quoted branch and every field
    came out empty. Only a match object can tell "matched nothing" from "did not match",
    and the distinction matters here because `note=""` is legitimately empty."""
    out = {}
    for m in FIELD_RE.finditer(text or ""):
        out[m.group(1)] = m.group(2) if m.group(2) is not None else m.group(3)
    return out


def load(since_ts):
    out = []
    for hit in mem.recall(ANCHOR, ns="/trajectory", k=500):
        rec = parse(hit.get("text", ""))
        if not rec.get("rubric"):
            continue
        # mneme's /query returns id, ns, score, text — and NO timestamp. A record therefore
        # has to carry its own `ts=`, which is why every writer puts one in the text. Absent
        # it, treat the time as unknown rather than as the epoch; a single undated record
        # otherwise drags the reported span back to 1970.
        try:
            rec["_ts"] = int(float(rec.get("ts") or 0))
        except ValueError:
            rec["_ts"] = 0
        if since_ts and rec["_ts"] < since_ts:
            continue
        rec["_ns"] = hit.get("ns", "")
        out.append(rec)
    return sorted(out, key=lambda r: r["_ts"])


def main():
    ap = argparse.ArgumentParser(add_help=True)
    ap.add_argument("--agent")
    ap.add_argument("--since", help="YYYY-MM-DD or a unix timestamp")
    ap.add_argument("--detail", action="store_true")
    a = ap.parse_args()

    since_ts = 0
    if a.since:
        try:
            since_ts = int(a.since)
        except ValueError:
            since_ts = int(time.mktime(time.strptime(a.since, "%Y-%m-%d")))

    recs = [r for r in load(since_ts) if not a.agent or r.get("agent") == a.agent]
    if not recs:
        print("no trajectory-eval records match."
              + ("" if since_ts or a.agent else
                 "\nIf that is unexpected: mem.save() fails SOFT, so a worker on a host without "
                 "/etc/bullpen/mneme-token discards its records silently — that is how the "
                 "grinder's went missing for a week (fixed 2026-08-02)."))
        return

    if a.detail:
        for r in recs:
            when = time.strftime("%Y-%m-%d %H:%M", time.localtime(r["_ts"])) if r["_ts"] else "?"
            print(f"{when}  {r.get('agent',''):<11} {r.get('rubric',''):<22} "
                  f"{r.get('verdict',''):<5} msg={r.get('msg_id','?'):<6} {r.get('note','')[:90]}")
        return

    groups = {}
    for r in recs:
        groups.setdefault((r.get("agent", "?"), r.get("rubric", "?")), []).append(r)

    dated = [r["_ts"] for r in recs if r["_ts"]]
    span = (f"{time.strftime('%Y-%m-%d', time.localtime(min(dated)))}"
            f" … {time.strftime('%Y-%m-%d', time.localtime(max(dated)))}") if dated else "undated"
    undated = len(recs) - len(dated)
    print(f"{len(recs)} trajectory-eval record(s), {span}"
          + (f"  ({undated} without a ts= field)" if undated else "") + "\n")
    for (agent, rubric), rs in sorted(groups.items()):
        verdicts = {}
        for r in rs:
            verdicts[r.get("verdict", "?")] = verdicts.get(r.get("verdict", "?"), 0) + 1
        line = ", ".join(f"{v}×{k}" for k, v in sorted(verdicts.items()))
        print(f"  {agent:<11} {rubric:<22} {len(rs):>4}  {line}")
        # A FAIL is the whole point of collecting these — never hide it behind a count.
        for r in rs:
            if r.get("verdict") == "FAIL":
                when = time.strftime("%m-%d %H:%M", time.localtime(r["_ts"])) if r["_ts"] else "?"
                print(f"      FAIL {when} msg={r.get('msg_id','?')}: {r.get('note','')[:96]}")

    ladder = [r for r in recs if r.get("rubric") == "escalation-discipline"]
    if ladder:
        climbed = [r for r in ladder if "no escalation needed" not in r.get("note", "")]
        won = [r for r in ladder if r.get("passed") == "True"]
        print(f"\n  ladder: {len(climbed)}/{len(ladder)} grind(s) had to climb, "
              f"{len(won)}/{len(ladder)} reached green")
        models = {}
        for r in ladder:
            models[r.get("final_model", "?")] = models.get(r.get("final_model", "?"), 0) + 1
        for m, c in sorted(models.items(), key=lambda x: -x[1]):
            print(f"    solved on {m}: {c}")


if __name__ == "__main__":
    main()
