#!/usr/bin/env python3
"""@researcher — bullpen worker: web-search + fetch top sources + synthesize a CITED digest.
Reactive, multi-step. Engine = dspark via the gated proxy (needs big context). Saves to /bullpen (mneme).
  bullpen-researcher --once "your question"
"""
import json, os, re, subprocess, sys, time, urllib.request
sys.path[:0] = [p for p in (os.path.join(os.path.dirname(os.path.realpath(__file__)), "..", "lib"), "/usr/local/lib/bullpen") if os.path.isdir(p)]
import bullpen_worker as bw
import bullpen_config as cfg
import bullpen_mem as mem

ENGINE   = cfg.PROXY
MODEL    = cfg.MODEL
N_SOURCES = 3
SRC_CAP   = 8000
THINK_DIR = os.path.join(cfg.STATE_DIR, "thinking")   # live reasoning side-files (THINKING REVEAL #2)


def _prune_thinking(max_age=3600):
    try:
        now = time.time()
        for fn in os.listdir(THINK_DIR):
            p = os.path.join(THINK_DIR, fn)
            if os.path.isfile(p) and now - os.path.getmtime(p) > max_age:
                os.remove(p)
    except Exception:
        pass

def web_search(query):
    try:
        return json.loads(bw.lmcp("web_search", query=query).stdout).get("results", [])
    except Exception:
        return []

def fetch(url):
    try:
        r = subprocess.run(["degater", "fetch_url", f"url={url}"], capture_output=True, text=True, timeout=90)
        return r.stdout.strip() if r.returncode == 0 else ""
    except Exception:
        return ""

def synthesize(question, srcs, thinking_id=None):
    blocks = [f"[{i}] {s['title']} — {s['url']}\n{s['text'][:SRC_CAP]}" for i, s in enumerate(srcs, 1)]
    sysmsg = ("You are a research analyst for the bullpen fleet. Every source below was retrieved "
              "through `degater`, a headless-Chromium tool YOU use that renders JavaScript and "
              "works around bot walls (Cloudflare/Anubis/'checking your browser'/CAPTCHA "
              "challenges) — that is how these pages were fetched at all. If asked about your own "
              "tools or how you retrieve pages, answer from this fact directly; do not web-search "
              "for 'degater' as if it were an unrelated external product to look up (it is not the "
              "VLSI reverse-engineering tool 'Degate' or an industrial degating machine — those are "
              "unrelated matches a naive search will surface).\n\n"
              "Answer the question using ONLY the numbered sources below. "
              "Be concise and structured; give concrete numbers where the sources provide them. Cite "
              "claims inline as [1]/[2]/[3]. Flag where sources disagree or are uncertain. Do NOT invent "
              "facts not in the sources. End with a 'Sources:' list mapping [n] to URL.")
    user = f"Question: {question}\n\nSources:\n" + "\n\n".join(blocks)
    payload = json.dumps({"model": MODEL,
        "messages": [{"role": "system", "content": sysmsg}, {"role": "user", "content": user}],
        "temperature": 0.3, "max_tokens": 1300, "stream": True}).encode()
    req = urllib.request.Request(ENGINE, payload, {"Content-Type": "application/json"})
    # THINKING REVEAL #2: stream the reasoning into a side file as it grows, so bullseye can
    # poll + render it live (the "progressing" view). Truncate at start; flush per delta.
    tf = None
    if thinking_id is not None:
        try:
            os.makedirs(THINK_DIR, exist_ok=True)
            _prune_thinking()
            tf = open(os.path.join(THINK_DIR, f"{thinking_id}.txt"), "w", encoding="utf-8")
        except Exception:
            tf = None
    content, reasoning, served = [], [], None
    with urllib.request.urlopen(req, timeout=240) as r:
        for raw in r:                                  # SSE: collect content + reasoning deltas
            line = raw.decode("utf-8", "replace").strip()
            if not line.startswith("data:"):
                continue
            chunk = line[5:].strip()
            if chunk == "[DONE]":
                break
            try:
                d = json.loads(chunk)
                delta = d["choices"][0].get("delta", {})
            except Exception:
                continue
            if served is None:
                served = d.get("model")   # the proxy may fail over; report what actually served
            if delta.get("content"):
                content.append(delta["content"])
            if delta.get("reasoning"):
                reasoning.append(delta["reasoning"])
                if tf:
                    try:
                        tf.write(delta["reasoning"]); tf.flush()
                    except Exception:
                        tf = None
    if tf:
        try:
            tf.close()
        except Exception:
            pass
    return "".join(content).strip(), "".join(reasoning).strip(), served

def remember(text):
    """Fire-and-forget: persist a research digest line to mneme under /bullpen/research
    (distinct from /bullpen/artifacts' raw fetch pointers and /bullpen/ops' policy facts)."""
    mem.save(text.replace("\n", " ").strip()[:900], ns="/bullpen/research")

def _eval_citations(msg_id, digest, n_srcs):
    """Trajectory eval (mechanical, no LLM): does the digest actually cite its sources?
    A digest with zero [n] markers despite having fetched sources is a fabrication risk
    the output-only check (did synthesize() return non-empty text) can't see — the text
    can read as confident and well-formed while citing nothing. See
    ~/claude/trajectory-evals-design.md §4b."""
    markers = sorted(set(re.findall(r"\[(\d+)\]", digest)))
    verdict = "PASS" if markers else "FAIL"
    note = (f"{len(markers)} distinct marker(s) {markers} of {n_srcs} sources cited" if markers
            else f"0 citation markers despite {n_srcs} fetched sources")
    mem.save(f"EVAL agent=researcher msg_id={msg_id} rubric=citation-integrity "
             f"verdict={verdict} note=\"{note}\" ts={int(time.time())}",
             ns="/trajectory/researcher")

def dispatch(msg):
    q = msg.get("body", "")
    results = web_search(q)
    if not results:
        # R3: long/verbose questions often return empty (observed 3/6 in the W30 run). Retry once
        # with a shortened query — the first clause usually carries the searchable intent.
        short = " ".join(q.split()[:12])
        if short and short != q:
            results = web_search(short)
    if not results:
        return "no search results (web_search failed/empty, incl. shortened retry)"
    srcs = []
    for r in results[:N_SOURCES + 2]:          # try a couple extra in case fetches fail
        if len(srcs) >= N_SOURCES: break
        txt = fetch(r.get("url", ""))
        if len(txt) > 300:
            srcs.append({"url": r.get("url", ""), "title": (r.get("title", "") or "")[:80], "text": txt})
    if not srcs:
        return "found results but couldn't fetch any source pages"
    try:
        digest, thinking, served = synthesize(q, srcs, thinking_id=msg.get("id"))
    except Exception as e:
        return f"synthesis error: {e}"
    # R5: mark web-derived digests so a later recall knows this text came from fetched pages
    # (a prompt-injected page could otherwise become trusted persistent memory).
    remember(f"[untrusted-web-derived] research: {q} -> {digest[:400]}")
    _eval_citations(msg.get("id"), digest, len(srcs))
    model_tag = f"asked {MODEL}, served by {served}" if served and served != MODEL else f"model {served or MODEL}"
    out = f"{digest}\n\n(synthesized from {len(srcs)} fetched sources, {model_tag})"
    if thinking:  # THINKING REVEAL (post-hoc): sentinel-wrapped so bullseye dims the whole block
        out += "\n\n⟦think⟧\n" + thinking[:4000] + ("…" if len(thinking) > 4000 else "") + "\n⟦/think⟧"
    return out

if __name__ == "__main__":
    if len(sys.argv) > 2 and sys.argv[1] == "--once":
        print(dispatch({"body": sys.argv[2], "id": 0}))
    else:
        bw.run("researcher", dispatch,
               online="researcher online — web search + multi-source cited synthesis (dspark). @researcher <question>",
               ack="…researching (search+fetch+synthesize, ~1-2 min)")
