bullpen/bin/catalog-diff

#!/usr/bin/env python3
"""catalog-diff — a hand-kept model list against the gateway's catalog. Read-only.

Roundhouse card 2. It changes nothing, disables nothing and proposes nothing; it makes
visible what currently drifts unseen. Measured on 2026-08-02: the pi-agents carry 13
providers and 53 model entries, byte-identical on two hosts, and six of those providers
point at endpoints that no longer answer.

Two distinctions decide whether this is a finding or noise, and the first draft got both
wrong:

  * NORMALISE BOTH SIDES. The hand list stores gateway ids WITH the display tag
    ("[local] deepseek-v4-flash") and so does the catalog. Stripping only one side
    guarantees zero matches and looks exactly like total drift.
  * NOT EVERY ENTRY BELONGS IN THE CATALOG. A provider pointing straight at a backend
    (boltzmann:8085, dirac:8081) deliberately bypasses the gateway — its model is absent
    from the catalog without being dead. Only providers aimed AT the gateway have to be
    found there; the rest are probed directly and reported separately.

  bin/catalog-diff [host ...]      # default: the pi-agent hosts

Needs `sic` to read the remote lists, so it runs on the coordinator, not inside the
gateway's own container.
"""
import json
import os
import subprocess
import sys
import urllib.parse
import urllib.request

sys.path[:0] = [p for p in (os.path.join(os.path.dirname(os.path.realpath(__file__)), "..", "lib"),
                            "/usr/local/lib/bullpen") if os.path.isdir(p)]
import bullpen_config as cfg

TAGS = ("[$] ", "[free] ", "[local] ")
MODELS_JSON = ".pi/agent/models.json"


def _untag(model_id):
    for t in TAGS:
        if model_id.startswith(t):
            return model_id[len(t):]
    return model_id


def catalog():
    url = cfg.PROXY.split("/v1/")[0].rstrip("/") + "/v1/models"
    with urllib.request.urlopen(url, timeout=30) as r:
        return {_untag(m["id"]): m for m in json.load(r).get("data", [])}, url


def hand_list(host):
    r = subprocess.run(["sic", host, "cat", MODELS_JSON],
                       capture_output=True, text=True, timeout=60)
    if r.returncode != 0 or not r.stdout.strip():
        return None
    out = []
    for name, p in json.loads(r.stdout).get("providers", {}).items():
        if not isinstance(p, dict):
            continue
        for m in p.get("models", []):
            mid = m.get("id") if isinstance(m, dict) else m
            if mid:
                out.append((name, _untag(mid), p.get("baseUrl", "")))
    return out


def answers(base_url):
    """True/False, or None when there is no URL to probe. A 401 counts as answering —
    an authenticated endpoint that refuses us is alive, which is the question here."""
    if not base_url:
        return None
    r = subprocess.run(["curl", "-s", "-o", "/dev/null", "-w", "%{http_code}", "-m", "6",
                        base_url.rstrip("/") + "/models"], capture_output=True, text=True)
    return r.stdout.strip() not in ("000", "")


def main():
    hosts = sys.argv[1:] or ["pica", "deus"]
    kat, url = catalog()
    gw = urllib.parse.urlparse(url)
    print(f"catalog: {len(kat)} models at {url}\n")
    for host in hosts:
        entries = hand_list(host)
        if entries is None:
            print(f"{host}: no {MODELS_JSON} readable\n")
            continue
        provs = {}
        for name, mid, base in entries:
            provs.setdefault(name, {"base": base, "models": []})["models"].append(mid)

        # "Points at the gateway" is decided by HOSTNAME, not by substring: the same box
        # also runs litellm on another port, and that is a different id namespace.
        via_gw = {n: i for n, i in provs.items()
                  if urllib.parse.urlparse(i["base"]).hostname == gw.hostname
                  and urllib.parse.urlparse(i["base"]).port == gw.port}
        direct = {n: i for n, i in provs.items() if n not in via_gw}

        print(f"== {host}: {len(provs)} providers, {len(entries)} entries ==")
        print(f"-- through the gateway ({len(via_gw)}): must appear in the catalog")
        for name, info in sorted(via_gw.items()):
            missing = [m for m in info["models"] if m not in kat]
            verdict = "ok" if not missing else f"{len(missing)}/{len(info['models'])} missing"
            print(f"   {name:<24} {verdict}")
            for m in sorted(missing):
                print(f"     - {m}")
        print(f"-- straight at a backend ({len(direct)}): bypassing the gateway on purpose")
        for name, info in sorted(direct.items()):
            up = answers(info["base"])
            state = "answers" if up else ("DEAD" if up is False else "?")
            print(f"   {name:<24} {state:<8} {len(info['models'])} model(s)  {info['base'][:44]}")
        print()


if __name__ == "__main__":
    main()