aish/test_router_model.lua

-- test_router_model.lua — Phase 5 commit #1 corpus for classify_model.
-- Run from repo root: `luajit test_router_model.lua` (exit 0 on pass).

package.path = "./?.lua;./vendor/?.lua;" .. package.path
local router = require("router")

local cfg = {
    routing = {
        auto = true,
        classes = {
            code      = "deep",
            reasoning = "cloud",
            default   = nil,   -- nil → keep current
        },
    },
}

-- (text, expected_class)
local CASES = {
    -- ── code class
    { "```python\ndef foo(): pass\n```",            "code" },
    { "Traceback (most recent call last):",          "code" },
    { "got a stack trace from my server",            "code" },
    { "error: undefined reference to `foo'",         "code" },
    { "exception: file not found",                   "code" },
    { "please look at ./src/main.lua",               "code" },
    { "the issue is in ~/repos/foo/bar.py",          "code" },
    { "check /usr/lib/python3/dist-packages/x.py",   "code" },
    { "fix this:\n    if x == 0:\n        return\n    else:\n        pass\n", "code" },

    -- ── reasoning class
    { "Explain how MMAP works on Linux",             "reasoning" },
    { "why does my disk fill up so fast",            "reasoning" },
    { "how does ASLR work?",                         "reasoning" },
    { "compare ZFS and btrfs in terms of snapshots", "reasoning" },
    { "Can you explain the difference between a process and a thread in detail?", "reasoning" },
    { "I have a long question with a question mark in it that goes well past one hundred characters does it route to reasoning?", "reasoning" },

    -- ── default class — short queries, no markers
    { "hi",                                          "default" },
    { "what time is it",                             "default" },
    { "ls /tmp",                                     "default" },
    { "thanks",                                      "default" },
    { "explain",                                     "default" },   -- bare "explain" < 30 chars threshold
    { "why",                                         "default" },   -- bare "why"
    { "?",                                           "default" },   -- bare ?
    { "hello world",                                 "default" },

    -- ── edge: ambiguous — prefer false-positive into reasoning over false-negative
    { "How does it feel to be a robot? Just curious.", "reasoning" },  -- 47 chars + how does
    -- ── edge: short error mention should still be code
    { "got error: foo",                              "code" },

    -- ── edge: a non-code path-like (e.g. README.md, .txt) should NOT match
    { "see notes.md for details",                    "default" },
    { "lookup /tmp/x.txt",                           "default" },
}

local pass, fail = 0, 0
local fails = {}
for i, c in ipairs(CASES) do
    local _model, class = router.classify_model(c[1], cfg)
    if class == c[2] then
        pass = pass + 1
    else
        fail = fail + 1
        fails[#fails+1] = string.format("  [%2d] text=%q expected=%s got=%s",
            i, c[1]:sub(1, 60), c[2], tostring(class))
    end
end

print(string.format("router.classify_model: %d/%d pass", pass, pass+fail))
for _, f in ipairs(fails) do print(f) end

-- Verify model routing: code → "deep", reasoning → "cloud", default → nil
local cases_model = {
    { "Traceback",                "deep",  "code"      },
    { "Explain in detail how X works", "cloud", "reasoning" },
    { "hi",                       nil,     "default"   },
}
print()
for _, c in ipairs(cases_model) do
    local m, k = router.classify_model(c[1], cfg)
    if m == c[2] and k == c[3] then
        pass = pass + 1
    else
        fail = fail + 1
        fails[#fails+1] = string.format(
            "  model: text=%q expected (%s,%s) got (%s,%s)",
            c[1], tostring(c[2]), c[3], tostring(m), tostring(k))
    end
end

-- Verify the R-N2 default: when classes.reasoning = nil, reasoning text → no override
local cfg_safe = { routing = { auto = true, classes = { code = "deep", reasoning = nil } } }
local m, k = router.classify_model("Explain in detail how X works", cfg_safe)
if m == nil and k == "reasoning" then
    pass = pass + 1
else
    fail = fail + 1
    fails[#fails+1] = string.format(
        "  reasoning-nil-default: got (%s,%s); want (nil,reasoning)",
        tostring(m), tostring(k))
end

print(string.format("\nTOTAL: %d/%d pass", pass, pass+fail))
for _, f in ipairs(fails) do print(f) end
os.exit(fail == 0 and 0 or 1)