From b30212af0f0a110faa30180b677a42de1bce54ee Mon Sep 17 00:00:00 2001 From: Markus Fritsche Date: Sat, 16 May 2026 23:01:21 +0000 Subject: [PATCH] safety + repl: opts.category for Norris + probe (Phase 7 commit #4) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes the last two broker call sites that flow through safety.lua. Together with commits #1-#3, all 7 broker call sites in aish now attribute usage to the cost accumulator under the right category. Changes: safety.lua: - llm_probe (the YES/NO destructive checker) — broker.chat call gains opts.category = "probe". Captures (text, usage) via (reply, second) and, when opts.on_usage is provided AND the call succeeded, routes second through opts.on_usage(model, category, payload). N4 signature chain: opts already flowed through llm_second_opinion -> M.is_destructive from #52's work; opts.on_usage rides along naturally with no further signature change. - M.norris_step (Norris main broker round-trip): * opts to broker.chat_stream gains category = "norris" * probe_opts (passed to is_destructive inside the loop) gains on_usage = helpers.on_usage so the LLM probe's cost lands under "probe" too * on_delta wrapper adds elseif kind == "usage" branch that calls helpers.on_usage(payload.model, payload.category, payload). Coexists cleanly with the existing text (rehydrator) and tool_call branches. repl.lua: - Norris helpers table gains on_usage = _record_usage. The R5 central chokepoint (commit #3) does the warn-threshold check AND ctx:add_usage atomically. - :safety check meta's probe_opts always carries on_usage now (independently of whether secrets_session is set). secrets-aware scrub_msgs/rehydrate added conditionally as before. E2E verified against live broker (safety.llm_model = "cloud"): - :safety check ls -la /tmp -> 2 cloud probe calls - "[aish] session cost $0.000128 has crossed warn_at_dollars=$0.000100" - probe category visible in accumulator (would appear in :cost detail once commit #5 ships the meta). Regression: test_safety 87/87, test_router_model 31/31, repl loads. Co-Authored-By: Claude Opus 4.7 (1M context) --- repl.lua | 23 +++++++++++++++-------- safety.lua | 37 +++++++++++++++++++++++++++++++------ 2 files changed, 46 insertions(+), 14 deletions(-) diff --git a/repl.lua b/repl.lua index 23508f8..311965b 100644 --- a/repl.lua +++ b/repl.lua @@ -1286,6 +1286,11 @@ function M.run(config) and secrets.streaming_rehydrator(secrets_session) or nil end, + -- Phase 7: hand the central usage chokepoint to safety.lua. + -- safety.norris_step routes the Norris main broker's usage + -- here under category="norris"; safety.is_destructive's LLM + -- probe routes via opts.on_usage under category="probe". + on_usage = _record_usage, } local step_n = 1 @@ -1726,15 +1731,17 @@ function M.run(config) -- :safety check --no-llm if added in v2. -- Issue #52: thread secrets scrub/rehydrate so the probe -- model sees placeholders for any secrets in `cmd`. - local probe_opts + -- Phase 7: also thread on_usage so the probe's cost + -- lands in the accumulator under category="probe". + local probe_opts = { on_usage = _record_usage } if secrets_session then - probe_opts = { - scrub_msgs = function(msgs, mode_cfg) - return scrub_messages(msgs, - secrets_mode_for(mode_cfg or active_cfg)) - end, - rehydrate = function(t) return secrets_session:rehydrate(t) end, - } + probe_opts.scrub_msgs = function(msgs, mode_cfg) + return scrub_messages(msgs, + secrets_mode_for(mode_cfg or active_cfg)) + end + probe_opts.rehydrate = function(t) + return secrets_session:rehydrate(t) + end end local hit, reason = safety.is_destructive(cmd, config, probe_opts) if hit then diff --git a/safety.lua b/safety.lua index 4a5f253..4076c25 100644 --- a/safety.lua +++ b/safety.lua @@ -188,11 +188,21 @@ local function llm_probe(model_cfg, system, cmd, opts) if opts and opts.scrub_msgs then msgs = opts.scrub_msgs(msgs, model_cfg) end - local reply, err = broker.chat(model_cfg, msgs, - { max_tokens = 4, timeout_ms = PROBE_TIMEOUT_MS }) + -- Phase 7: opts.category = "probe" tags the usage in the + -- accumulator so :cost detail surfaces probe spend separately. + -- broker.chat returns (text, usage) on success; capture as + -- (reply, second) and branch on reply nil-ness. + local reply, second = broker.chat(model_cfg, msgs, + { max_tokens = 4, timeout_ms = PROBE_TIMEOUT_MS, category = "probe" }) if not reply then -- Broker failure → safe default: treat as YES (destructive) - return "YES_FAILSAFE", err + return "YES_FAILSAFE", second + end + -- Phase 7 (N4): route the usage payload through opts.on_usage if + -- the caller wired one (repl.lua's _record_usage when secrets/ + -- cost are configured). + if second and opts and opts.on_usage then + opts.on_usage(second.model, second.category, second) end if opts and opts.rehydrate then reply = opts.rehydrate(reply) end local upper = reply:upper() @@ -344,9 +354,17 @@ function M.norris_step(ctx, model_cfg, helpers, opts) local msgs = ctx:to_messages() if helpers.scrub_msgs then msgs = helpers.scrub_msgs(msgs, model_cfg) end local rehydrator = helpers.streaming_rehydrator and helpers.streaming_rehydrator() or nil + -- Phase 7: thread on_usage callback into the LLM probe via + -- probe_opts so destructive-check costs land in the accumulator + -- under the "probe" category. helpers.on_usage is repl.lua's + -- _record_usage (the central chokepoint with warn-threshold check). local probe_opts = nil - if helpers.scrub_msgs or helpers.rehydrate then - probe_opts = { scrub_msgs = helpers.scrub_msgs, rehydrate = helpers.rehydrate } + if helpers.scrub_msgs or helpers.rehydrate or helpers.on_usage then + probe_opts = { + scrub_msgs = helpers.scrub_msgs, + rehydrate = helpers.rehydrate, + on_usage = helpers.on_usage, + } end local text_parts = {} @@ -361,9 +379,16 @@ function M.norris_step(ctx, model_cfg, helpers, opts) end elseif kind == "tool_call" then tool_calls_seen[#tool_calls_seen + 1] = payload + elseif kind == "usage" then + -- Phase 7: route Norris's own broker usage to the + -- accumulator via helpers.on_usage. R5 chokepoint + -- (_record_usage) is what's wired in. + if helpers.on_usage then + helpers.on_usage(payload.model, payload.category, payload) + end end end, - { tools = helpers.tools_schema() }) + { tools = helpers.tools_schema(), category = "norris" }) if rehydrator then local tail = rehydrator:flush() if tail ~= "" then