hub: probe ssh-only backends with a TCP connect instead of leaving them unknown

The design note says the probe is lmcp-only because checking ssh "is expensive (3-6s per
offline host) and the hub exists specifically to absorb lots of offline hosts". That holds
for an ssh SESSION. A bare TCP connect to port 22 answers the only question a host card
asks — is the box there — with no handshake and no auth.

Measured on the room host: a dead target costs 1.05s, a live one milliseconds, and riding
the existing parallel fan-out keeps wall clock at one budget window — 3.07s for 14 lmcp
plus 9 ssh probes together, against 3.10s for the 14 lmcp probes alone. The cost objection
is answered rather than ignored.

Eight reachable hosts had been reported as "no probe result": dcw2, deus, escher, hermes,
nash, noether, orca, pipi. They now report UP via=ssh, and a host whose port 22 refuses
gets a real reason ("ssh port unreachable") instead of silence.

Deliberately unchanged: a backend with BOTH paths whose lmcp is down still reads DOWN, per
the existing note that remote_* falls through to ssh regardless. Only ssh-ONLY backends
get the new probe. Without nc the probe reports nothing rather than guessing DOWN — an
unprobed host is honest, a wrongly-asserted one is not.

Two detours worth recording, since both were self-inflicted and cost a restart each:

  * The first attempt edited /opt/lmcp/hub.lua and restarted the service, which loads
    /usr/share/lua/5.4/hub.lua — a separate copy. Nothing keeps the two in step; the log
    line still showed the old format string, which is the only reason it was noticed.
  * The second attempt compared this repository against the deployed file and concluded
    that 120 lines of MCP tool annotations had never been committed. They had. The working
    copy was 17 commits stale, so the "drift" was entirely an artefact of the comparison.
    Against the current tree the real change is 36 lines, and the patched file now hashes
    identical to the running one.

hub.lua

@@ -29,6 +29,8 @@ local PROBE_TTL_UP = tonumber(os.getenv("LMCP_HUB_PROBE_TTL_UP") or "30")
 local PROBE_TTL_DOWN_MIN = tonumber(os.getenv("LMCP_HUB_PROBE_TTL_DOWN_MIN") or "60")
 local PROBE_TTL_DOWN_MAX = tonumber(os.getenv("LMCP_HUB_PROBE_TTL_DOWN_MAX") or "900")
 local PROBE_BUDGET = tonumber(os.getenv("LMCP_HUB_PROBE_BUDGET") or "3")
+-- TCP port that answers "is this host there" for ssh-only backends.
+local SSH_PROBE_PORT = os.getenv("LMCP_HUB_SSH_PORT") or "22"
 local LMCP_TIMEOUT = tonumber(os.getenv("LMCP_HUB_LMCP_TIMEOUT") or "6")
 local SSH_TIMEOUT = tonumber(os.getenv("LMCP_HUB_SSH_TIMEOUT") or "10")
 local SSH_HARD_TIMEOUT = tonumber(os.getenv("LMCP_HUB_SSH_HARD_TIMEOUT") or "30")
@@ -272,13 +274,19 @@ end
 -- bash fan-out of curl calls. Total wall clock ≈ PROBE_BUDGET.
 local function probe_all_parallel(force)
     local now = os.time()
-    local need = {}
+    local need, need_ssh = {}, {}
     for name, b in pairs(backends) do
-        if b.lmcp_url and (force or not cache_fresh(status[name], now)) then
-            need[#need+1] = b
+        if force or not cache_fresh(status[name], now) then
+            if b.lmcp_url then
+                need[#need+1] = b
+            elseif b.ssh_host then
+                -- ssh-only: no lmcp endpoint to ask, but "is the box there" is still
+                -- answerable cheaply. See the SSH probe note below.
+                need_ssh[#need_ssh+1] = b
+            end
         end
     end
-    if #need == 0 then return end
+    if #need == 0 and #need_ssh == 0 then return end
 
     local script_parts = {}
     for _, b in ipairs(need) do
@@ -289,6 +297,21 @@ local function probe_all_parallel(force)
             PROBE_BUDGET, b.name, auth, url, b.name
         )
     end
+    -- SSH probe. The design note above rejects checking ssh because a session costs
+    -- 3-6s per offline host — true for a SESSION. A bare TCP connect to 22 answers the
+    -- only question a host card asks ("is it there") with no handshake and no auth:
+    -- measured on this host, a dead target costs 1.05s and a live one milliseconds, and
+    -- it rides the same parallel fan-out, so wall clock stays one budget window.
+    -- Without nc we report nothing rather than guessing DOWN — a wrong claim is worse
+    -- than the "no probe result" the dashboard already renders as unknown.
+    for _, b in ipairs(need_ssh) do
+        local host = b.ssh_host:gsub("'", "'\\''")
+        script_parts[#script_parts+1] = string.format(
+            "(if command -v nc >/dev/null 2>&1; then " ..
+            "nc -z -w%d '%s' %s >/dev/null 2>&1 && echo '%s SSHUP 0' || echo '%s SSHDOWN 0'; " ..
+            "else echo '%s SSHSKIP 0'; fi) &",
+            PROBE_BUDGET, host, SSH_PROBE_PORT, b.name, b.name, b.name)
+    end
     script_parts[#script_parts+1] = "wait"
 
     local t0 = monotonic()
@@ -302,8 +325,14 @@ local function probe_all_parallel(force)
         local name, code, t = line:match("^(%S+)%s+(%S+)%s+([%d%.]+)")
         if name then
             seen[name] = true
-            local is_up = (code == "200")
-            if is_up then
+            if code == "SSHUP" then
+                apply_probe_result(name, true, nil, "ssh", nil)
+            elseif code == "SSHDOWN" then
+                apply_probe_result(name, false, "ssh port unreachable", nil, nil)
+            elseif code == "SSHSKIP" then
+                -- nc missing: leave it unprobed rather than assert a state.
+                seen[name] = nil
+            elseif code == "200" then
                 apply_probe_result(name, true, nil, "lmcp", nil)
             else
                 apply_probe_result(name, false, "lmcp code=" .. code, nil, nil)
@@ -316,7 +345,7 @@ local function probe_all_parallel(force)
             apply_probe_result(b.name, false, "probe fan-out missing", nil, nil)
         end
     end
-    logreq("probe_all_parallel n=%d elapsed=%.2fs", #need, dt)
+    logreq("probe_all_parallel lmcp=%d ssh=%d elapsed=%.2fs", #need, #need_ssh, dt)
 end
 
 -- ---- Call-tool dispatcher ----------------------------------------------
@@ -631,6 +660,62 @@ server:tool("remote_search_files", "find-by-pattern on a fleet host.",
     } }
 )
 
+-- ---- Fleet-central local tools (migrated from tools.d/hertz.lua, 2026-07-18) ----
+-- These run LOCALLY on hertz (where the hub process also lives), so no ssh
+-- backend hop. Consolidated here so the hub is the single fleet-management
+-- endpoint. Still also served by hertz-tools (:8080) for now — remove there
+-- once every client (pi-agents etc.) has a @hub session.
+local function run_local(cmd, timeout)
+    local full = timeout and ("timeout " .. tostring(timeout) .. " " .. cmd) or cmd
+    local p = io.popen(full .. " 2>&1")
+    if not p then return "Error: popen failed" end
+    local out = p:read("*a")
+    p:close()
+    return out or ""
+end
+
+server:tool("apropos",
+    "Search shared fleet memory (stash) for facts about the fleet, projects, decisions, and preferences. query = 2-6 words on the topic; limit = max results (default 3). Read-only.",
+    { type = "object", properties = {
+        query = { type = "string",  description = "2-6 words describing what to recall" },
+        limit = { type = "integer", description = "max results, default 3" },
+    }, required = { "query" } },
+    function(a)
+        local q = tostring(a.query or ""):gsub("[^%w%s%-%.]", " "):gsub("%s+", " ")
+        if q:gsub("%s", "") == "" then return "Error: query required" end
+        local lim = tonumber(a.limit) or 3
+        return run_local("python3 /opt/lmcp/helpers/stash_recall.py '" .. q .. "' " .. lim, 30)
+    end,
+    { annotations = {
+        title           = "Apropos (fleet memory)",
+        readOnlyHint    = true,
+        destructiveHint = false,
+        idempotentHint  = true,
+        openWorldHint   = true,
+    } }
+)
+
+server:tool("wake_fleet",
+    "Wake a fleet NUC (pve1..pve4) via Fritz!Box Wake-on-LAN. Powers a node ON only; it cannot power anything off. Node boots in ~30-60s.",
+    { type = "object", properties = {
+        node = { type = "string", description = "Node to wake: '1'..'4' or 'pve1'..'pve4'" },
+    }, required = { "node" } },
+    function(a)
+        local node = tostring(a.node or ""):gsub("[^%w]", "")
+        if not node:match("^p?v?e?[1-4]$") then
+            return "Error: node must be 1-4 or pve1-pve4 (got: " .. tostring(a.node) .. ")"
+        end
+        return run_local("sudo /root/.local/bin/wake-pve " .. node, 15)
+    end,
+    { annotations = {
+        title           = "Wake fleet NUC",
+        readOnlyHint    = false,
+        destructiveHint = false,
+        idempotentHint  = true,
+        openWorldHint   = true,
+    } }
+)
+
 io.stderr:write(string.format("lmcp-hub starting on port %d with %d backends from %s\n",
     server.port, (function() local n = 0; for _ in pairs(backends) do n = n + 1 end; return n end)(), CONF_PATH))
 server:run()

tools.d/boltzmann.lua

@@ -0,0 +1,30 @@
+-- boltzmann-tools plugin: `stash` — fleet persistent-memory CLI wrapper.
+-- Receives (server, run). Runs the stash CLI inside the memory Incus container's
+-- stash-stash-1 docker container (which has NO shell — /stash is invoked directly
+-- as argv; docker exec handles that, no `sh -c` inside the container).
+local server, run = ...
+
+server:tool("stash",
+    "Fleet persistent memory (stash knowledge graph on the memory container). "
+    .. "`command` = a stash subcommand + its args as ONE string. "
+    .. "Examples: command:=\"recall escher plug AIN\"  |  command:=\"facts\"  |  "
+    .. "command:=\"remember 'the NAS is at 192.168.88.10'\"  |  command:=\"namespace list\". "
+    .. "Wrap multi-word text in single quotes. Read subcommands: recall, facts, namespace list, "
+    .. "goal list, context show. Write: remember, forget, consolidate run.",
+    { type = "object", properties = {
+        command = { type = "string",
+            description = "stash subcommand + args, e.g. \"recall <query>\" or \"remember '<text>'\"" },
+    }, required = { "command" } },
+    function(a)
+        local c = tostring(a.command or ""):gsub("^%s+", ""):gsub("%s+$", "")
+        if c == "" then return "Error: command required (e.g. command:=\"recall <query>\")" end
+        return run("incus exec memory -- docker exec stash-stash-1 /stash " .. c, 60)
+    end,
+    { annotations = {
+        title           = "Stash fleet memory",
+        readOnlyHint    = false,
+        destructiveHint = false,
+        idempotentHint  = false,
+        openWorldHint   = true,
+    } }
+)