v0.5.3: hub hardening — hard ssh timeout, parallel probes, sticky DOWN cache

Three fixes addressing the recurring "hub wedges on offline backends" class
of failures (2 incidents in 24h, root-caused to single-threaded Lua + an
uninterruptible os.execute ssh call):

1. Hard wall-clock cap on ssh fallback via GNU `timeout --kill-after=2 30`.
   ConnectTimeout alone only bounds TCP connect; a half-dead sshd (auth
   stall, remote bash-s hang) used to freeze the whole event loop
   indefinitely. Configurable via LMCP_HUB_SSH_HARD_TIMEOUT. Also adds
   ServerAliveInterval=5/Count=2 so an established-but-dead tunnel dies.

2. Parallel lmcp probes for remote_list_hosts. Shells out a single bash
   fan-out of curl -m 3 calls, bounded by PROBE_BUDGET. Wall clock for a
   full 12-backend probe went from ~28 s (sum of per-host ssh connect
   timeouts) to ~3 s.

3. Probe is lmcp-only — ssh is no longer used as health check. The hub
   exists to absorb lots of offline hosts, so an expensive ssh per probe
   was the exact wrong tradeoff. Actual remote_* tool calls still fall
   through to ssh fallback when lmcp is down.

4. Sticky DOWN cache with exponential backoff: 60 → 120 → 240 → 480 →
   900 s. Prevents a sleeping fleet from burning probe budget on every
   health check. UP hosts still use 30 s TTL. Tunable via
   LMCP_HUB_PROBE_TTL_{UP,DOWN_MIN,DOWN_MAX}.

5. Per-request logging to stderr (tool, host, via, elapsed) — invisible
   before, now captured in journal for the next hang's RCA.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

examples/hub-backends.conf.example

@@ -0,0 +1,22 @@
+# lmcp hub backend registry.
+#
+# Format: whitespace-separated "name ssh_host lmcp_url token"
+# Use "-" for not-applicable fields.
+#   - Missing ssh_host (col 2 = -): no ssh fallback available for this backend
+#   - Missing lmcp_url (col 3 = -): this backend is ssh-only
+#   - Missing token   (col 4 = -): backend accepts unauth lmcp (LAN-only hosts)
+#
+# Lines starting with # are comments. Blank lines ignored.
+#
+# LXD-container caveat: if your hub lives on the LXD host and the backend is
+# a sibling LXD container, Fritz DNS often caches a stale DHCP lease for the
+# container's hostname. Hardcode the container IP here and update when it
+# rotates, or wire the .lxd stub zone into systemd-resolved.
+
+# name         ssh_host               lmcp_url                                   token
+# ---          ---                    ---                                        ---
+# boltzmann    boltzmann.fritz.box    http://boltzmann.fritz.box:8080/mcp        <64hex>
+# tesla        192.168.88.67          http://192.168.88.67:8080/mcp              -
+# nc           nc.reauktion.de        http://nc.reauktion.de:8080/mcp            <opaque>
+# pve1         pve1.fritz.box         http://pve1.fritz.box:8080/mcp             <opaque>
+# hertz        -                      http://hertz.fritz.box:8080/mcp            <opaque>

examples/lmcp-hub.service

@@ -0,0 +1,25 @@
+[Unit]
+Description=lmcp hub — fleet MCP broker
+After=network.target
+Wants=network.target
+
+[Service]
+Type=simple
+User=mfritsche
+Group=mfritsche
+# When deploying, copy hub.lua to /opt/lmcp/ or adjust the path below.
+ExecStart=/usr/bin/lua5.4 /usr/share/lua/5.4/hub.lua
+Environment=LMCP_NAME=hub-tools
+Environment=LMCP_PORT=8090
+# Backend registry: space-separated "name ssh_host lmcp_url token" per line.
+# See hub-backends.conf.example in this examples/ dir.
+Environment=LMCP_HUB_BACKENDS=/opt/herding/etc/hub-backends.conf
+# Bearer token file (key `.godparticle=<hex>`).
+Environment=LMCP_HUB_CONF=/opt/herding/etc/lmcp-hub.conf
+Restart=on-failure
+RestartSec=5
+StandardOutput=journal
+StandardError=journal
+
+[Install]
+WantedBy=multi-user.target

hub.lua

@@ -0,0 +1,580 @@
+#!/usr/bin/env lua
+-- lmcp hub — fleet-wide MCP broker.
+--
+-- One MCP endpoint that fans out to every lmcp-backed host, with an SSH
+-- fallback for hosts whose lmcp is temporarily down (or not installed).
+-- Exposes a small set of "remote_*" tools that all take a `host` arg
+-- naming the target in the backend registry.
+--
+-- Registry file (default /opt/herding/etc/hub-backends.conf):
+--   # name  ssh_host            lmcp_url                               token
+--   boltzmann boltzmann.fritz.box http://boltzmann.fritz.box:8080/mcp   <bearer>
+--   tesla     tesla               http://tesla.fritz.box:8080/mcp       -
+--   broglie   -                   http://broglie.fritz.box:8080/mcp     -
+-- Use `-` for "not applicable". Lines starting with # are comments.
+-- Missing `ssh_host`: no ssh fallback available. Missing `lmcp_url`: ssh-only.
+--
+-- SPDX-License-Identifier: MIT
+
+local dir = arg[0]:match('(.*/)') or './'
+package.path = package.path .. ';' .. dir .. '?.lua'
+local lmcp = require('lmcp')
+local json = require('json')
+local socket = require('socket')
+
+-- ---- Backend registry ---------------------------------------------------
+
+local CONF_PATH = os.getenv("LMCP_HUB_BACKENDS") or "/opt/herding/etc/hub-backends.conf"
+local PROBE_TTL_UP = tonumber(os.getenv("LMCP_HUB_PROBE_TTL_UP") or "30")
+local PROBE_TTL_DOWN_MIN = tonumber(os.getenv("LMCP_HUB_PROBE_TTL_DOWN_MIN") or "60")
+local PROBE_TTL_DOWN_MAX = tonumber(os.getenv("LMCP_HUB_PROBE_TTL_DOWN_MAX") or "900")
+local PROBE_BUDGET = tonumber(os.getenv("LMCP_HUB_PROBE_BUDGET") or "3")
+local LMCP_TIMEOUT = tonumber(os.getenv("LMCP_HUB_LMCP_TIMEOUT") or "6")
+local SSH_TIMEOUT = tonumber(os.getenv("LMCP_HUB_SSH_TIMEOUT") or "10")
+local SSH_HARD_TIMEOUT = tonumber(os.getenv("LMCP_HUB_SSH_HARD_TIMEOUT") or "30")
+local LOG_REQUESTS = (os.getenv("LMCP_HUB_LOG") or "1") ~= "0"
+
+local function logreq(fmt, ...)
+    if not LOG_REQUESTS then return end
+    io.stderr:write(string.format("[hub %s] " .. fmt .. "\n",
+        os.date("%H:%M:%S"), ...))
+    io.stderr:flush()
+end
+
+local function monotonic()
+    local ok, s = pcall(socket.gettime)
+    return ok and s or os.time()
+end
+
+local backends = {}  -- name -> { name, ssh_host, lmcp_url, token }
+local status = {}    -- name -> { up=bool, via="lmcp"|"ssh"|nil, checked=t, err=... }
+
+local function load_registry()
+    local f = io.open(CONF_PATH, "r")
+    if not f then
+        io.stderr:write("hub: no backend registry at " .. CONF_PATH .. "\n")
+        return
+    end
+    backends = {}
+    for line in f:lines() do
+        line = line:gsub("^%s+", ""):gsub("%s+$", "")
+        if line ~= "" and not line:match("^#") then
+            local parts = {}
+            for p in line:gmatch("%S+") do parts[#parts+1] = p end
+            if #parts >= 2 then
+                local name = parts[1]
+                local ssh_host = (parts[2] ~= "-" and parts[2]) or nil
+                local lmcp_url = (parts[3] ~= "-" and parts[3]) or nil
+                local token = (parts[4] ~= "-" and parts[4]) or nil
+                backends[name] = {
+                    name = name,
+                    ssh_host = ssh_host,
+                    lmcp_url = lmcp_url,
+                    token = token,
+                }
+            end
+        end
+    end
+    f:close()
+end
+
+-- ---- Outbound HTTP client (plain, no TLS) ------------------------------
+
+local function parse_url(url)
+    local scheme, host, port, path = url:match("^(%w+)://([^:/]+):?(%d*)(/?.*)$")
+    if not scheme then return nil, "bad url" end
+    port = tonumber(port) or (scheme == "https" and 443 or 80)
+    if path == "" then path = "/" end
+    return { scheme = scheme, host = host, port = port, path = path }
+end
+
+local function http_post_json(url, body, token, timeout)
+    local u, err = parse_url(url)
+    if not u then return nil, err end
+    if u.scheme ~= "http" then return nil, "hub only speaks http to backends" end
+
+    local sock = socket.tcp()
+    sock:settimeout(timeout or 6)
+    local ok, e = sock:connect(u.host, u.port)
+    if not ok then sock:close(); return nil, "connect: " .. e end
+
+    local headers = {
+        "POST " .. u.path .. " HTTP/1.1",
+        "Host: " .. u.host .. ":" .. u.port,
+        "Content-Type: application/json",
+        "Content-Length: " .. tostring(#body),
+        "Connection: close",
+    }
+    if token then
+        headers[#headers+1] = "Authorization: Bearer " .. token
+    end
+    local req = table.concat(headers, "\r\n") .. "\r\n\r\n" .. body
+    local _, se = sock:send(req)
+    if se then sock:close(); return nil, "send: " .. se end
+
+    local chunks = {}
+    while true do
+        local data, rerr, partial = sock:receive(4096)
+        if data then chunks[#chunks+1] = data
+        elseif partial and #partial > 0 then chunks[#chunks+1] = partial
+        end
+        if not data then
+            if rerr == "timeout" then sock:close(); return nil, "timeout" end
+            break
+        end
+    end
+    sock:close()
+
+    local resp = table.concat(chunks)
+    local hend = resp:find("\r\n\r\n", 1, true)
+    if not hend then return nil, "no body separator" end
+    local status_line = resp:sub(1, resp:find("\r\n", 1, true) - 1)
+    local status_code = tonumber(status_line:match("HTTP/[%d%.]+%s+(%d+)"))
+    local body_str = resp:sub(hend + 4)
+    if status_code ~= 200 then
+        return nil, "http " .. tostring(status_code) .. ": " .. body_str:sub(1, 200)
+    end
+    local ok2, parsed = pcall(json.decode, body_str)
+    if not ok2 then return nil, "bad json: " .. body_str:sub(1, 200) end
+    return parsed
+end
+
+local function jsonrpc_call(url, token, method, params)
+    local body = json.encode({
+        jsonrpc = "2.0",
+        id = os.time() * 1000 + math.random(1000, 9999),
+        method = method,
+        params = params or {},
+    })
+    local resp, err = http_post_json(url, body, token, LMCP_TIMEOUT)
+    if not resp then return nil, err end
+    if resp.error then
+        return nil, "rpc: " .. (resp.error.message or "unknown")
+    end
+    return resp.result
+end
+
+-- ---- SSH exec (shell-escape-free via bash -s on stdin) -----------------
+
+local function shell_quote(s)
+    -- Single-quote wrap, replace embedded single quotes with '\''
+    return "'" .. tostring(s):gsub("'", "'\\''") .. "'"
+end
+
+local function tmpwrite(content)
+    local path = os.tmpname()
+    local f = io.open(path, "w")
+    if not f then return nil, "tmpfile create failed" end
+    f:write(content)
+    f:close()
+    return path
+end
+
+local function ssh_run_script(ssh_host, script)
+    -- Write the script body locally, then ssh with stdin redirected from it.
+    -- bash -s reads the script from stdin. Nothing inside `script` is subject
+    -- to another shell round of expansion — this is the escape-free path.
+    local spath, perr = tmpwrite(script)
+    if not spath then return nil, perr, -1 end
+    local outpath = spath .. ".out"
+    -- Hard-cap wall time with GNU `timeout` — ssh's ConnectTimeout only
+    -- bounds TCP connect, not the session. Without this, a half-dead sshd
+    -- (auth stall, remote bash-s hang) locks the hub event loop indefinitely.
+    local cmd = string.format(
+        "timeout --kill-after=2 %d ssh -o ConnectTimeout=%d -o ServerAliveInterval=5 -o ServerAliveCountMax=2 -o BatchMode=yes -o StrictHostKeyChecking=accept-new %s 'bash -s' < %s > %s 2>&1; echo $? > %s.rc",
+        SSH_HARD_TIMEOUT, SSH_TIMEOUT, shell_quote(ssh_host), shell_quote(spath),
+        shell_quote(outpath), shell_quote(spath)
+    )
+    local t0 = monotonic()
+    os.execute(cmd)
+    local dt = monotonic() - t0
+    local rc_f = io.open(spath .. ".rc", "r")
+    local rc = -1
+    if rc_f then
+        local s = rc_f:read("*a") or ""
+        s = s:match("^(%S+)") or ""
+        rc = tonumber(s) or -1
+        rc_f:close()
+    end
+    local out_f = io.open(outpath, "r")
+    local output = out_f and out_f:read("*a") or ""
+    if out_f then out_f:close() end
+    os.remove(spath); os.remove(outpath); os.remove(spath .. ".rc")
+    -- timeout(1) exits 124 on wall-clock expiry, 137 on SIGKILL. Surface it.
+    if rc == 124 or rc == 137 then
+        logreq("ssh HARD-TIMEOUT host=%s after=%.2fs rc=%d", ssh_host, dt, rc)
+        return nil, string.format("ssh hard-timeout after %ds", SSH_HARD_TIMEOUT), rc
+    end
+    logreq("ssh host=%s elapsed=%.2fs rc=%d bytes=%d", ssh_host, dt, rc, #output)
+    return output, nil, rc
+end
+
+-- ---- Health probe ------------------------------------------------------
+--
+-- Design notes:
+--   - Probe is lmcp-only. SSH is NOT checked here: it's expensive (3-6s per
+--     offline host) and the hub exists specifically to absorb lots of
+--     offline hosts. Hosts with lmcp down but ssh up show as DOWN in the
+--     health list, but actual remote_* calls still fall through to ssh
+--     fallback correctly.
+--   - Sticky DOWN cache with exponential backoff: a DOWN host is re-probed
+--     at intervals that grow from PROBE_TTL_DOWN_MIN (default 60s) up to
+--     PROBE_TTL_DOWN_MAX (default 900s). Prevents a sleeping fleet from
+--     burning probe budget every health check.
+--   - `remote_list_hosts` uses a parallel curl fan-out for all hosts at
+--     once so wall-clock is bounded by PROBE_BUDGET (default 3s), not the
+--     sum of per-host timeouts.
+
+local function cache_fresh(s, now)
+    if not s then return false end
+    local ttl
+    if s.up then
+        ttl = PROBE_TTL_UP
+    else
+        ttl = math.min(PROBE_TTL_DOWN_MAX, PROBE_TTL_DOWN_MIN * (2 ^ (s.down_streak or 0)))
+    end
+    return (now - s.checked) < ttl
+end
+
+local function apply_probe_result(name, up, err, via, tool_count)
+    local prev = status[name]
+    local new_s = {
+        checked = os.time(),
+        up = up,
+        via = up and (via or "lmcp") or nil,
+        err = err,
+        tool_count = tool_count,
+        down_streak = up and 0 or ((prev and prev.down_streak or 0) + 1),
+    }
+    status[name] = new_s
+    return new_s
+end
+
+-- Fallback single-host probe used only when a caller explicitly needs a
+-- status refresh outside the parallel path (currently only
+-- remote_list_hosts when a single host is looked up). lmcp-only.
+local function probe(name, force)
+    local b = backends[name]
+    if not b then return { up = false, err = "unknown host" } end
+    local s = status[name]
+    if not force and cache_fresh(s, os.time()) then return s end
+    if not b.lmcp_url then
+        return apply_probe_result(name, false, "no lmcp url", nil, nil)
+    end
+    local result, err = jsonrpc_call(b.lmcp_url, b.token, "tools/list", {})
+    if result and result.tools then
+        return apply_probe_result(name, true, nil, "lmcp", #result.tools)
+    end
+    return apply_probe_result(name, false, "lmcp: " .. tostring(err), nil, nil)
+end
+
+-- Parallel lmcp probe for every backend with an lmcp_url, via a single
+-- bash fan-out of curl calls. Total wall clock ≈ PROBE_BUDGET.
+local function probe_all_parallel(force)
+    local now = os.time()
+    local need = {}
+    for name, b in pairs(backends) do
+        if b.lmcp_url and (force or not cache_fresh(status[name], now)) then
+            need[#need+1] = b
+        end
+    end
+    if #need == 0 then return end
+
+    local script_parts = {}
+    for _, b in ipairs(need) do
+        local auth = b.token and (" -H 'Authorization: Bearer " .. b.token .. "'") or ""
+        local url = b.lmcp_url:gsub("'", "'\\''")
+        script_parts[#script_parts+1] = string.format(
+            "(curl --max-time %d -s -o /dev/null -w '%s %%{http_code} %%{time_total}\\n' -X POST%s -H 'Content-Type: application/json' -d '{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"tools/list\",\"params\":{}}' '%s' || echo '%s ERR 0') &",
+            PROBE_BUDGET, b.name, auth, url, b.name
+        )
+    end
+    script_parts[#script_parts+1] = "wait"
+
+    local t0 = monotonic()
+    local p = io.popen(table.concat(script_parts, "\n"))
+    local out = p and p:read("*a") or ""
+    if p then p:close() end
+    local dt = monotonic() - t0
+
+    local seen = {}
+    for line in out:gmatch("[^\n]+") do
+        local name, code, t = line:match("^(%S+)%s+(%S+)%s+([%d%.]+)")
+        if name then
+            seen[name] = true
+            local is_up = (code == "200")
+            if is_up then
+                apply_probe_result(name, true, nil, "lmcp", nil)
+            else
+                apply_probe_result(name, false, "lmcp code=" .. code, nil, nil)
+            end
+        end
+    end
+    -- Hosts with no output (fan-out error): mark DOWN
+    for _, b in ipairs(need) do
+        if not seen[b.name] then
+            apply_probe_result(b.name, false, "probe fan-out missing", nil, nil)
+        end
+    end
+    logreq("probe_all_parallel n=%d elapsed=%.2fs", #need, dt)
+end
+
+-- ---- Call-tool dispatcher ----------------------------------------------
+
+-- `allow_ssh`: whether the tool has an SSH fallback path
+-- `ssh_impl`: function(backend, args) -> output string, or nil + error
+local function call_remote(tool, args, allow_ssh, ssh_impl)
+    local t_start = monotonic()
+    local host = args.host
+    if type(host) ~= "string" or host == "" then
+        logreq("tool=%s ERR missing-host", tool)
+        return "Error: missing `host` parameter"
+    end
+    local b = backends[host]
+    if not b then
+        logreq("tool=%s host=%s ERR unknown-host", tool, host)
+        return string.format("Error: unknown host %q (registry: %s)", host, CONF_PATH)
+    end
+
+    local errs = {}
+    local via = nil
+
+    -- Try lmcp first
+    if b.lmcp_url then
+        local args_pass = {}
+        for k, v in pairs(args) do if k ~= "host" then args_pass[k] = v end end
+        local result, err = jsonrpc_call(b.lmcp_url, b.token, "tools/call",
+            { name = tool, arguments = args_pass })
+        if result then
+            via = "lmcp"
+            logreq("tool=%s host=%s via=lmcp elapsed=%.2fs", tool, host, monotonic() - t_start)
+            -- Propagate backend's content, extract first text block
+            if type(result.content) == "table" and result.content[1] and result.content[1].text then
+                local prefix = result.isError and "[lmcp isError] " or ""
+                return prefix .. result.content[1].text
+            end
+            return json.encode(result)
+        end
+        errs[#errs+1] = "lmcp: " .. tostring(err)
+    else
+        errs[#errs+1] = "lmcp: no url configured"
+    end
+
+    -- Fallback to ssh
+    if allow_ssh and ssh_impl and b.ssh_host then
+        local out, serr = ssh_impl(b, args)
+        if out ~= nil then
+            logreq("tool=%s host=%s via=ssh elapsed=%.2fs", tool, host, monotonic() - t_start)
+            return "[via ssh fallback]\n" .. out
+        end
+        errs[#errs+1] = "ssh: " .. tostring(serr)
+    elseif allow_ssh and not b.ssh_host then
+        errs[#errs+1] = "ssh: no host configured"
+    end
+
+    logreq("tool=%s host=%s FAIL elapsed=%.2fs err=%s", tool, host, monotonic() - t_start, table.concat(errs, " | "):sub(1, 200))
+    return "Error: " .. table.concat(errs, " | ")
+end
+
+-- ---- SSH implementations for each fallback-capable tool ----------------
+
+local function ssh_shell(b, a)
+    if type(a.command) ~= "string" then return nil, "command required" end
+    local cwd_prefix = ""
+    if a.cwd and a.cwd ~= "" then
+        cwd_prefix = "cd " .. shell_quote(a.cwd) .. " && "
+    end
+    local script = cwd_prefix .. a.command .. "\n"
+    local out, err, rc = ssh_run_script(b.ssh_host, script)
+    if err then return nil, err end
+    -- ssh exits 255 when it couldn't connect/authenticate. Treat that as
+    -- fallback-failed so the caller sees the combined lmcp+ssh errors.
+    if rc == 255 then
+        local line1 = (out:match("^([^\n]+)") or ""):gsub("%s+$", "")
+        return nil, "ssh connect failed: " .. line1
+    end
+    return string.format("[rc=%d]\n%s", rc, out)
+end
+
+local function ssh_read_file(b, a)
+    if type(a.path) ~= "string" then return nil, "path required" end
+    local script = "cat -- " .. shell_quote(a.path) .. "\n"
+    local out, err, rc = ssh_run_script(b.ssh_host, script)
+    if err then return nil, err end
+    if rc ~= 0 then return nil, "cat exit " .. rc .. ": " .. out:sub(1, 200) end
+    return out
+end
+
+local function ssh_write_file(b, a)
+    if type(a.path) ~= "string" or type(a.content) ~= "string" then
+        return nil, "path and content required"
+    end
+    -- Emit a script that decodes content from base64, so no shell
+    -- interpretation of the payload is possible. Lua stdlib has no base64;
+    -- implement a minimal encoder inline.
+    local b64_alphabet = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"
+    local function b64enc(data)
+        local out = {}
+        local pad = (3 - (#data % 3)) % 3
+        local padded = data .. ("\0"):rep(pad)
+        for i = 1, #padded, 3 do
+            local a1, a2, a3 = padded:byte(i), padded:byte(i+1), padded:byte(i+2)
+            local n = a1 * 65536 + a2 * 256 + a3
+            out[#out+1] = b64_alphabet:sub(((n >> 18) & 63) + 1, ((n >> 18) & 63) + 1)
+            out[#out+1] = b64_alphabet:sub(((n >> 12) & 63) + 1, ((n >> 12) & 63) + 1)
+            out[#out+1] = b64_alphabet:sub(((n >>  6) & 63) + 1, ((n >>  6) & 63) + 1)
+            out[#out+1] = b64_alphabet:sub(((n      ) & 63) + 1, ((n      ) & 63) + 1)
+        end
+        local s = table.concat(out)
+        if pad > 0 then s = s:sub(1, -pad - 1) .. ("="):rep(pad) end
+        return s
+    end
+    local payload = b64enc(a.content)
+    local script = string.format(
+        "base64 -d > %s <<'EOF_B64_PAYLOAD'\n%s\nEOF_B64_PAYLOAD\n",
+        shell_quote(a.path), payload
+    )
+    local out, err, rc = ssh_run_script(b.ssh_host, script)
+    if err then return nil, err end
+    if rc ~= 0 then return nil, "write exit " .. rc .. ": " .. out:sub(1, 200) end
+    return string.format("Written %d bytes to %s (via ssh)", #a.content, a.path)
+end
+
+local function ssh_list_dir(b, a)
+    local path = (a and type(a.path) == "string" and a.path) or "."
+    local script = "ls -1 -- " .. shell_quote(path) .. "\n"
+    local out, err, rc = ssh_run_script(b.ssh_host, script)
+    if err then return nil, err end
+    if rc ~= 0 then return nil, "ls exit " .. rc .. ": " .. out:sub(1, 200) end
+    return out
+end
+
+local function ssh_search_files(b, a)
+    if type(a.pattern) ~= "string" then return nil, "pattern required" end
+    local path = (a.path and a.path ~= "") and a.path or "/"
+    local script = string.format(
+        "find %s -name %s 2>/dev/null | head -200\n",
+        shell_quote(path), shell_quote(a.pattern)
+    )
+    local out, err, rc = ssh_run_script(b.ssh_host, script)
+    if err then return nil, err end
+    return out
+end
+
+-- ---- Server setup ------------------------------------------------------
+
+load_registry()
+math.randomseed(os.time())
+
+local server = lmcp.new(os.getenv("LMCP_NAME") or "hub-tools", {
+    port = tonumber(os.getenv("LMCP_PORT") or arg[1]) or 8090,
+    version = "0.5.3",
+    conf = os.getenv("LMCP_HUB_CONF") or "/opt/herding/etc/lmcp-hub.conf",
+})
+
+local HOST_ARG = {
+    type = "string",
+    description = "Name of the backend host (see remote_list_hosts)",
+}
+
+server:tool("remote_list_hosts",
+    "List all registered fleet hosts and their live status (lmcp vs ssh vs down).",
+    { type = "object", properties = {
+        force = { type = "boolean", description = "Force re-probe (bypass cache)", default = false },
+    } },
+    function(a)
+        local force = a and a.force or false
+        probe_all_parallel(force)
+        local lines = {}
+        local names = {}
+        for n in pairs(backends) do names[#names+1] = n end
+        table.sort(names)
+        for _, n in ipairs(names) do
+            local b = backends[n]
+            local s = status[n] or { up = false, err = "no probe result" }
+            local paths = {}
+            if b.lmcp_url then paths[#paths+1] = "lmcp" end
+            if b.ssh_host then paths[#paths+1] = "ssh" end
+            lines[#lines+1] = string.format(
+                "%-14s  %-6s  via=%-4s  paths=%s  %s",
+                n,
+                s.up and "UP" or "DOWN",
+                tostring(s.via or "-"),
+                table.concat(paths, ","),
+                s.err and ("[" .. s.err .. "]") or ""
+            )
+        end
+        return table.concat(lines, "\n")
+    end
+)
+
+server:tool("remote_shell", "Run a shell command on a fleet host. lmcp-primary with ssh fallback.",
+    { type = "object", properties = {
+        host = HOST_ARG,
+        command = { type = "string", description = "Shell command" },
+        cwd = { type = "string", description = "Working directory" },
+        timeout = { type = "integer", description = "Timeout (seconds)", default = 120 },
+    }, required = { "host", "command" } },
+    function(a) return call_remote("shell", a, true, ssh_shell) end
+)
+
+server:tool("remote_read_file", "Read a file from a fleet host.",
+    { type = "object", properties = {
+        host = HOST_ARG,
+        path = { type = "string", description = "File path" },
+    }, required = { "host", "path" } },
+    function(a) return call_remote("read_file", a, true, ssh_read_file) end
+)
+
+server:tool("remote_write_file", "Write content to a file on a fleet host.",
+    { type = "object", properties = {
+        host = HOST_ARG,
+        path = { type = "string" },
+        content = { type = "string" },
+    }, required = { "host", "path", "content" } },
+    function(a) return call_remote("write_file", a, true, ssh_write_file) end
+)
+
+server:tool("remote_edit_file",
+    "Literal-match edit on a fleet host. **Requires backend lmcp up** — no ssh fallback.",
+    { type = "object", properties = {
+        host = HOST_ARG,
+        path = { type = "string" },
+        old_string = { type = "string" },
+        new_string = { type = "string" },
+        replace_all = { type = "boolean", default = false },
+    }, required = { "host", "path", "old_string", "new_string" } },
+    function(a) return call_remote("edit_file", a, false, nil) end
+)
+
+server:tool("remote_shell_bg",
+    "Launch a detached background command on a fleet host (Linux). Returns PID + log path immediately. Requires backend lmcp v0.5.2+.",
+    { type = "object", properties = {
+        host = HOST_ARG,
+        command = { type = "string", description = "Shell command" },
+        cwd = { type = "string" },
+        log = { type = "string", description = "Log file path" },
+    }, required = { "host", "command" } },
+    function(a) return call_remote("shell_bg", a, false, nil) end
+)
+
+server:tool("remote_list_dir", "List directory entries on a fleet host.",
+    { type = "object", properties = {
+        host = HOST_ARG,
+        path = { type = "string", default = "." },
+    }, required = { "host" } },
+    function(a) return call_remote("list_dir", a, true, ssh_list_dir) end
+)
+
+server:tool("remote_search_files", "find-by-pattern on a fleet host.",
+    { type = "object", properties = {
+        host = HOST_ARG,
+        pattern = { type = "string" },
+        path = { type = "string", default = "/" },
+    }, required = { "host", "pattern" } },
+    function(a) return call_remote("search_files", a, true, ssh_search_files) end
+)
+
+io.stderr:write(string.format("lmcp-hub starting on port %d with %d backends from %s\n",
+    server.port, (function() local n = 0; for _ in pairs(backends) do n = n + 1 end; return n end)(), CONF_PATH))
+server:run()

scripts/lmcp-install-macos.sh

@@ -1,10 +1,11 @@
 #!/bin/bash
 # Install lmcp on macOS via Homebrew.
 #
-# Idempotent. Pulls lua + luasocket from brew, copies lmcp library files
-# into brew's share/lua/5.4/ tree, mints a Bearer token (or reuses an
-# existing one at $(brew --prefix)/etc/lmcp/token), installs a LaunchAgent,
-# starts the service, and prints the token for Claude Code MCP config.
+# Pins to lua@5.4 (keg-only brew formula) to keep library paths aligned
+# with the rest of the fleet (Arch/ALARM, Debian). Installs luasocket via
+# luarocks into the user-local rocks tree (~/.luarocks/) and bakes the
+# required LUA_PATH / LUA_CPATH into the LaunchAgent plist so the service
+# can `require 'socket'`.
 #
 # Usage (from lmcp repo root):
 #   ./scripts/lmcp-install-macos.sh
@@ -16,37 +17,55 @@
 
 set -euo pipefail
 
-PREFIX=$(brew --prefix)
-REPO=${REPO:-$(cd "$(dirname "$0")/.." && pwd)}
-LABEL="de.reauktion.marfrit.lmcp"
-PLIST="$HOME/Library/LaunchAgents/$LABEL.plist"
-TOKEN_FILE="$PREFIX/etc/lmcp/token"
-PORT="${LMCP_PORT:-8080}"
-NAME="${LMCP_NAME:-$(hostname -s)-tools}"
+# brew is only on PATH in login shells by default; source its env explicitly
+# so this works under a plain non-login bash too.
+if ! command -v brew >/dev/null 2>&1; then
+    for candidate in /opt/homebrew/bin/brew /usr/local/bin/brew; do
+        if [ -x "$candidate" ]; then
+            eval "$("$candidate" shellenv)"
+            break
+        fi
+    done
+fi
+command -v brew >/dev/null 2>&1 || { echo "error: Homebrew not found"; exit 1; }
 
+REPO=${REPO:-$(cd "$(dirname "$0")/.." && pwd)}
 for f in lmcp.lua json.lua server.lua example_server.lua; do
     [ -f "$REPO/$f" ] || { echo "error: $REPO/$f not found — run from lmcp repo root or set REPO="; exit 1; }
 done
 
-echo "==> brew install lua + luasocket"
-brew install --quiet lua luasocket
+echo "==> brew install lua@5.4 luarocks"
+brew install --quiet lua@5.4 luarocks
 
-LUA="$PREFIX/bin/lua"
-[ -x "$LUA" ] || { echo "error: $LUA not executable after brew install"; exit 1; }
+LUA54_PREFIX=$(brew --prefix lua@5.4)
+LUA54=$LUA54_PREFIX/bin/lua5.4
+BREW_PREFIX=$(brew --prefix)
+[ -x "$LUA54" ] || { echo "error: $LUA54 missing after brew install"; exit 1; }
 
-echo "==> install library files into $PREFIX/share/lua/5.4/"
-install -d "$PREFIX/share/lua/5.4"
-install -m 644 "$REPO/lmcp.lua"   "$PREFIX/share/lua/5.4/lmcp.lua"
-install -m 644 "$REPO/json.lua"   "$PREFIX/share/lua/5.4/json.lua"
-install -m 644 "$REPO/server.lua" "$PREFIX/share/lua/5.4/server.lua"
-install -m 755 "$REPO/example_server.lua" "$PREFIX/bin/lmcp-example"
+echo "==> luarocks install luasocket (user-local, pinned to lua@5.4)"
+luarocks --lua-version 5.4 --lua-dir "$LUA54_PREFIX" install --local luasocket
 
-# Token: retain existing, otherwise mint 32 bytes of hex.
+LR_PATH=$(luarocks --lua-version 5.4 --lua-dir "$LUA54_PREFIX" path --lr-path)
+LR_CPATH=$(luarocks --lua-version 5.4 --lua-dir "$LUA54_PREFIX" path --lr-cpath)
+DEFAULT_LUA_P="$BREW_PREFIX/share/lua/5.4/?.lua;$BREW_PREFIX/share/lua/5.4/?/init.lua;$BREW_PREFIX/lib/lua/5.4/?.lua;$BREW_PREFIX/lib/lua/5.4/?/init.lua;./?.lua;./?/init.lua"
+DEFAULT_LUA_CP="$BREW_PREFIX/lib/lua/5.4/?.so;./?.so"
+FULL_LUA_P="$DEFAULT_LUA_P;$LR_PATH"
+FULL_LUA_CP="$DEFAULT_LUA_CP;$LR_CPATH"
+
+echo "==> install library files into $BREW_PREFIX/share/lua/5.4/"
+install -d "$BREW_PREFIX/share/lua/5.4"
+install -m 644 "$REPO/lmcp.lua"   "$BREW_PREFIX/share/lua/5.4/lmcp.lua"
+install -m 644 "$REPO/json.lua"   "$BREW_PREFIX/share/lua/5.4/json.lua"
+install -m 644 "$REPO/server.lua" "$BREW_PREFIX/share/lua/5.4/server.lua"
+install -m 755 "$REPO/example_server.lua" "$BREW_PREFIX/bin/lmcp-example"
+
+# Token: retain existing, mint new otherwise
+TOKEN_FILE="$BREW_PREFIX/etc/lmcp/token"
+install -d -m 755 "$BREW_PREFIX/etc/lmcp"
 if [ -r "$TOKEN_FILE" ]; then
     TOKEN=$(cat "$TOKEN_FILE")
     echo "==> reusing token from $TOKEN_FILE"
 else
-    install -d -m 700 "$(dirname "$TOKEN_FILE")"
     TOKEN=$(openssl rand -hex 32)
     umask 077
     printf '%s\n' "$TOKEN" > "$TOKEN_FILE"
@@ -54,6 +73,11 @@ else
     echo "==> minted new token, stored at $TOKEN_FILE (0600)"
 fi
 
+LABEL="de.reauktion.marfrit.lmcp"
+PLIST="$HOME/Library/LaunchAgents/$LABEL.plist"
+PORT="${LMCP_PORT:-8080}"
+NAME="${LMCP_NAME:-$(hostname -s)-tools}"
+
 echo "==> write LaunchAgent $PLIST"
 mkdir -p "$HOME/Library/LaunchAgents"
 cat > "$PLIST" <<PLIST
@@ -65,8 +89,8 @@ cat > "$PLIST" <<PLIST
     <string>$LABEL</string>
     <key>ProgramArguments</key>
     <array>
-        <string>$LUA</string>
-        <string>$PREFIX/share/lua/5.4/server.lua</string>
+        <string>$LUA54</string>
+        <string>$BREW_PREFIX/share/lua/5.4/server.lua</string>
     </array>
     <key>EnvironmentVariables</key>
     <dict>
@@ -76,6 +100,10 @@ cat > "$PLIST" <<PLIST
         <string>$NAME</string>
         <key>LMCP_TOKEN</key>
         <string>$TOKEN</string>
+        <key>LUA_PATH</key>
+        <string>$FULL_LUA_P</string>
+        <key>LUA_CPATH</key>
+        <string>$FULL_LUA_CP</string>
     </dict>
     <key>RunAtLoad</key>
     <true/>
@@ -94,12 +122,12 @@ echo "==> (re)load LaunchAgent"
 launchctl unload "$PLIST" 2>/dev/null || true
 launchctl load "$PLIST"
 
-sleep 1
+sleep 2
 echo "==> smoke test (unauth expected 401, Bearer expected 200)"
-unauth=$(curl -s -o /dev/null -w '%{http_code}' -X POST "http://127.0.0.1:$PORT/mcp" \
+unauth=$(curl -s -o /dev/null -w '%{http_code}' --max-time 3 -X POST "http://127.0.0.1:$PORT/mcp" \
      -H 'Content-Type: application/json' \
      -d '{"jsonrpc":"2.0","id":1,"method":"tools/list"}' || echo "000")
-auth=$(curl -s -X POST "http://127.0.0.1:$PORT/mcp" \
+auth=$(curl -s --max-time 3 -X POST "http://127.0.0.1:$PORT/mcp" \
      -H "Authorization: Bearer $TOKEN" \
      -H 'Content-Type: application/json' \
      -d '{"jsonrpc":"2.0","id":1,"method":"tools/list"}' || true)
@@ -121,7 +149,7 @@ Add to Claude Code ~/.claude.json on the client machine:
 
     "$NAME": {
         "type": "http",
-        "url": "http://$(hostname -s).local:$PORT/mcp",
+        "url": "http://$(hostname -s).fritz.box:$PORT/mcp",
         "headers": { "Authorization": "Bearer $TOKEN" }
     }
 

server.lua

@@ -170,6 +170,49 @@ server:tool("shell", "Execute a shell command.", {
     return run(cmd, a.timeout or 120)
 end)
 
+server:tool("shell_bg",
+    "Fire-and-forget shell command (Linux-only). Fully detaches via setsid+nohup+stdio-redirect and returns immediately with PID and log path. Use for daemons that must outlive the lmcp request.",
+    {
+        type = "object",
+        properties = {
+            command = { type = "string", description = "Shell command to launch" },
+            cwd     = { type = "string", description = "Working directory" },
+            log     = { type = "string", description = "Log file (stdout+stderr). Default: /tmp/lmcp-bg-<ts>-<rand>.log" },
+        },
+        required = { "command" },
+    },
+    function(a)
+        if WINDOWS then
+            return "Error: shell_bg is Linux-only (Windows Start-Process equivalent TBD)"
+        end
+        if type(a.command) ~= "string" or a.command == "" then
+            return "Error: command required"
+        end
+        local log = a.log
+        if not log or log == "" then
+            log = string.format("/tmp/lmcp-bg-%d-%d.log", os.time(), math.random(1000, 9999))
+        end
+        local pid_file = log .. ".pid"
+        local inner = a.command
+        if a.cwd and a.cwd ~= "" then
+            inner = "cd '" .. a.cwd:gsub("'", "'\\''") .. "' && " .. inner
+        end
+        local sq = function(s) return "'" .. s:gsub("'", "'\\''") .. "'" end
+        local full = string.format(
+            "setsid nohup sh -c %s </dev/null >%s 2>&1 & echo $! > %s",
+            sq(inner), sq(log), sq(pid_file)
+        )
+        os.execute(full)
+        local f = io.open(pid_file, 'r')
+        local pid = "?"
+        if f then
+            pid = (f:read('*a') or ""):match("(%d+)") or "?"
+            f:close()
+            os.remove(pid_file)
+        end
+        return string.format("launched pid=%s log=%s", pid, log)
+    end)
+
 server:tool("read_file", "Read a file.", {
     type = "object",
     properties = { path = { type = "string" } },
@@ -272,7 +315,10 @@ server:tool("search_files", "Search for files by pattern.", {
     if WINDOWS then
         return run('dir /b /s "' .. path .. '\\' .. a.pattern .. '"', 30)
     else
-        return run("find '" .. path:gsub("'", "'\\''") .. "' -name '" .. a.pattern:gsub("'", "'\\''") .. "' 2>/dev/null", 30)
+        -- -L: follow symlinks on the start path. macOS BSD find otherwise
+        -- silently emits nothing when the start path is itself a symlink
+        -- (common on Homebrew, e.g. /usr/local/share/lua -> Cellar/…/share/lua).
+        return run("find -L '" .. path:gsub("'", "'\\''") .. "' -name '" .. a.pattern:gsub("'", "'\\''") .. "' 2>/dev/null", 30)
     end
 end)