shell: kill process group on timeout/cancel (no orphans) + shell_bg job registry + kill_job/list_jobs

Root cause of the shell->shell_bg thrash: run() backgrounded POSIX commands
with a bare & and never captured the pid, so on timeout it returned an error
while the children kept running. Now: setsid (own process group), capture the
leader pid, SIGTERM+SIGKILL the whole group on timeout/cancel, and a message
telling the model it was killed + to use shell_bg. Plus shell_bg registers jobs
to /tmp/lmcp-bg-jobs.tsv and new kill_job/list_jobs let a runaway job be reaped
without a reboot.

helpers/stash_recall.py

@@ -0,0 +1,29 @@
+import urllib.request, json, threading, queue, sys
+BASE="http://192.168.88.184:8080"
+q=queue.Queue()
+def sse():
+    try:
+        r=urllib.request.urlopen(BASE+"/sse", timeout=30)
+        for raw in r:
+            s=raw.decode(errors="replace").strip()
+            if s.startswith("data:"): q.put(s[5:].strip())
+    except Exception as e: q.put("ERR:"+str(e))
+threading.Thread(target=sse,daemon=True).start()
+try:
+    ep=q.get(timeout=10)
+    if ep.startswith("ERR:"): print("stash unreachable:",ep); sys.exit(1)
+    purl=BASE+ep if ep.startswith("/") else ep
+    def post(o):
+        urllib.request.urlopen(urllib.request.Request(purl,data=json.dumps(o).encode(),headers={"Content-Type":"application/json"}),timeout=15).read()
+    post({"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2024-11-05","capabilities":{},"clientInfo":{"name":"apropos","version":"1"}}})
+    q.get(timeout=10)
+    post({"jsonrpc":"2.0","method":"notifications/initialized"})
+    query=sys.argv[1] if len(sys.argv)>1 else ""
+    limit=int(sys.argv[2]) if len(sys.argv)>2 else 3
+    post({"jsonrpc":"2.0","id":2,"method":"tools/call","params":{"name":"recall","arguments":{"query":query,"limit":limit}}})
+    d=json.loads(q.get(timeout=25))
+    txt=d.get("result",{}).get("content",[{}])[0].get("text","[]")
+    facts=json.loads(txt)
+    if not facts: print("(no memory found for: %s)"%query); sys.exit(0)
+    for f in facts: print("- %s (score %.2f)" % (f.get("content","").strip(), f.get("score",0)))
+except Exception as e: print("recall error:",e); sys.exit(1)

server.lua

@@ -174,22 +174,39 @@ local function run(cmd, timeout_sec)
         end
         return output and output ~= "" and output or "(no output)"
     else
-        -- POSIX: use shell backgrounding + wait with timeout
-        -- sh -c '(cmd > out 2>&1; echo $? > done) &' then poll
+        -- POSIX: run in its OWN session/process group (setsid) so a
+        -- timeout or cancel can kill the WHOLE tree instead of orphaning
+        -- backgrounded children (the classic "shell timed out, children
+        -- kept thrashing" bug). $! is the setsid leader pid == pgid.
+        local pid_file = base .. ".pid"
         local sh_cmd = string.format(
             "(%s) > '%s' 2>&1; echo $? > '%s'",
             cmd, out_file, done_file
         )
-        os.execute("sh -c '" .. sh_cmd:gsub("'", "'\\''") .. "' &")
+        os.execute("setsid sh -c '" .. sh_cmd:gsub("'", "'\\''")
+            .. "' & echo $! > '" .. pid_file .. "'")
+        local pgid = (read_file(pid_file) or ""):match("(%d+)")
+        remove_silent(pid_file)
 
         local completed = poll_loop()
+
+        -- Timeout or cancel -> kill the entire process group. No orphans.
+        if not completed and pgid then
+            os.execute("kill -TERM -" .. pgid .. " 2>/dev/null")
+            sleep_ms(300)
+            os.execute("kill -KILL -" .. pgid .. " 2>/dev/null")
+        end
+
         local output = read_file(out_file)
         remove_silent(out_file)
         remove_silent(done_file)
 
         if not completed then
-            if cancelled then return "(cancelled)" end
-            return output or ("Error: command timed out after " .. timeout_sec .. "s")
+            if cancelled then return "(cancelled -- process group killed)" end
+            return (output and output ~= "" and (output .. "\n") or "")
+                .. "Error: command timed out after " .. timeout_sec
+                .. "s -- the process group was KILLED (nothing is still running). "
+                .. "For a long-running command, re-run it with shell_bg."
         end
         return output and output ~= "" and output or "(no output)"
     end
@@ -283,7 +300,15 @@ server:tool("shell_bg",
             f:close()
             os.remove(pid_file)
         end
-        return string.format("launched pid=%s log=%s", pid, log)
+        -- register so list_jobs/kill_job can see and reap it (no more reboots)
+        if pid ~= "?" then
+            local reg = io.open("/tmp/lmcp-bg-jobs.tsv", "a")
+            if reg then
+                reg:write(pid.."\t"..log.."\t"..os.date("%Y-%m-%dT%H:%M:%S").."\t"..inner:gsub("[\t\n]"," ").."\n")
+                reg:close()
+            end
+        end
+        return string.format("launched pid=%s log=%s (kill with kill_job pid=%s)", pid, log, pid)
     end, {
         annotations = {
             title           = "Run shell (background)",
@@ -294,6 +319,42 @@ server:tool("shell_bg",
         },
     })
 
+server:tool("kill_job",
+    "Kill a runaway background job by PID. SIGKILLs the whole process group of a shell_bg/setsid job so no children survive. Use when a background job is thrashing a machine.",
+    { type = "object", properties = { pid = { type = "integer", description = "PID from shell_bg / list_jobs" } }, required = { "pid" } },
+    function(a)
+        if WINDOWS then return "Error: kill_job is Linux-only" end
+        local pid = tostring(a.pid or ""):match("(%d+)")
+        if not pid then return "Error: numeric pid required" end
+        os.execute("kill -KILL -"..pid.." 2>/dev/null; kill -KILL "..pid.." 2>/dev/null")
+        sleep_ms(200)
+        local alive = os.execute("kill -0 "..pid.." 2>/dev/null")
+        if alive == true or alive == 0 then return "pid "..pid.." may still be alive (uninterruptible?)" end
+        return "killed pid "..pid.." (process group)"
+    end,
+    { annotations = { title = "Kill background job", destructiveHint = true } })
+
+server:tool("list_jobs",
+    "List background jobs started via shell_bg and whether each is still running. Use to find runaway jobs to kill_job.",
+    { type = "object", properties = {} },
+    function()
+        if WINDOWS then return "Error: list_jobs is Linux-only" end
+        local reg = io.open("/tmp/lmcp-bg-jobs.tsv", "r")
+        if not reg then return "(no background jobs recorded)" end
+        local out = {}
+        for line in reg:lines() do
+            local pid, log, ts, cmd = line:match("^(%d+)\t([^\t]*)\t([^\t]*)\t(.*)$")
+            if pid then
+                local alive = os.execute("kill -0 "..pid.." 2>/dev/null")
+                local st = (alive == true or alive == 0) and "RUNNING" or "done"
+                table.insert(out, string.format("pid=%s [%s] %s log=%s\n    %s", pid, st, ts, log, (cmd or ""):sub(1,100)))
+            end
+        end
+        reg:close()
+        return #out>0 and table.concat(out, "\n") or "(no background jobs recorded)"
+    end,
+    { annotations = { title = "List background jobs", readOnlyHint = true } })
+
 server:tool("read_file", "Read a file.", {
     type = "object",
     properties = { path = { type = "string" } },

tools.d/hertz.lua

@@ -254,3 +254,39 @@ server:tool("mediagrab",
         return run("incus exec doppler -- python3 /opt/mediagrab/mediagrab.py "
             .. a.command, 120)
     end)
+
+-- ---- Fleet wake (pipi: wake-only, no power-off) ----
+server:tool("wake_fleet",
+    "Wake a fleet NUC (pve1..pve4) via Fritz!Box Wake-on-LAN. Powers a node ON only; it cannot power anything off. Node boots in ~30-60s.",
+    {
+        type = "object",
+        properties = {
+            node = { type = "string", description = "Node to wake: '1'..'4' or 'pve1'..'pve4'" },
+        },
+        required = { "node" },
+    },
+    function(a)
+        local node = tostring(a.node or ""):gsub("[^%w]", "")
+        if not node:match("^p?v?e?[1-4]$") then
+            return "Error: node must be 1-4 or pve1-pve4 (got: " .. tostring(a.node) .. ")"
+        end
+        return run("sudo /root/.local/bin/wake-pve " .. node .. " 2>&1", 15)
+    end)
+
+-- ---- apropos: lean facade for stash recall (read-only memory) ----
+server:tool("apropos",
+    "Search shared fleet memory (stash) for facts about the fleet, projects, decisions, and preferences. query = 2-6 words on the topic; limit = max results (default 3). Read-only.",
+    {
+        type = "object",
+        properties = {
+            query = { type = "string",  description = "2-6 words describing what to recall" },
+            limit = { type = "integer", description = "max results, default 3" },
+        },
+        required = { "query" },
+    },
+    function(a)
+        local q = tostring(a.query or ""):gsub("[^%w%s%-%.]", " "):gsub("%s+", " ")
+        if q:gsub("%s","") == "" then return "Error: query required" end
+        local lim = tonumber(a.limit) or 3
+        return run("python3 /opt/lmcp/helpers/stash_recall.py '" .. q .. "' " .. lim, 30)
+    end)