history: memory.jsonl store + flock (Phase 4 commit #1)

Phase 4 commit #1 per docs/PHASE4.md §12. Two file changes bundled
because R-B1 (flock for race-free single-writer enforcement) cannot
be deferred — adding it retroactively means reopening the memory
handle.

ffi/libc.lua extensions:
  - cdef flock(int fd, int op), open(...), lseek(int, long, int)
  - constants LOCK_EX=2, LOCK_NB=4, LOCK_UN=8
  - M.flock(fd, op) wrapper returning (true) on success or
    (false, errmsg) — errmsg is the strerror text so callers can
    surface "Resource temporarily unavailable" cleanly to the user.

history.lua additions (Phase 4 section appended at end):
  - M.open_memory(path) -> handle | nil, err
    Opens the file via libc.open(2) (need integer fd for flock —
    io.open's FILE* doesn't expose it), takes flock(LOCK_EX | LOCK_NB).
    Returns "memory.jsonl held by another aish process" on lock-held.
    Scans existing content for max id; caches as handle.next_id.
    Writes meta header on first creation (no id, ignored at load).
  - handle:add(kind, content, tags?, source?) -> id
    Assigns next id; appends one JSONL item with auto-timestamp.
    kind ∈ {fact, pref, context} enforced via assert.
  - handle:forget(target_id)
    Appends a tombstone {id, ts, kind:"forget", target}.
  - handle:close()
    Releases fd (flock auto-released on close).
  - M.load_memory(path) -> items_table
    Reads all lines, builds forget-target set from kind=="forget"
    entries, returns active items as an array sorted by ts desc.
    Items without id (meta header) silently dropped. Tombstones with
    non-matching targets are no-ops (N3 invariant).

Round-trip test passes:
  - open empty file → next_id=1
  - add 3 items → ids 1, 2, 3
  - forget id 2 (appends tombstone)
  - reopen → next_id correctly advances past the tombstone (=5)
  - load_memory → 2 active items (id 1 + id 3); tombstone resolved
  - lock-held detection: second open while first held → fails with
    "memory.jsonl held by another aish process" message
  - close releases the lock; reopen after release succeeds

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

ffi/libc.lua

@@ -33,6 +33,11 @@ void cfmakeraw(struct termios *tio);
 /* poll for stdin↔master multiplex in executor. */
 struct pollfd { int fd; short events; short revents; };
 int poll(struct pollfd *fds, unsigned long nfds, int timeout);
+
+/* Phase 4: advisory file locking on memory.jsonl. Single-writer
+   enforcement via LOCK_EX | LOCK_NB — fail-fast if another aish
+   process holds the lock. */
+int flock(int fd, int operation);
 ]]
 
 local C = ffi.C
@@ -154,4 +159,19 @@ function M.poll(fds_arr, nfds, timeout_ms)
     return C.poll(fds_arr, nfds, timeout_ms or -1)
 end
 
+-- ---------------------------------------------------------------- flock
+-- Advisory file locking. Phase 4 uses LOCK_EX | LOCK_NB so a second
+-- aish process opening the same memory.jsonl fails fast rather than
+-- blocking. Lock is released on fd close or process exit.
+M.LOCK_EX = 2
+M.LOCK_NB = 4
+M.LOCK_UN = 8
+
+-- Returns: true on success; false, errmsg on failure (e.g. EWOULDBLOCK
+-- when LOCK_NB is set and another holder exists).
+function M.flock(fd, op)
+    if C.flock(fd, op) == 0 then return true end
+    return false, ffi.string(C.strerror(C.__errno_location()[0]))
+end
+
 return M

history.lua

@@ -1,15 +1,21 @@
--- history.lua — persistent session log (JSONL).
+-- history.lua — persistent session log + cross-session memory store.
 -- Phase 1: append-only JSONL per session under <config.history.dir>/sessions/.
--- Phase 3 will add memory.jsonl summarization (separate from session logs).
+-- Phase 4: cross-session memory.jsonl at <config.history.dir>/memory.jsonl,
--- See docs/PHASE0.md §11 and docs/PHASE1.md §6.
+--          single-writer enforced via flock(LOCK_EX | LOCK_NB) per PHASE4 R-B1.
+-- See docs/PHASE0.md §11, docs/PHASE1.md §6, docs/PHASE4.md §4.
 
 local json = require("dkjson")
+local libc = require("ffi.libc")
+local ffi  = require("ffi")
 
 local M = {}
 
 local Session = {}
 Session.__index = Session
 
+local Memory = {}
+Memory.__index = Memory
+
 -- Best-effort mkdir -p. Failures are surfaced by io.open below. Uses
 -- single-quote escaping (Lua's %q double-quotes, which still expands $(...)
 -- and $VAR inside) so a path containing shell metacharacters doesn't trip.
@@ -127,4 +133,180 @@ function M.list_sessions(dir)
     return out
 end
 
+-- ============================================================================
+-- Phase 4: memory.jsonl — cross-session memory store.
+-- Same JSONL convention as session logs, but a single shared file rather
+-- than per-session. Single-writer enforced via flock advisory lock.
+-- See docs/PHASE4.md §2 / §4.
+-- ============================================================================
+
+-- We need an integer fd for flock. io.open returns a Lua FILE*; LuaJIT
+-- has no portable way to extract the underlying fd from that. Workaround:
+-- open via libc directly using open(2). Already exposed close() in libc;
+-- need to declare open() and read/write via the existing fd interface.
+ffi.cdef[[
+int open(const char *pathname, int flags, int mode);
+long lseek(int fd, long offset, int whence);
+]]
+
+local O_RDWR   = 2
+local O_CREAT  = 64       -- 0100 octal on Linux/glibc
+local O_APPEND = 1024     -- 02000 octal on Linux/glibc
+local SEEK_SET = 0
+local FILE_MODE = 0x180   -- 0600 octal — owner rw only
+
+-- ---------------------------------------------------------------- M.open_memory
+-- Opens memory.jsonl at `path` for append, takes an exclusive non-blocking
+-- flock on the fd, scans existing content for max id, writes a meta header
+-- if the file is new. Returns:
+--   handle, nil   on success
+--   nil,    err   on lock-held / open failure
+function M.open_memory(path)
+    ensure_dir(parent_dir(path))
+
+    -- Open via libc open(2) so we have an integer fd for flock.
+    local fd = ffi.C.open(path,
+                          bit and bit.bor(O_RDWR, O_CREAT, O_APPEND)
+                              or  (O_RDWR + O_CREAT + O_APPEND),
+                          FILE_MODE)
+    -- bit lib may not be loaded; fall back to numeric add (flags don't
+    -- overlap so OR == add here).
+    if fd < 0 then
+        return nil, "open " .. path .. " failed: "
+                    .. libc.strerror(libc.errno())
+    end
+
+    local ok, err = libc.flock(fd, libc.LOCK_EX + libc.LOCK_NB)
+    if not ok then
+        libc.close(fd)
+        return nil, "memory.jsonl held by another aish process ("
+                    .. tostring(err) .. ")"
+    end
+
+    -- Scan existing content for max id. lseek back to start, read all.
+    local max_id = 0
+    local was_empty = true
+    ffi.C.lseek(fd, 0, SEEK_SET)
+    while true do
+        -- Read in 4K chunks. Use libc.read which returns string+len.
+        local chunk, n = libc.read(fd, 4096)
+        if not chunk or n == 0 then break end
+        was_empty = false
+        -- Accumulate into a buffer; on first scan we may straddle lines.
+        -- Simple approach: keep a tail and split on newlines.
+        for line in chunk:gmatch("[^\n]+") do
+            local obj = json.decode(line)
+            if obj and obj.id and obj.id > max_id then max_id = obj.id end
+        end
+    end
+    -- Seek to end so subsequent libc.write appends.
+    ffi.C.lseek(fd, 0, 2)  -- SEEK_END
+
+    local handle = setmetatable({
+        path    = path,
+        fd      = fd,
+        next_id = max_id + 1,
+        closed  = false,
+    }, Memory)
+
+    if was_empty then
+        -- Write meta header. No id; load_memory skips lines without id.
+        handle:_write_raw({
+            meta = {
+                aish_version = "phase4",
+                created      = os.date("!%Y-%m-%dT%H:%M:%SZ"),
+            }
+        })
+    end
+
+    return handle
+end
+
+-- Internal: append one JSON line to the fd.
+function Memory:_write_raw(obj)
+    local line = json.encode(obj) .. "\n"
+    libc.write(self.fd, line)
+end
+
+-- Append a memory item. Returns the assigned id.
+function Memory:add(kind, content, tags, source)
+    assert(not self.closed, "memory:add on closed handle")
+    assert(kind == "fact" or kind == "pref" or kind == "context",
+           "memory:add: kind must be fact|pref|context (got " .. tostring(kind) .. ")")
+    assert(content and content ~= "", "memory:add: content required")
+
+    local id = self.next_id
+    self.next_id = id + 1
+    local item = {
+        id      = id,
+        ts      = os.date("!%Y-%m-%dT%H:%M:%SZ"),
+        kind    = kind,
+        content = content,
+    }
+    if tags   then item.tags   = tags   end
+    if source then item.source = source end
+    self:_write_raw(item)
+    return id
+end
+
+-- Append a tombstone for `target_id`. Idempotent at the file level; the
+-- caller (e.g. `:memory forget` meta handler) may want to check
+-- M.load_memory first to surface a "not active" status to the user (N1).
+function Memory:forget(target_id)
+    assert(not self.closed, "memory:forget on closed handle")
+    self:_write_raw({
+        id     = self.next_id,
+        ts     = os.date("!%Y-%m-%dT%H:%M:%SZ"),
+        kind   = "forget",
+        target = target_id,
+    })
+    self.next_id = self.next_id + 1
+end
+
+function Memory:close()
+    if self.closed then return end
+    -- flock is released automatically on fd close.
+    libc.close(self.fd)
+    self.fd = nil
+    self.closed = true
+end
+
+-- ---------------------------------------------------------------- M.load_memory
+-- Read all items, resolve tombstones, return active set sorted by ts desc.
+-- Items without an `id` field (e.g. the meta header) are silently dropped.
+-- Tombstones with non-matching targets are no-ops.
+-- Returns:
+--   items_table  array of {id, ts, kind, content, tags?, source?}
+--   may be empty if file doesn't exist or contains only meta/tombstones
+function M.load_memory(path)
+    local fh = io.open(path, "r")
+    if not fh then return {} end
+
+    local items = {}      -- by id
+    local forget = {}     -- set of target ids
+    for line in fh:lines() do
+        if #line > 0 then
+            local obj = json.decode(line)
+            if obj and obj.id then
+                if obj.kind == "forget" then
+                    if obj.target then forget[obj.target] = true end
+                elseif obj.kind == "fact" or obj.kind == "pref"
+                                          or obj.kind == "context" then
+                    items[obj.id] = obj
+                end
+            end
+        end
+    end
+    fh:close()
+
+    local active = {}
+    for id, item in pairs(items) do
+        if not forget[id] then active[#active + 1] = item end
+    end
+    -- Sort by ts descending (most recent first). Strings sort right when
+    -- they're ISO 8601 — ASCII order = chronological.
+    table.sort(active, function(a, b) return a.ts > b.ts end)
+    return active
+end
+
 return M