history: memory.jsonl store + flock (Phase 4 commit #1)
Phase 4 commit #1 per docs/PHASE4.md §12. Two file changes bundled because R-B1 (flock for race-free single-writer enforcement) cannot be deferred — adding it retroactively means reopening the memory handle. ffi/libc.lua extensions: - cdef flock(int fd, int op), open(...), lseek(int, long, int) - constants LOCK_EX=2, LOCK_NB=4, LOCK_UN=8 - M.flock(fd, op) wrapper returning (true) on success or (false, errmsg) — errmsg is the strerror text so callers can surface "Resource temporarily unavailable" cleanly to the user. history.lua additions (Phase 4 section appended at end): - M.open_memory(path) -> handle | nil, err Opens the file via libc.open(2) (need integer fd for flock — io.open's FILE* doesn't expose it), takes flock(LOCK_EX | LOCK_NB). Returns "memory.jsonl held by another aish process" on lock-held. Scans existing content for max id; caches as handle.next_id. Writes meta header on first creation (no id, ignored at load). - handle:add(kind, content, tags?, source?) -> id Assigns next id; appends one JSONL item with auto-timestamp. kind ∈ {fact, pref, context} enforced via assert. - handle:forget(target_id) Appends a tombstone {id, ts, kind:"forget", target}. - handle:close() Releases fd (flock auto-released on close). - M.load_memory(path) -> items_table Reads all lines, builds forget-target set from kind=="forget" entries, returns active items as an array sorted by ts desc. Items without id (meta header) silently dropped. Tombstones with non-matching targets are no-ops (N3 invariant). Round-trip test passes: - open empty file → next_id=1 - add 3 items → ids 1, 2, 3 - forget id 2 (appends tombstone) - reopen → next_id correctly advances past the tombstone (=5) - load_memory → 2 active items (id 1 + id 3); tombstone resolved - lock-held detection: second open while first held → fails with "memory.jsonl held by another aish process" message - close releases the lock; reopen after release succeeds Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -33,6 +33,11 @@ void cfmakeraw(struct termios *tio);
|
||||
/* poll for stdin↔master multiplex in executor. */
|
||||
struct pollfd { int fd; short events; short revents; };
|
||||
int poll(struct pollfd *fds, unsigned long nfds, int timeout);
|
||||
|
||||
/* Phase 4: advisory file locking on memory.jsonl. Single-writer
|
||||
enforcement via LOCK_EX | LOCK_NB — fail-fast if another aish
|
||||
process holds the lock. */
|
||||
int flock(int fd, int operation);
|
||||
]]
|
||||
|
||||
local C = ffi.C
|
||||
@@ -154,4 +159,19 @@ function M.poll(fds_arr, nfds, timeout_ms)
|
||||
return C.poll(fds_arr, nfds, timeout_ms or -1)
|
||||
end
|
||||
|
||||
-- ---------------------------------------------------------------- flock
|
||||
-- Advisory file locking. Phase 4 uses LOCK_EX | LOCK_NB so a second
|
||||
-- aish process opening the same memory.jsonl fails fast rather than
|
||||
-- blocking. Lock is released on fd close or process exit.
|
||||
M.LOCK_EX = 2
|
||||
M.LOCK_NB = 4
|
||||
M.LOCK_UN = 8
|
||||
|
||||
-- Returns: true on success; false, errmsg on failure (e.g. EWOULDBLOCK
|
||||
-- when LOCK_NB is set and another holder exists).
|
||||
function M.flock(fd, op)
|
||||
if C.flock(fd, op) == 0 then return true end
|
||||
return false, ffi.string(C.strerror(C.__errno_location()[0]))
|
||||
end
|
||||
|
||||
return M
|
||||
|
||||
+185
-3
@@ -1,15 +1,21 @@
|
||||
-- history.lua — persistent session log (JSONL).
|
||||
-- history.lua — persistent session log + cross-session memory store.
|
||||
-- Phase 1: append-only JSONL per session under <config.history.dir>/sessions/.
|
||||
-- Phase 3 will add memory.jsonl summarization (separate from session logs).
|
||||
-- See docs/PHASE0.md §11 and docs/PHASE1.md §6.
|
||||
-- Phase 4: cross-session memory.jsonl at <config.history.dir>/memory.jsonl,
|
||||
-- single-writer enforced via flock(LOCK_EX | LOCK_NB) per PHASE4 R-B1.
|
||||
-- See docs/PHASE0.md §11, docs/PHASE1.md §6, docs/PHASE4.md §4.
|
||||
|
||||
local json = require("dkjson")
|
||||
local libc = require("ffi.libc")
|
||||
local ffi = require("ffi")
|
||||
|
||||
local M = {}
|
||||
|
||||
local Session = {}
|
||||
Session.__index = Session
|
||||
|
||||
local Memory = {}
|
||||
Memory.__index = Memory
|
||||
|
||||
-- Best-effort mkdir -p. Failures are surfaced by io.open below. Uses
|
||||
-- single-quote escaping (Lua's %q double-quotes, which still expands $(...)
|
||||
-- and $VAR inside) so a path containing shell metacharacters doesn't trip.
|
||||
@@ -127,4 +133,180 @@ function M.list_sessions(dir)
|
||||
return out
|
||||
end
|
||||
|
||||
-- ============================================================================
|
||||
-- Phase 4: memory.jsonl — cross-session memory store.
|
||||
-- Same JSONL convention as session logs, but a single shared file rather
|
||||
-- than per-session. Single-writer enforced via flock advisory lock.
|
||||
-- See docs/PHASE4.md §2 / §4.
|
||||
-- ============================================================================
|
||||
|
||||
-- We need an integer fd for flock. io.open returns a Lua FILE*; LuaJIT
|
||||
-- has no portable way to extract the underlying fd from that. Workaround:
|
||||
-- open via libc directly using open(2). Already exposed close() in libc;
|
||||
-- need to declare open() and read/write via the existing fd interface.
|
||||
ffi.cdef[[
|
||||
int open(const char *pathname, int flags, int mode);
|
||||
long lseek(int fd, long offset, int whence);
|
||||
]]
|
||||
|
||||
local O_RDWR = 2
|
||||
local O_CREAT = 64 -- 0100 octal on Linux/glibc
|
||||
local O_APPEND = 1024 -- 02000 octal on Linux/glibc
|
||||
local SEEK_SET = 0
|
||||
local FILE_MODE = 0x180 -- 0600 octal — owner rw only
|
||||
|
||||
-- ---------------------------------------------------------------- M.open_memory
|
||||
-- Opens memory.jsonl at `path` for append, takes an exclusive non-blocking
|
||||
-- flock on the fd, scans existing content for max id, writes a meta header
|
||||
-- if the file is new. Returns:
|
||||
-- handle, nil on success
|
||||
-- nil, err on lock-held / open failure
|
||||
function M.open_memory(path)
|
||||
ensure_dir(parent_dir(path))
|
||||
|
||||
-- Open via libc open(2) so we have an integer fd for flock.
|
||||
local fd = ffi.C.open(path,
|
||||
bit and bit.bor(O_RDWR, O_CREAT, O_APPEND)
|
||||
or (O_RDWR + O_CREAT + O_APPEND),
|
||||
FILE_MODE)
|
||||
-- bit lib may not be loaded; fall back to numeric add (flags don't
|
||||
-- overlap so OR == add here).
|
||||
if fd < 0 then
|
||||
return nil, "open " .. path .. " failed: "
|
||||
.. libc.strerror(libc.errno())
|
||||
end
|
||||
|
||||
local ok, err = libc.flock(fd, libc.LOCK_EX + libc.LOCK_NB)
|
||||
if not ok then
|
||||
libc.close(fd)
|
||||
return nil, "memory.jsonl held by another aish process ("
|
||||
.. tostring(err) .. ")"
|
||||
end
|
||||
|
||||
-- Scan existing content for max id. lseek back to start, read all.
|
||||
local max_id = 0
|
||||
local was_empty = true
|
||||
ffi.C.lseek(fd, 0, SEEK_SET)
|
||||
while true do
|
||||
-- Read in 4K chunks. Use libc.read which returns string+len.
|
||||
local chunk, n = libc.read(fd, 4096)
|
||||
if not chunk or n == 0 then break end
|
||||
was_empty = false
|
||||
-- Accumulate into a buffer; on first scan we may straddle lines.
|
||||
-- Simple approach: keep a tail and split on newlines.
|
||||
for line in chunk:gmatch("[^\n]+") do
|
||||
local obj = json.decode(line)
|
||||
if obj and obj.id and obj.id > max_id then max_id = obj.id end
|
||||
end
|
||||
end
|
||||
-- Seek to end so subsequent libc.write appends.
|
||||
ffi.C.lseek(fd, 0, 2) -- SEEK_END
|
||||
|
||||
local handle = setmetatable({
|
||||
path = path,
|
||||
fd = fd,
|
||||
next_id = max_id + 1,
|
||||
closed = false,
|
||||
}, Memory)
|
||||
|
||||
if was_empty then
|
||||
-- Write meta header. No id; load_memory skips lines without id.
|
||||
handle:_write_raw({
|
||||
meta = {
|
||||
aish_version = "phase4",
|
||||
created = os.date("!%Y-%m-%dT%H:%M:%SZ"),
|
||||
}
|
||||
})
|
||||
end
|
||||
|
||||
return handle
|
||||
end
|
||||
|
||||
-- Internal: append one JSON line to the fd.
|
||||
function Memory:_write_raw(obj)
|
||||
local line = json.encode(obj) .. "\n"
|
||||
libc.write(self.fd, line)
|
||||
end
|
||||
|
||||
-- Append a memory item. Returns the assigned id.
|
||||
function Memory:add(kind, content, tags, source)
|
||||
assert(not self.closed, "memory:add on closed handle")
|
||||
assert(kind == "fact" or kind == "pref" or kind == "context",
|
||||
"memory:add: kind must be fact|pref|context (got " .. tostring(kind) .. ")")
|
||||
assert(content and content ~= "", "memory:add: content required")
|
||||
|
||||
local id = self.next_id
|
||||
self.next_id = id + 1
|
||||
local item = {
|
||||
id = id,
|
||||
ts = os.date("!%Y-%m-%dT%H:%M:%SZ"),
|
||||
kind = kind,
|
||||
content = content,
|
||||
}
|
||||
if tags then item.tags = tags end
|
||||
if source then item.source = source end
|
||||
self:_write_raw(item)
|
||||
return id
|
||||
end
|
||||
|
||||
-- Append a tombstone for `target_id`. Idempotent at the file level; the
|
||||
-- caller (e.g. `:memory forget` meta handler) may want to check
|
||||
-- M.load_memory first to surface a "not active" status to the user (N1).
|
||||
function Memory:forget(target_id)
|
||||
assert(not self.closed, "memory:forget on closed handle")
|
||||
self:_write_raw({
|
||||
id = self.next_id,
|
||||
ts = os.date("!%Y-%m-%dT%H:%M:%SZ"),
|
||||
kind = "forget",
|
||||
target = target_id,
|
||||
})
|
||||
self.next_id = self.next_id + 1
|
||||
end
|
||||
|
||||
function Memory:close()
|
||||
if self.closed then return end
|
||||
-- flock is released automatically on fd close.
|
||||
libc.close(self.fd)
|
||||
self.fd = nil
|
||||
self.closed = true
|
||||
end
|
||||
|
||||
-- ---------------------------------------------------------------- M.load_memory
|
||||
-- Read all items, resolve tombstones, return active set sorted by ts desc.
|
||||
-- Items without an `id` field (e.g. the meta header) are silently dropped.
|
||||
-- Tombstones with non-matching targets are no-ops.
|
||||
-- Returns:
|
||||
-- items_table array of {id, ts, kind, content, tags?, source?}
|
||||
-- may be empty if file doesn't exist or contains only meta/tombstones
|
||||
function M.load_memory(path)
|
||||
local fh = io.open(path, "r")
|
||||
if not fh then return {} end
|
||||
|
||||
local items = {} -- by id
|
||||
local forget = {} -- set of target ids
|
||||
for line in fh:lines() do
|
||||
if #line > 0 then
|
||||
local obj = json.decode(line)
|
||||
if obj and obj.id then
|
||||
if obj.kind == "forget" then
|
||||
if obj.target then forget[obj.target] = true end
|
||||
elseif obj.kind == "fact" or obj.kind == "pref"
|
||||
or obj.kind == "context" then
|
||||
items[obj.id] = obj
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
fh:close()
|
||||
|
||||
local active = {}
|
||||
for id, item in pairs(items) do
|
||||
if not forget[id] then active[#active + 1] = item end
|
||||
end
|
||||
-- Sort by ts descending (most recent first). Strings sort right when
|
||||
-- they're ISO 8601 — ASCII order = chronological.
|
||||
table.sort(active, function(a, b) return a.ts > b.ts end)
|
||||
return active
|
||||
end
|
||||
|
||||
return M
|
||||
|
||||
Reference in New Issue
Block a user