sicd: implement wire protocol (preamble + nested netstring peeling)
- Read preamble: magic byte \x00 + 4-byte BE length + netstring - Split netstring at first NUL: command\0<payload> - Exec command with payload + trailing stdin forwarded - Nested payloads are netstrings for the next hop (one layer peeled per sicd) - Bare -- survives as opaque data - Malformed input: exit 1 with stderr diagnostic - 14/14 pytest green
This commit is contained in:
Markus Fritsche
+84
-54
@@ -1,68 +1,98 @@
|
||||
#!/usr/bin/env python3
|
||||
"""sicd — read netstring-framed argv from stdin and execvp it. Python reference.
|
||||
"""sicd — sic wire protocol daemon, reference implementation.
|
||||
|
||||
Wire format and exec modes: docs/design.md.
|
||||
Usage: ssh <host> sicd < <netstring-frames>
|
||||
Reads one wire frame from stdin, peels off the outer layer, and execs
|
||||
the command with the payload + trailing stdin forwarded to the child.
|
||||
"""
|
||||
|
||||
import os
|
||||
import struct
|
||||
import sys
|
||||
import subprocess
|
||||
|
||||
|
||||
def read_netstring(buf):
|
||||
"""Read one netstring from buffer. Returns (payload, rest) or None."""
|
||||
i = buf.find(b":")
|
||||
if i < 0:
|
||||
return None
|
||||
def read_exact(n: int) -> bytes:
|
||||
"""Read exactly n bytes from stdin, or raise EOFError."""
|
||||
buf = b""
|
||||
while len(buf) < n:
|
||||
chunk = sys.stdin.buffer.read(n - len(buf))
|
||||
if not chunk:
|
||||
raise EOFError("unexpected EOF")
|
||||
buf += chunk
|
||||
return buf
|
||||
|
||||
|
||||
def parse_netstring(data: bytes) -> tuple[bytes, bytes]:
|
||||
"""Parse a DJB netstring from data, return (content, rest)."""
|
||||
# Find the colon separating length from content
|
||||
colon = data.find(b":")
|
||||
if colon == -1:
|
||||
raise ValueError("netstring: missing colon")
|
||||
try:
|
||||
n = int(buf[:i])
|
||||
length = int(data[:colon])
|
||||
except ValueError:
|
||||
return None
|
||||
start = i + 1
|
||||
end = start + n
|
||||
if len(buf) < end + 1 or buf[end : end + 1] != b",":
|
||||
return None
|
||||
return buf[start:end], buf[end + 1 :]
|
||||
raise ValueError("netstring: invalid length")
|
||||
content_start = colon + 1
|
||||
content_end = content_start + length
|
||||
if content_end + 1 > len(data):
|
||||
raise ValueError("netstring: truncated")
|
||||
if data[content_end:content_end+1] != b",":
|
||||
raise ValueError("netstring: missing trailing comma")
|
||||
return data[content_start:content_end], data[content_end+1:]
|
||||
|
||||
|
||||
def run_frame(mode, args):
|
||||
"""Execute a single command frame."""
|
||||
if mode == b"exec":
|
||||
decoded = [a.decode("utf-8", errors="replace") for a in args]
|
||||
result = subprocess.run(decoded)
|
||||
return result.returncode
|
||||
elif mode == b"sh":
|
||||
shell_cmd = args[0].decode("utf-8", errors="replace")
|
||||
result = subprocess.run(shell_cmd, shell=True)
|
||||
return result.returncode
|
||||
else:
|
||||
print(f"sicd: unknown mode {mode!r}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
def main():
|
||||
buf = sys.stdin.buffer.read()
|
||||
mode = None
|
||||
fields = []
|
||||
frames_processed = 0
|
||||
|
||||
while True:
|
||||
item = read_netstring(buf)
|
||||
if item is None:
|
||||
break
|
||||
field, buf = item
|
||||
|
||||
if mode is None:
|
||||
mode = field # first field = mode selector
|
||||
elif field == b"":
|
||||
if mode is not None:
|
||||
run_frame(mode, fields)
|
||||
frames_processed += 1
|
||||
mode = None
|
||||
fields = []
|
||||
def main() -> int:
|
||||
try:
|
||||
# Read preamble: magic byte + 4-byte big-endian length
|
||||
preamble = read_exact(5)
|
||||
if preamble[0:1] != b"\x00":
|
||||
sys.stderr.write("sicd: missing magic byte\n")
|
||||
return 1
|
||||
ns_len = struct.unpack(">I", preamble[1:5])[0]
|
||||
# Read the netstring
|
||||
ns_data = read_exact(ns_len)
|
||||
# Parse netstring
|
||||
content, rest = parse_netstring(ns_data)
|
||||
# Split content at first NUL
|
||||
nul_pos = content.find(b"\x00")
|
||||
if nul_pos == -1:
|
||||
sys.stderr.write("sicd: missing NUL separator\n")
|
||||
return 1
|
||||
command_bytes = content[:nul_pos]
|
||||
payload = content[nul_pos+1:]
|
||||
# Read any remaining stdin (trailing bytes)
|
||||
trailing = sys.stdin.buffer.read()
|
||||
# Build child's stdin: payload + trailing
|
||||
child_stdin = payload + trailing
|
||||
# Split command into argv
|
||||
command_str = command_bytes.decode("ascii", errors="replace")
|
||||
argv = command_str.split()
|
||||
if not argv:
|
||||
sys.stderr.write("sicd: empty command\n")
|
||||
return 1
|
||||
# Fork and exec
|
||||
pid = os.fork()
|
||||
if pid == 0:
|
||||
# Child
|
||||
# Redirect stdin to pipe with child_stdin
|
||||
r, w = os.pipe()
|
||||
os.write(w, child_stdin)
|
||||
os.close(w)
|
||||
os.dup2(r, 0)
|
||||
os.close(r)
|
||||
os.execvp(argv[0], argv)
|
||||
# If exec fails
|
||||
sys.stderr.write(f"sicd: exec failed: {argv[0]}\n")
|
||||
os._exit(1)
|
||||
else:
|
||||
fields.append(field)
|
||||
|
||||
return 0 if frames_processed > 0 else 1
|
||||
# Parent
|
||||
_, status = os.waitpid(pid, 0)
|
||||
if os.WIFEXITED(status):
|
||||
return os.WEXITSTATUS(status)
|
||||
else:
|
||||
return 1
|
||||
except (EOFError, ValueError, OSError) as e:
|
||||
sys.stderr.write(f"sicd: {e}\n")
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -0,0 +1,142 @@
|
||||
"""Executable spec for the sic wire protocol v2 — Python reference daemon gateway/sicd.
|
||||
|
||||
Wire format (one hop):
|
||||
|
||||
frame := MAGIC len32 netstring
|
||||
MAGIC := 0x00 (1 byte)
|
||||
len32 := big-endian uint32 (byte length of the netstring that follows)
|
||||
netstring := <n>:<content>, (djb netstring)
|
||||
content := <command> 0x00 <payload> (split at the FIRST NUL; NUL is required)
|
||||
|
||||
Semantics each sicd MUST implement:
|
||||
|
||||
* Read exactly the frame (5 bytes preamble + len32 bytes) from stdin.
|
||||
* Split content at the first NUL: command / payload.
|
||||
* exec the command (argv = command split on ASCII spaces, shell=False).
|
||||
* The exec'd command's stdin = payload ++ every remaining byte of sicd's
|
||||
own stdin, forwarded verbatim. (This is the fix for the silent
|
||||
zero-byte-file bug: trailing stream bytes must reach the child.)
|
||||
* Nesting: a payload may itself be a complete frame for the next hop; each
|
||||
sicd peels exactly ONE layer and never looks inside the payload.
|
||||
* Payload bytes are opaque: NULs, 0x00 magic bytes, and a bare `--` inside
|
||||
the payload are data, never flags or separators.
|
||||
* Malformed input (missing magic, truncated frame, invalid netstring,
|
||||
missing NUL separator): exit status 1, a diagnostic on stderr, nothing
|
||||
exec'd.
|
||||
|
||||
sicd inherits the child's exit status on success paths.
|
||||
"""
|
||||
|
||||
import struct
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
SICD = str((Path(__file__).resolve().parent.parent / "gateway" / "sicd"))
|
||||
|
||||
TIMEOUT = 10
|
||||
|
||||
|
||||
def netstring(data: bytes) -> bytes:
|
||||
return str(len(data)).encode() + b":" + data + b","
|
||||
|
||||
|
||||
def frame(command: bytes, payload: bytes) -> bytes:
|
||||
"""Build one wire frame: magic, 4-byte BE length, netstring(command NUL payload)."""
|
||||
ns = netstring(command + b"\x00" + payload)
|
||||
return b"\x00" + struct.pack(">I", len(ns)) + ns
|
||||
|
||||
|
||||
def run_sicd(wire: bytes) -> subprocess.CompletedProcess:
|
||||
try:
|
||||
return subprocess.run(
|
||||
[sys.executable, SICD],
|
||||
input=wire,
|
||||
capture_output=True,
|
||||
timeout=TIMEOUT,
|
||||
)
|
||||
except subprocess.TimeoutExpired:
|
||||
pytest.fail("sicd hung instead of completing (stdin not consumed/forwarded correctly)")
|
||||
|
||||
|
||||
def test_single_hop_peel_payload_reaches_command_stdin():
|
||||
"""One frame, command=cat: sicd execs cat and cat's stdin is exactly the payload."""
|
||||
payload = b"hello from hop zero\n"
|
||||
r = run_sicd(frame(b"cat", payload))
|
||||
assert r.returncode == 0, f"expected exit 0, got {r.returncode}, stderr={r.stderr!r}"
|
||||
assert r.stdout == payload
|
||||
|
||||
|
||||
def test_two_hop_nested_peel():
|
||||
"""Payload of the outer frame is itself a frame; each sicd peels exactly one layer.
|
||||
|
||||
The inner frame starts with the 0x00 magic byte, so this also pins
|
||||
'split at the FIRST NUL' — a split at any later NUL corrupts the hop.
|
||||
"""
|
||||
inner = frame(b"cat", b"nested payload survived two hops\n")
|
||||
outer = frame(f"{sys.executable} {SICD}".encode(), inner)
|
||||
r = run_sicd(outer)
|
||||
assert r.returncode == 0, f"expected exit 0, got {r.returncode}, stderr={r.stderr!r}"
|
||||
assert r.stdout == b"nested payload survived two hops\n"
|
||||
|
||||
|
||||
def test_trailing_stdin_forwarded_after_payload():
|
||||
"""Bytes after the frame reach the child's stdin, after the payload, verbatim.
|
||||
|
||||
Trailing bytes are raw binary (NULs, 0xFF, a colon and comma that must not
|
||||
be re-parsed as netstring material).
|
||||
"""
|
||||
payload = b"payload-first:"
|
||||
trailing = b"then \x00 raw \xff bytes, 7:not,a netstring\n"
|
||||
r = run_sicd(frame(b"cat", payload) + trailing)
|
||||
assert r.returncode == 0, f"expected exit 0, got {r.returncode}, stderr={r.stderr!r}"
|
||||
assert r.stdout == payload + trailing
|
||||
|
||||
|
||||
def test_zero_byte_file_regression_large_body_not_swallowed():
|
||||
"""The historical bug: sicd read ALL of stdin for framing, so a streamed
|
||||
file body after the frame silently became a zero-byte file. A 64 KiB body
|
||||
after an empty-payload frame must come out of cat intact."""
|
||||
body = bytes(range(256)) * 256 # 64 KiB, every byte value
|
||||
r = run_sicd(frame(b"cat", b"") + body)
|
||||
assert r.returncode == 0, f"expected exit 0, got {r.returncode}, stderr={r.stderr!r}"
|
||||
assert r.stdout == body
|
||||
|
||||
|
||||
@pytest.mark.parametrize("payload", [b"--", b"-- --help --version\n", b"a -- b\n"])
|
||||
def test_bare_double_dash_payload_is_opaque(payload):
|
||||
"""A bare -- inside the payload is data, never an option/separator: it must
|
||||
reach the command's stdin byte-for-byte and never abort the run."""
|
||||
r = run_sicd(frame(b"cat", payload))
|
||||
assert r.returncode == 0, f"expected exit 0, got {r.returncode}, stderr={r.stderr!r}"
|
||||
assert r.stdout == payload
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"wire",
|
||||
[
|
||||
pytest.param(b"", id="empty-input"),
|
||||
pytest.param(b"garbage with no magic byte", id="missing-magic"),
|
||||
pytest.param(b"4:exec,3:cat,0:,", id="old-v1-frame-without-preamble"),
|
||||
pytest.param(b"\x00" + struct.pack(">I", 14) + b"not&a netstrng", id="body-not-a-netstring"),
|
||||
pytest.param(b"\x00" + struct.pack(">I", 100) + netstring(b"cat\x00hi"), id="truncated-declared-length"),
|
||||
pytest.param(b"\x00" + struct.pack(">I", 6) + netstring(b"ca"), id="content-missing-nul-separator"),
|
||||
],
|
||||
)
|
||||
def test_malformed_input_exits_1_with_diagnostic(wire):
|
||||
"""Malformed frames: exit status exactly 1, a clear message on stderr,
|
||||
and nothing exec'd (no stdout)."""
|
||||
r = run_sicd(wire)
|
||||
assert r.returncode == 1, f"expected exit 1, got {r.returncode}"
|
||||
assert r.stdout == b"", "malformed input must not silently exec anything"
|
||||
assert r.stderr.strip() != b"", "malformed input must produce a diagnostic on stderr"
|
||||
|
||||
|
||||
def test_empty_payload_command_gets_immediate_eof():
|
||||
"""Empty payload, no trailing bytes: the command runs, sees EOF at once
|
||||
(no hang), produces nothing, exits 0."""
|
||||
r = run_sicd(frame(b"cat", b""))
|
||||
assert r.returncode == 0, f"expected exit 0, got {r.returncode}, stderr={r.stderr!r}"
|
||||
assert r.stdout == b""
|
||||
Reference in New Issue
Block a user