From dbf01eddb8ca84a167465c4c225ada596771fb74 Mon Sep 17 00:00:00 2001 From: Markus Fritsche Date: Tue, 26 May 2026 14:15:13 +0200 Subject: [PATCH] daemon: shadow_decoder wiring (PR-Q3a.1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Toolchain plumbing for the upcoming daedalus-decoder shadow-mode path. Production behaviour is unchanged. What lands here: 1. CMake links libdaedalus_decoder via pkg-config. Static archive, so no .so dependency change in the daemon's link map. 2. ffmpeg_loader resolves ff_h264_set_mb_inspect_cb NULL-tolerantly. Stock libavcodec lacks the symbol (logged as INFO at startup); the marfrit-packages ffmpeg-v4l2-request-fourier fork's 0016 patch exports it. The shadow path activates only when both env DAEDALUS_SHADOW_MODE=1 AND the symbol resolves. 3. New shadow_decoder.[ch] module: - shadow_decoder_create() gates on env + symbol presence, returns NULL in production state (the common case). - shadow_decoder_install_cb() registers a per-MB callback on the H.264 AVCodecContext; lazily-created daedalus_decoder context will pick up dimensions from the first AVFrame. - shadow_decoder_on_frame() logs per-frame MB-observed count. Every entry point is NULL-safe so decoder.c stays clean of conditionals. 4. decoder.{c,h} grow a `struct shadow_decoder *shadow` field on daedalus_decoder. Install hook fires once per H.264 codec open; frame hook fires after each successful avcodec_receive_frame. PR-Q3a.1 scope ENDS here. The callback just counts MBs; no daedalus_decoder_append_mb or flush_frame yet. Real-coeffs / edges extraction needs the patched FFmpeg source-tree headers (DAEDALUS_FFMPEG_SRC) to introspect H264Context internals — that lands in PR-Q3a.2. dejavu-check: this path is daedalus-decoder's frame-major UMA dispatch architecture (one cmdbuf per frame, one submit) running alongside libavcodec's reference decode for validation. It is NOT per-kernel libavcodec function-pointer substitution. No new libavcodec patches; the existing 0016 callback is the only intercept point. Verified on hertz: - Build: clean, libdaedalus_decoder.a linked. - Disabled state (env unset OR symbol absent): no shadow log lines, daemon init continues normally, INFO logs "libavcodec lacks ff_h264_set_mb_inspect_cb (stock build, no daedalus-fourier 0016 patch) — shadow-mode unavailable". - Enabled state would require ffmpeg-v4l2-request-fourier .deb rebuilt with patches 0016/0017 deployed to hertz (current .deb release 10 predates them). That's a deployment task, separate from this PR. Co-Authored-By: Claude Opus 4.7 (1M context) --- daemon/CMakeLists.txt | 15 +++- daemon/src/decoder.c | 30 +++++++ daemon/src/decoder.h | 6 ++ daemon/src/ffmpeg_loader.c | 18 ++++ daemon/src/ffmpeg_loader.h | 29 +++++++ daemon/src/shadow_decoder.c | 162 ++++++++++++++++++++++++++++++++++++ daemon/src/shadow_decoder.h | 75 +++++++++++++++++ 7 files changed, 333 insertions(+), 2 deletions(-) create mode 100644 daemon/src/shadow_decoder.c create mode 100644 daemon/src/shadow_decoder.h diff --git a/daemon/CMakeLists.txt b/daemon/CMakeLists.txt index 2051019..d5edeae 100644 --- a/daemon/CMakeLists.txt +++ b/daemon/CMakeLists.txt @@ -40,6 +40,11 @@ pkg_check_modules(FFMPEG REQUIRED IMPORTED_TARGET # libdaedalus_core.a must precede -lvulkan because the static archive # references vulkan symbols and the linker resolves left-to-right. pkg_check_modules(DAEDALUS_FOURIER REQUIRED daedalus-fourier) +# daedalus-decoder — frame-major UMA H.264 decoder. Linked into the +# shadow-mode path (env DAEDALUS_SHADOW_MODE=1) and inert otherwise. +# Linked unconditionally to keep CMake configurations symmetrical +# between production and shadow-mode runs. +pkg_check_modules(DAEDALUS_DECODER REQUIRED daedalus-decoder) find_package(Vulkan REQUIRED) add_executable(daedalus_v4l2_daemon @@ -48,6 +53,7 @@ add_executable(daedalus_v4l2_daemon src/log.c src/parser.c src/decoder.c + src/shadow_decoder.c src/chardev_client.c src/dmabuf_capture.c src/bitstream_writer.c @@ -61,20 +67,25 @@ target_include_directories(daedalus_v4l2_daemon ${CMAKE_CURRENT_SOURCE_DIR}/../include ${FFMPEG_INCLUDE_DIRS} ${DAEDALUS_FOURIER_INCLUDE_DIRS} + ${DAEDALUS_DECODER_INCLUDE_DIRS} ) # dl for dlopen, pthread for future threading work. target_link_directories(daedalus_v4l2_daemon PRIVATE ${DAEDALUS_FOURIER_LIBRARY_DIRS} + ${DAEDALUS_DECODER_LIBRARY_DIRS} ) target_link_libraries(daedalus_v4l2_daemon PRIVATE dl pthread - # Order matters: libdaedalus_core.a first (so its undefined - # vulkan symbols register), then -lvulkan to satisfy them. + # Order matters for left-to-right linker resolution of + # static archives. daedalus-decoder references symbols + # from daedalus-fourier; daedalus-fourier references + # vulkan symbols. So: decoder, fourier, vulkan. + ${DAEDALUS_DECODER_LIBRARIES} ${DAEDALUS_FOURIER_LIBRARIES} Vulkan::Vulkan ) diff --git a/daemon/src/decoder.c b/daemon/src/decoder.c index 83e0b24..cd18f24 100644 --- a/daemon/src/decoder.c +++ b/daemon/src/decoder.c @@ -6,6 +6,7 @@ #include "ffmpeg_loader.h" #include "h264_nal_synth.h" #include "log.h" +#include "shadow_decoder.h" #include #include @@ -110,6 +111,13 @@ int daedalus_decoder_init(struct daedalus_decoder *dec, loader->av_packet_free(&dec->pkt); return -ENOMEM; } + /* + * Returns NULL when DAEDALUS_SHADOW_MODE != "1" or the loaded + * libavcodec lacks the per-MB inspection callback. Both are + * the normal production state — the rest of decoder.c is + * shadow-aware via NULL-safe shadow_decoder_* entry points. + */ + dec->shadow = shadow_decoder_create(loader); return 0; } @@ -117,6 +125,8 @@ void daedalus_decoder_cleanup(struct daedalus_decoder *dec) { if (!dec || !dec->loader) return; + if (dec->shadow) + shadow_decoder_destroy(dec->shadow); if (dec->ctx_vp9) dec->loader->avcodec_free_context(&dec->ctx_vp9); if (dec->ctx_av1) @@ -211,6 +221,16 @@ static int decoder_open_codec(struct daedalus_decoder *dec, uint32_t codec_id, *cache = ctx; *out = ctx; log_info("decoder: opened %s context", codec->name); + + /* + * Shadow-mode hook on H.264 only: install the per-MB inspection + * callback once the AVCodecContext is open. NULL-safe — when + * shadow mode is disabled (the normal production case) this + * does nothing. + */ + if (codec_id == DAEDALUS_CODEC_H264) + shadow_decoder_install_cb(dec->shadow, ctx); + return 0; } @@ -595,6 +615,16 @@ int daedalus_decoder_run_request(struct daedalus_decoder *dec, goto out; } + /* + * Shadow-mode frame-boundary hook. H.264-only — the per-MB + * callback is only registered for H.264, so on VP9/AV1 frames + * shadow->mbs_this_frame stays zero anyway, but keeping the + * codec gate here makes the log lines easier to read. + * NULL-safe. + */ + if (req->codec_id == DAEDALUS_CODEC_H264) + shadow_decoder_on_frame(dec->shadow, dec->frame); + { struct AVFrame *fr = dec->frame; const AVPixFmtDescriptor *desc = diff --git a/daemon/src/decoder.h b/daemon/src/decoder.h index f6eba56..1527e2d 100644 --- a/daemon/src/decoder.h +++ b/daemon/src/decoder.h @@ -21,6 +21,7 @@ struct ffmpeg_loader; struct AVCodecContext; struct AVPacket; struct AVFrame; +struct shadow_decoder; /** * struct daedalus_decoder - per-daemon decoder state @@ -31,6 +32,10 @@ struct AVFrame; * @ctx_h264: lazily-opened H.264 AVCodecContext * @pkt: shared AVPacket reused across requests * @frame: shared AVFrame reused across requests + * @shadow: env-gated daedalus-decoder shadow path; NULL when + * DAEDALUS_SHADOW_MODE != "1" or libavcodec lacks the + * per-MB inspection callback. Production path doesn't + * care; all shadow_decoder_* entry points are NULL-safe. */ struct daedalus_decoder { struct ffmpeg_loader *loader; @@ -39,6 +44,7 @@ struct daedalus_decoder { struct AVCodecContext *ctx_h264; struct AVPacket *pkt; struct AVFrame *frame; + struct shadow_decoder *shadow; }; /** diff --git a/daemon/src/ffmpeg_loader.c b/daemon/src/ffmpeg_loader.c index 4b9f940..8f603fa 100644 --- a/daemon/src/ffmpeg_loader.c +++ b/daemon/src/ffmpeg_loader.c @@ -109,6 +109,24 @@ int ffmpeg_loader_init(struct ffmpeg_loader *loader) RESOLVE(libavutil, LIBAVUTIL_SONAME, av_version_info); RESOLVE(libavutil, LIBAVUTIL_SONAME, av_pix_fmt_desc_get); + /* + * Optional symbols. Resolved NULL-tolerantly — stock libavcodec + * does not export these; the marfrit-packages + * ffmpeg-v4l2-request-fourier fork does (patches 0016/0017). + * Callers MUST NULL-check before invoking. Clear any stale + * dlerror() the previous lookups left behind so we read a clean + * status here. + */ + (void) dlerror(); + *(void **) &loader->ff_h264_set_mb_inspect_cb = + dlsym(loader->libavcodec, "ff_h264_set_mb_inspect_cb"); + if (!loader->ff_h264_set_mb_inspect_cb) { + log_info("libavcodec lacks ff_h264_set_mb_inspect_cb " + "(stock build, no daedalus-fourier 0016 patch) " + "— shadow-mode unavailable"); + (void) dlerror(); /* discard the not-found message */ + } + { unsigned int v = loader->avformat_version(); log_info("FFmpeg loaded: %s (libavformat %u.%u.%u)", diff --git a/daemon/src/ffmpeg_loader.h b/daemon/src/ffmpeg_loader.h index c43ee41..ef89d21 100644 --- a/daemon/src/ffmpeg_loader.h +++ b/daemon/src/ffmpeg_loader.h @@ -35,6 +35,14 @@ #include #include +/* + * Forward declaration must precede ff_h264_set_mb_inspect_cb's + * function-pointer signature below — otherwise the compiler treats + * `struct H264Context` as a parameter-scope declaration and the type + * is incompatible with later uses in shadow_decoder.c. + */ +struct H264Context; /* opaque outside libavcodec */ + /** * struct ffmpeg_loader - resolved FFmpeg API entry points * @libavformat: dlopen handle (close in cleanup) @@ -88,6 +96,27 @@ struct ffmpeg_loader { const char *(*av_get_media_type_string)(enum AVMediaType); const char *(*av_version_info)(void); const AVPixFmtDescriptor *(*av_pix_fmt_desc_get)(enum AVPixelFormat); + + /* + * Optional libavcodec symbols. NULL when the loaded + * libavcodec.so doesn't carry the corresponding marfrit-packages + * patch. Callers must NULL-check before invoking. + * + * ff_h264_set_mb_inspect_cb — marfrit-packages patch 0016. + * Registers a per-MB callback that fires at the end of + * ff_h264_hl_decode_mb. Used by daedalus-v4l2's shadow-mode + * path to drive daedalus-decoder's frame-major dispatch + * alongside libavcodec's reference decode. H264Context stays + * opaque to the daemon — extraction of its private fields needs + * the patched FFmpeg source-tree headers (see the CLI in + * daedalus-decoder/tools/daedalus_decode_h264.c) and is + * deferred to PR-Q3a.2. + */ + void (*ff_h264_set_mb_inspect_cb)(struct AVCodecContext *avctx, + void (*cb)(void *opaque, + const struct H264Context *h, + int mb_x, int mb_y), + void *opaque); }; /** diff --git a/daemon/src/shadow_decoder.c b/daemon/src/shadow_decoder.c new file mode 100644 index 0000000..b3b032b --- /dev/null +++ b/daemon/src/shadow_decoder.c @@ -0,0 +1,162 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * shadow_decoder.c — env-gated parallel daedalus-decoder wiring. + * + * PR-Q3a.1 scope: prove the toolchain. + * + * 1. DAEDALUS_SHADOW_MODE=1 + libavcodec carries marfrit-packages + * 0016 (ff_h264_set_mb_inspect_cb) → shadow path active. + * 2. Per-MB callback fires on every macroblock libavcodec emits. + * We only count the firings here. + * 3. Frame boundary creates a daedalus_decoder context lazily + * (sized from the first AVFrame); destroy + recreate on + * resolution change. + * 4. Per-frame log line surfaces MB count + has_qpu state. + * + * No daedalus_decoder_append_mb / flush_frame calls yet — that + * needs H264Context introspection which depends on the patched + * FFmpeg source-tree headers (DAEDALUS_FFMPEG_SRC) and lands in + * PR-Q3a.2. This module's job here is to confirm the link + * survives, the callback resolves, the context creates, and + * tearing the path back down doesn't perturb the production + * AVFrame → V4L2 pipeline. + */ +#include "shadow_decoder.h" + +#include "ffmpeg_loader.h" +#include "log.h" + +#include +#include + +#include + +#include +#include + +struct shadow_decoder { + struct ffmpeg_loader *loader; + daedalus_decoder *dec; /* lazily created on first frame */ + int ctx_w; /* coded-frame width at last create */ + int ctx_h; + uint64_t mbs_this_frame; + uint64_t total_frames; + uint64_t total_mbs; +}; + +static void shadow_mb_inspect(void *opaque, + const struct H264Context *h __attribute__((unused)), + int mb_x __attribute__((unused)), + int mb_y __attribute__((unused))) +{ + struct shadow_decoder *sh = opaque; + sh->mbs_this_frame++; +} + +struct shadow_decoder *shadow_decoder_create(struct ffmpeg_loader *loader) +{ + const char *env = getenv("DAEDALUS_SHADOW_MODE"); + + if (!env || strcmp(env, "1") != 0) + return NULL; + + if (!loader || !loader->ff_h264_set_mb_inspect_cb) { + log_warn("shadow_decoder: DAEDALUS_SHADOW_MODE=1 set but " + "libavcodec lacks ff_h264_set_mb_inspect_cb — disabled"); + return NULL; + } + + struct shadow_decoder *sh = calloc(1, sizeof(*sh)); + if (!sh) { + log_err("shadow_decoder: out of memory"); + return NULL; + } + sh->loader = loader; + log_info("shadow_decoder: enabled (DAEDALUS_SHADOW_MODE=1, " + "daedalus-decoder version %s)", + daedalus_decoder_version()); + return sh; +} + +void shadow_decoder_destroy(struct shadow_decoder *sh) +{ + if (!sh) + return; + if (sh->dec) + daedalus_decoder_destroy(sh->dec); + log_info("shadow_decoder: shutdown — observed %llu frames / %llu MBs", + (unsigned long long) sh->total_frames, + (unsigned long long) sh->total_mbs); + free(sh); +} + +void shadow_decoder_install_cb(struct shadow_decoder *sh, + struct AVCodecContext *avctx) +{ + if (!sh || !avctx) + return; + /* + * Loader's optional-symbol pointer was checked at create time + * (we wouldn't be non-NULL otherwise), so the call is safe. + */ + sh->loader->ff_h264_set_mb_inspect_cb(avctx, shadow_mb_inspect, sh); + log_info("shadow_decoder: per-MB callback installed on H.264 ctx"); +} + +/* + * Ensure the daedalus_decoder context matches the frame's dimensions. + * Rounds up to the H.264 macroblock grid (16-pixel multiples) — the + * coded picture is always 16-aligned even when the displayed crop + * isn't. Returns 0 on success, -1 on failure (ctx left NULL; caller + * logs and continues without shadow dispatch this frame). + */ +static int shadow_ensure_ctx(struct shadow_decoder *sh, int w, int h) +{ + int coded_w = (w + 15) & ~15; + int coded_h = (h + 15) & ~15; + + if (sh->dec && sh->ctx_w == coded_w && sh->ctx_h == coded_h) + return 0; + + if (sh->dec) { + daedalus_decoder_destroy(sh->dec); + sh->dec = NULL; + } + + sh->dec = daedalus_decoder_create(coded_w, coded_h); + if (!sh->dec) { + log_warn("shadow_decoder: daedalus_decoder_create(%dx%d) " + "failed — shadow dispatch skipped this stream", + coded_w, coded_h); + sh->ctx_w = sh->ctx_h = 0; + return -1; + } + sh->ctx_w = coded_w; + sh->ctx_h = coded_h; + log_info("shadow_decoder: ctx ready (%dx%d coded, has_qpu=%d)", + coded_w, coded_h, daedalus_decoder_has_qpu(sh->dec)); + return 0; +} + +void shadow_decoder_on_frame(struct shadow_decoder *sh, + const struct AVFrame *fr) +{ + if (!sh || !fr) + return; + + (void) shadow_ensure_ctx(sh, fr->width, fr->height); + + sh->total_frames++; + sh->total_mbs += sh->mbs_this_frame; + + uint64_t expected = (uint64_t) ((fr->width + 15) >> 4) * + (uint64_t) ((fr->height + 15) >> 4); + log_info("shadow_decoder: frame #%llu %dx%d — %llu MBs observed " + "(expected %llu)", + (unsigned long long) sh->total_frames, + fr->width, fr->height, + (unsigned long long) sh->mbs_this_frame, + (unsigned long long) expected); + + sh->mbs_this_frame = 0; +} diff --git a/daemon/src/shadow_decoder.h b/daemon/src/shadow_decoder.h new file mode 100644 index 0000000..8590273 --- /dev/null +++ b/daemon/src/shadow_decoder.h @@ -0,0 +1,75 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * shadow_decoder.h — env-gated parallel daedalus-decoder path. + * + * When the daemon is launched with DAEDALUS_SHADOW_MODE=1, shadow_decoder + * runs alongside libavcodec's normal H.264 decode: a per-MB inspection + * callback fires for every macroblock libavcodec emits, and a frame- + * boundary hook lets the shadow path observe and (in future PRs) + * dispatch the same frame's worth of work through daedalus-decoder's + * frame-major UMA pipeline. Production output (AVFrame → V4L2 NV12) + * is unchanged regardless of this module's state. + * + * PR-Q3a.1 scope: wiring only. The callback counts MBs and the per- + * frame hook logs the count. No daedalus-decoder dispatch yet; that + * lands in PR-Q3a.2 along with the H264Context-introspection path + * gated on the patched FFmpeg source-tree headers. + * + * Disabled state (env unset or libavcodec lacks ff_h264_set_mb_inspect_cb) + * is a hard NULL — shadow_decoder_create() returns NULL, all other + * entry points are safe with NULL and become no-ops. + * + * The daedalus-decoder context, when active, is created lazily on the + * first observed frame (dimensions come from libavcodec's AVFrame, not + * from the SPS — keeps init independent of stream-header bring-up + * order) and re-created on resolution change. + */ +#ifndef DAEDALUS_V4L2_SHADOW_DECODER_H +#define DAEDALUS_V4L2_SHADOW_DECODER_H + +#include + +struct ffmpeg_loader; +struct AVCodecContext; +struct AVFrame; +struct shadow_decoder; + +/** + * shadow_decoder_create - allocate shadow state if env-enabled + * @loader: borrowed FFmpeg loader (must outlive the returned ctx) + * + * Probes DAEDALUS_SHADOW_MODE env var and the loader's optional + * ff_h264_set_mb_inspect_cb pointer. Returns NULL when shadow mode + * is disabled or unsupported; that's the normal production state. + * Returns a usable handle otherwise. Caller owns the handle and must + * call shadow_decoder_destroy. + */ +struct shadow_decoder *shadow_decoder_create(struct ffmpeg_loader *loader); + +/** + * shadow_decoder_destroy - tear down. Safe with NULL. + */ +void shadow_decoder_destroy(struct shadow_decoder *sh); + +/** + * shadow_decoder_install_cb - install the per-MB inspection callback + * on a freshly-opened H.264 AVCodecContext + * + * Safe with NULL @sh (NOP). Should be called once per H.264 codec + * open; repeated calls just reinstall and are harmless. + */ +void shadow_decoder_install_cb(struct shadow_decoder *sh, + struct AVCodecContext *avctx); + +/** + * shadow_decoder_on_frame - per-frame boundary hook + * + * Called after avcodec_receive_frame returns a frame. Logs the per- + * frame MB counter, resets it, and (in future PRs) drives + * daedalus_decoder_flush_frame + the AVFrame-vs-shadow diff. Safe + * with NULL @sh. + */ +void shadow_decoder_on_frame(struct shadow_decoder *sh, + const struct AVFrame *fr); + +#endif /* DAEDALUS_V4L2_SHADOW_DECODER_H */ -- 2.47.3