wip: deblock dispatch

include/daedalus_decoder.h

@@ -41,6 +41,46 @@ extern "C" {
  * ----------------------------------------------------------------- */
 typedef struct daedalus_decoder daedalus_decoder;
 
+/* -------------------------------------------------------------------
+ * Per-edge deblock metadata.  One entry per filter-edge; the caller
+ * derives these from H.264 §8.7.2.1 boundary-strength rules.
+ *
+ * Coordinate convention:
+ *   mb_x / mb_y  — the MB whose top-left this edge sits on (the "right"
+ *                  side for vertical edges, "bottom" side for horizontal
+ *                  edges, in H.264 spec's q-side convention).
+ *   edge_idx     — 0..3 within the MB:
+ *                    luma:   edge 0 = MB boundary, edges 1..3 = internal
+ *                            at cols/rows 4, 8, 12.
+ *                    chroma: edge 0 = MB boundary, edge 1 = internal at
+ *                            col/row 4.  edge_idx > 1 invalid for chroma.
+ *                  Edges at frame boundaries (top row of MBs for H edges;
+ *                  left column for V edges) MUST be bS=0 — the kernel
+ *                  reads p3 at four samples beyond the edge.
+ *   orient       — 0 = vertical edge (filtered horizontally across), 1 = horizontal.
+ *   plane        — 0 = luma, 1 = chroma Cb, 2 = chroma Cr.  Cb and Cr
+ *                  always share the same filter parameters per H.264
+ *                  spec, but are listed separately so the caller can
+ *                  omit one or the other if needed.
+ *   bS           — 0 = skip this edge (no GPU work), 1..3 = bS<4 path
+ *                  (uses tc0), 4 = bS=4 "intra" path (ignores tc0).
+ *   alpha, beta  — H.264 §8.7.2.2 table 8-16/8-17 values, both 0..255.
+ *   tc0[4]      — per-4-cell segment strength along the edge (luma has
+ *                  4 segments; chroma has 4 also, with 2 cells each).
+ *                  IGNORED when bS == 4.
+ * ----------------------------------------------------------------- */
+struct daedalus_decoder_edge {
+    uint16_t mb_x;
+    uint16_t mb_y;
+    uint8_t  edge_idx;
+    uint8_t  orient;
+    uint8_t  plane;
+    uint8_t  bS;
+    uint8_t  alpha;
+    uint8_t  beta;
+    int8_t   tc0[4];
+};
+
 /* -------------------------------------------------------------------
  * Per-macroblock input.  Mirrors §3 of DESIGN.md.  The caller's
  * libavcodec intercept populates this from the H264SliceContext
@@ -109,6 +149,21 @@ struct daedalus_decoder_mb_input {
      * (the per-frame predicted buffer is zeroed at flush time so a NULL
      * is indistinguishable from explicit zeros). */
     const uint8_t *predicted;    /* NULL or exactly 384 uint8_t */
+
+    /* Per-MB deblock edges — caller-derived per H.264 §8.7.2.  Typical
+     * count: 4 V-luma + 4 H-luma + 2 V-Cb + 2 H-Cb + 2 V-Cr + 2 H-Cr
+     * = 16 edges per MB (omit zero-bS edges if preferred — frame
+     * boundaries MUST be bS=0 since the kernels read p3 at four
+     * samples beyond the edge).  daedalus_decoder routes each entry
+     * to the appropriate luma/chroma × V/H × bS=4/<4 dispatch in
+     * flush_frame and pays a single Vulkan submit per non-empty
+     * (direction × bS-band) partition (≤8 deblock submits / frame
+     * total) per the Q1 architecture decision (one-submit-per-kernel
+     * for now; cmdbuf-builder deferred to Stage 4).
+     *
+     * NULL or n_edges == 0 → no deblock on this MB. */
+    const struct daedalus_decoder_edge *edges;
+    uint8_t                              n_edges;
 };
 
 /* -------------------------------------------------------------------

src/daedalus_decoder.c

@@ -64,8 +64,14 @@ daedalus_decoder *daedalus_decoder_create(int width, int height)
     dec->predicted_y  = calloc(1, pred_y_size);
     dec->predicted_uv = calloc(1, pred_uv_size);
 
+    /* Edge buffer sized for the typical worst case (see daedalus_decoder.h).
+     * 16 edges/MB × n_mbs.  ~130k entries for 1080p; ~2 MB at sizeof(edge). */
+    dec->edges_capacity = (size_t) dec->n_mbs * 16;
+    dec->edges_count    = 0;
+    dec->edges = malloc(dec->edges_capacity * sizeof(*dec->edges));
+
     if (!dec->mb_descs || !dec->coeffs ||
-        !dec->predicted_y || !dec->predicted_uv) {
+        !dec->predicted_y || !dec->predicted_uv || !dec->edges) {
         daedalus_decoder_destroy(dec);
         return NULL;
     }
@@ -77,6 +83,7 @@ void daedalus_decoder_destroy(daedalus_decoder *dec)
 {
     if (!dec)
         return;
+    free(dec->edges);
     free(dec->predicted_uv);
     free(dec->predicted_y);
     free(dec->coeffs);
@@ -194,10 +201,123 @@ int daedalus_decoder_append_mb(daedalus_decoder *dec,
         }
     }
 
+    /* Append per-MB deblock edges into the frame-scoped flat buffer.
+     * Frame-boundary edges (mx=0 V or my=0 H) MUST have bS=0 per the
+     * kernel's p3-at-±4 contract; we don't validate here (caller is
+     * derived from H.264 spec which already enforces this). */
+    if (mb->edges && mb->n_edges > 0) {
+        if (dec->edges_count + mb->n_edges > dec->edges_capacity)
+            return -1;
+        memcpy(&dec->edges[dec->edges_count],
+               mb->edges,
+               mb->n_edges * sizeof(*dec->edges));
+        dec->edges_count += mb->n_edges;
+    }
+
     dec->mbs_appended++;
     return 0;
 }
 
+/* --------------------------------------------------------------------
+ * Deblock helper — walks dec->edges once for a given (plane, orient,
+ * bS_band) selector, builds the corresponding daedalus-fourier
+ * deblock-meta array, and dispatches it through the matching kernel.
+ *
+ * One call → one Vulkan submit, OR zero submits when the selector
+ * matches no edges (a common case for B/P frames with most edges in
+ * bS<4 and only MB-boundary edges in bS=4, or vice versa).
+ *
+ * Edge → dst_off math:
+ *   luma:    px_x = mb_x*16, px_y = mb_y*16, edge step = 4 cells
+ *   chroma:  px_x = mb_x*8,  px_y = mb_y*8,  edge step = 4 cells
+ *            Cb edges land at offset 0..cb_plane in scratch_uv;
+ *            Cr edges land at offset cb_plane..2*cb_plane (planar
+ *            layout matching the chroma IDCT scratch).
+ *
+ * orient == 0 (vertical edge filtered horizontally across):
+ *   dst_off = px_y * stride + px_x + edge_idx * 4
+ *
+ * orient == 1 (horizontal edge filtered vertically across):
+ *   dst_off = (px_y + edge_idx * 4) * stride + px_x
+ *
+ * Edges at frame boundaries (mb_x=0 V, mb_y=0 H with edge_idx=0) MUST
+ * have bS=0 (the kernel reads p3 at four samples beyond the edge);
+ * caller-side spec compliance is assumed, no validation here.
+ *
+ * Returns the dispatch's rc (0 = success; <0 = failure).  No-op when
+ * the selector matches no edges, returning 0.
+ */
+static int dispatch_deblock_pass(
+    daedalus_decoder *dec, daedalus_substrate sub,
+    int target_plane,       /* 0 = luma, 1 = chroma (Cb|Cr by plane field) */
+    int target_orient,      /* 0 = V, 1 = H */
+    int target_bS_intra,    /* 0 = bS<4 path, 1 = bS=4 intra path */
+    uint8_t *scratch, size_t stride,
+    size_t cb_plane_size,   /* chroma: bytes from scratch_uv start to Cr plane (0 for luma calls) */
+    daedalus_h264_deblock_meta *meta_scratch)
+{
+    size_t n = 0;
+    for (size_t i = 0; i < dec->edges_count; i++) {
+        const struct daedalus_decoder_edge *e = &dec->edges[i];
+        if (e->bS == 0) continue;
+        int is_intra = (e->bS == 4) ? 1 : 0;
+        if (is_intra != target_bS_intra) continue;
+        if (e->orient != target_orient) continue;
+        int is_luma = (e->plane == 0) ? 1 : 0;
+        if (is_luma != (target_plane == 0)) continue;
+
+        uint32_t off;
+        if (is_luma) {
+            const size_t px_y = (size_t) e->mb_y * 16;
+            const size_t px_x = (size_t) e->mb_x * 16;
+            if (target_orient == 0)  /* V */
+                off = (uint32_t)(px_y * stride + px_x + (size_t) e->edge_idx * 4);
+            else                      /* H */
+                off = (uint32_t)((px_y + (size_t) e->edge_idx * 4) * stride + px_x);
+        } else {
+            const size_t px_y = (size_t) e->mb_y * 8;
+            const size_t px_x = (size_t) e->mb_x * 8;
+            const size_t plane_base = (e->plane == 2) ? cb_plane_size : 0;
+            if (target_orient == 0)
+                off = (uint32_t)(plane_base + px_y * stride + px_x + (size_t) e->edge_idx * 4);
+            else
+                off = (uint32_t)(plane_base + (px_y + (size_t) e->edge_idx * 4) * stride + px_x);
+        }
+
+        meta_scratch[n].dst_off = off;
+        meta_scratch[n].alpha   = e->alpha;
+        meta_scratch[n].beta    = e->beta;
+        memcpy(meta_scratch[n].tc0, e->tc0, 4);
+        n++;
+    }
+
+    if (n == 0) return 0;
+
+    typedef int (*deblock_dispatch_fn)(
+        daedalus_ctx *, daedalus_substrate,
+        uint8_t *, size_t, size_t,
+        const daedalus_h264_deblock_meta *);
+
+    deblock_dispatch_fn fn;
+    if (target_plane == 0) {
+        if (target_orient == 0)
+            fn = target_bS_intra ? daedalus_dispatch_h264_deblock_luma_v_intra
+                                 : daedalus_dispatch_h264_deblock_luma_v;
+        else
+            fn = target_bS_intra ? daedalus_dispatch_h264_deblock_luma_h_intra
+                                 : daedalus_dispatch_h264_deblock_luma_h;
+    } else {
+        if (target_orient == 0)
+            fn = target_bS_intra ? daedalus_dispatch_h264_deblock_chroma_v_intra
+                                 : daedalus_dispatch_h264_deblock_chroma_v;
+        else
+            fn = target_bS_intra ? daedalus_dispatch_h264_deblock_chroma_h_intra
+                                 : daedalus_dispatch_h264_deblock_chroma_h;
+    }
+
+    return fn(dec->dctx, sub, scratch, stride, n, meta_scratch);
+}
+
 /* Phase 1 stage 1 — frame-scaled IDCT 4x4 dispatch (luma + chroma).
  *
  * Brings up the GPU substrate by calling daedalus-fourier's existing
@@ -362,6 +482,33 @@ int daedalus_decoder_flush_frame(daedalus_decoder *dec,
         if (dr != 0) { rc = -3; goto cleanup; }
     }
 
+    /* ---- Luma deblock V then H ----
+     * Per H.264 §8.7 deblock order is V edges first, then H edges,
+     * within each MB.  At frame scale we hit the same dependency: a
+     * row of V-filtered samples is the input to the H filter for
+     * the row's H edges.  Order: V bS<4 + V bS=4 (independent edges,
+     * either order), barrier (implicit at each dispatch's wait), then
+     * H bS<4 + H bS=4. */
+    daedalus_h264_deblock_meta *dbk_meta = NULL;
+    if (dec->edges_count > 0) {
+        dbk_meta = malloc(dec->edges_count * sizeof(*dbk_meta));
+        if (!dbk_meta) { rc = -1; goto cleanup; }
+
+        int dr;
+        dr = dispatch_deblock_pass(dec, sub, 0, 0, 0,
+                                    scratch_y, y_stride_int, 0, dbk_meta);
+        if (dr != 0) { rc = -3; goto cleanup; }
+        dr = dispatch_deblock_pass(dec, sub, 0, 0, 1,
+                                    scratch_y, y_stride_int, 0, dbk_meta);
+        if (dr != 0) { rc = -3; goto cleanup; }
+        dr = dispatch_deblock_pass(dec, sub, 0, 1, 0,
+                                    scratch_y, y_stride_int, 0, dbk_meta);
+        if (dr != 0) { rc = -3; goto cleanup; }
+        dr = dispatch_deblock_pass(dec, sub, 0, 1, 1,
+                                    scratch_y, y_stride_int, 0, dbk_meta);
+        if (dr != 0) { rc = -3; goto cleanup; }
+    }
+
     /* ---- Copy Y out to caller's plane at the requested stride. ---- */
     for (int r = 0; r < dec->height; r++)
         memcpy(out_y + (size_t) r * y_stride,
@@ -455,6 +602,30 @@ int daedalus_decoder_flush_frame(daedalus_decoder *dec,
             goto chroma_cleanup;
         }
 
+        /* ---- Chroma deblock V then H ----
+         * scratch_uv is PLANAR Cb||Cr with stride = chroma_w; both
+         * planes filtered in the same dispatch via Cb's dst_off and
+         * Cr's dst_off = cb_plane_size + (same). */
+        if (dec->edges_count > 0 && dbk_meta) {
+            int dr;
+            dr = dispatch_deblock_pass(dec, sub, 1, 0, 0,
+                                        scratch_uv, chroma_w,
+                                        cb_plane_size, dbk_meta);
+            if (dr != 0) { rc = -3; goto chroma_cleanup; }
+            dr = dispatch_deblock_pass(dec, sub, 1, 0, 1,
+                                        scratch_uv, chroma_w,
+                                        cb_plane_size, dbk_meta);
+            if (dr != 0) { rc = -3; goto chroma_cleanup; }
+            dr = dispatch_deblock_pass(dec, sub, 1, 1, 0,
+                                        scratch_uv, chroma_w,
+                                        cb_plane_size, dbk_meta);
+            if (dr != 0) { rc = -3; goto chroma_cleanup; }
+            dr = dispatch_deblock_pass(dec, sub, 1, 1, 1,
+                                        scratch_uv, chroma_w,
+                                        cb_plane_size, dbk_meta);
+            if (dr != 0) { rc = -3; goto chroma_cleanup; }
+        }
+
         /* CPU NV12 interleave: out_uv[r][2c+0] = Cb[r][c], [2c+1] = Cr. */
         const uint8_t *cb_plane = scratch_uv;
         const uint8_t *cr_plane = scratch_uv + cb_plane_size;
@@ -477,6 +648,7 @@ int daedalus_decoder_flush_frame(daedalus_decoder *dec,
     }
 
 cleanup:
+    free(dbk_meta);
     free(meta8);
     free(meta4);
     free(coeffs8);
@@ -491,6 +663,9 @@ cleanup:
     if (dec->predicted_uv)
         memset(dec->predicted_uv, 0, (size_t) dec->width * (size_t) dec->height / 2);
 
+    /* Reset edges_count for the next frame; capacity stays. */
+    dec->edges_count = 0;
+
     dec->mbs_appended = 0;
     return rc;
 }

src/internal.h

@@ -76,6 +76,15 @@ struct daedalus_decoder {
     uint8_t *predicted_y;
     uint8_t *predicted_uv;
 
+    /* Per-frame flat deblock-edge buffer, accumulated by append_mb's
+     * `edges` array and consumed by flush_frame.  Capacity is sized
+     * for the typical maximum of 16 edges/MB (4 V-luma + 4 H-luma +
+     * 2 V-Cb + 2 H-Cb + 2 V-Cr + 2 H-Cr — see daedalus_decoder.h).
+     * Overflow returns -1 from append_mb. */
+    struct daedalus_decoder_edge *edges;
+    size_t                        edges_capacity;  /* allocated entries */
+    size_t                        edges_count;     /* used entries this frame */
+
     /* Output format. */
     daedalus_decoder_output_format   output_fmt;