daedalus-fourier/src/v3d_h264deblock.comp

// daedalus-fourier cycle 8 — H.264 luma "v_loop_filter" (vertical
// filtering across a horizontal edge), non-intra bS<4 variant.
// V3D 7.1 via Mesa v3dv compute.
//
// Per cycle 8 Phase 4 plan + Phase 5 Sonnet review fixes:
//   - 256 invocations / WG, 16 edges/WG (16 lanes/edge = 1 sg/edge)
//   - uint8_t dst SSBO via storageBuffer8BitAccess
//   - No barrier (each lane independent)
//   - Multiple early returns SAFE (no barrier follows; Phase 5 GREEN-3)
//   - RED-1: clamp p1', q1' to [0,255] before write (matching p0', q0')
//   - RED-2: contract m.x >= 4*stride enforced by bench
//
// Filter contract (per H.264 §8.7.2.4):
//   1. m.x ≥ 4 * pc.dst_stride_u8 (bench-enforced; reads p3 at -4*stride)
//   2. pc.dst_stride_u8 = byte stride between rows
//   3. tc0_s pre-stored as signed int8 in m.z packed 4 bytes
//
// License: BSD-2-Clause. Algorithm transcribed from tests/h264_deblock_ref.c
// which mirrors FFmpeg ff_h264_v_loop_filter_luma_neon (LGPL-2.1+).

#version 450
#extension GL_EXT_shader_8bit_storage              : require
#extension GL_EXT_shader_explicit_arithmetic_types : require

layout(local_size_x = 256, local_size_y = 1, local_size_z = 1) in;

layout(binding = 0) readonly buffer Meta {
    uvec4 meta[];   // per edge: (dst_off, alpha|beta<<8, packed_tc0, _pad)
} u_meta;

layout(binding = 1) buffer Dst {
    uint8_t dst[];
} u_dst;

layout(push_constant) uniform PC {
    uint n_edges;
    uint dst_stride_u8;
    uint _pad0;
    uint _pad1;
} pc;

void main()
{
    uint gid          = gl_GlobalInvocationID.x;
    uint wg_id        = gl_WorkGroupID.x;
    uint lane_in_wg   = gid & 255u;
    uint edge_in_wg   = lane_in_wg >> 4;       // 0..15 (16 edges/WG)
    uint col_in_edge  = lane_in_wg & 15u;      // 0..15

    uint edge_idx = wg_id * 16u + edge_in_wg;
    if (edge_idx >= pc.n_edges) return;        // safe — no barrier follows

    uvec4 m = u_meta.meta[edge_idx];
    uint dst_off = m.x + col_in_edge;
    uint stride  = pc.dst_stride_u8;
    int alpha = int(m.y & 0xffu);
    int beta  = int((m.y >> 8) & 0xffu);

    // Unpack tc0[seg] from packed int8 (4 in low 32 bits of m.z).
    uint seg = col_in_edge >> 2;
    uint tc0_byte = (m.z >> (seg * 8u)) & 0xffu;
    int tc0_s = int(tc0_byte);
    if (tc0_s >= 128) tc0_s -= 256;            // two's-complement sign-extend

    if (alpha == 0 || beta == 0) return;
    if (tc0_s < 0) return;                     // segment skip

    // Read 8 rows of vertical context at this column.
    // (p3 unused in bS<4 path; compiler will DCE if we skip it. Kept for
    // clarity. Per Phase 5 GREEN-6, can be omitted as a micro-opt.)
    int p2 = int(u_dst.dst[dst_off - 3u * stride]);
    int p1 = int(u_dst.dst[dst_off - 2u * stride]);
    int p0 = int(u_dst.dst[dst_off - 1u * stride]);
    int q0 = int(u_dst.dst[dst_off]);
    int q1 = int(u_dst.dst[dst_off + 1u * stride]);
    int q2 = int(u_dst.dst[dst_off + 2u * stride]);

    // Edge preconditions.
    if (abs(p0 - q0) >= alpha) return;
    if (abs(p1 - p0) >= beta)  return;
    if (abs(q1 - q0) >= beta)  return;

    int ap = abs(p2 - p0);
    int aq = abs(q2 - q0);
    bool ap_lt = ap < beta;
    bool aq_lt = aq < beta;
    int tc = tc0_s + int(ap_lt) + int(aq_lt);  // tc >= 0 (tc0_s >= 0)

    int delta = clamp(((q0 - p0) * 4 + (p1 - q1) + 4) >> 3, -tc, tc);
    int p0p = clamp(p0 + delta, 0, 255);
    int q0p = clamp(q0 - delta, 0, 255);

    int p1p = p1;
    if (ap_lt) {
        int d_p1 = clamp((p2 + ((p0 + q0 + 1) >> 1) - 2*p1) >> 1, -tc0_s, tc0_s);
        p1p = clamp(p1 + d_p1, 0, 255);        // RED-1: explicit clip
    }
    int q1p = q1;
    if (aq_lt) {
        int d_q1 = clamp((q2 + ((p0 + q0 + 1) >> 1) - 2*q1) >> 1, -tc0_s, tc0_s);
        q1p = clamp(q1 + d_q1, 0, 255);        // RED-1: explicit clip
    }

    u_dst.dst[dst_off - 2u * stride] = uint8_t(p1p);
    u_dst.dst[dst_off - 1u * stride] = uint8_t(p0p);
    u_dst.dst[dst_off            ]  = uint8_t(q0p);
    u_dst.dst[dst_off + 1u * stride] = uint8_t(q1p);
}