h264: max_num_ref_frames fallback + libva-boundary instrumentation

Closes the libva-side portion of marfrit/libva-v4l2-request-fourier#8.

Two small additions to h264_set_controls:

1. When VAPicture->num_ref_frames is 0 (older ffmpeg-vaapi paths /
   some daedalus_v4l2 consumers), count valid (non-INVALID) DPB
   entries in ReferenceFrames[16]. If even that returns 0, fall back
   to a per-profile spec minimum (1 for baseline, 4 for main/high).
   Hardware decoders (rkvdec, hantro, rpi-hevc-dec) tolerated the
   prior 0; libavcodec-via-daedalus enforces sps.max_num_ref_frames
   strictly and rejected every frame.

2. One request_log line at function entry dumping the raw VAAPI
   fields (seq_fields.value, pic_fields.value, num_ref_frames,
   bit_depth_*, picture_*_in_mbs_minus1). Disambiguates "ffmpeg-vaapi
   never populated" from "daedalus_v4l2 wire protocol corrupted" for
   the bit-fields-read-as-zero portion of issue #8.

Out of scope here (separate issue if pursued): profile_idc and
level_idc remain session-derived. VAAPI's VAPictureParameterBufferH264
omits both (verified higgs libva 2.22.0-3, /usr/include/va/va.h:
3571-3622) — same VAAPI-blindspot family as the HEVC SPS fields. A
real fix requires SPS-NAL parsing from surface->source_data OR a
daedalus wire-protocol pass-through; both are operator design calls,
not a libva-only patch.

Build verified on higgs (Debian 13 trixie, gcc 14.2.0, libva 2.22.0):
clean ninja link of v4l2_request_drv_video.so, vainfo enumerates all
8 codec profiles, no init regression.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>

src/config.c

@@ -172,15 +172,20 @@ VAStatus RequestDestroyConfig(VADriverContextP context, VAConfigID config_id)
 static bool any_fd_supports_output_format(struct request_data *driver_data,
 					  unsigned int fmt)
 {
-	int fds[5] = {
+	int fds[6] = {
 		driver_data->video_fd,
 		driver_data->video_fd_rkvdec,
 		driver_data->video_fd_hantro,
 		driver_data->video_fd_rpi_hevc_dec,  /* iter40 */
 		driver_data->video_fd_vpu981,        /* ampere-av1 Phase 2 */
+#ifdef HAVE_DAEDALUS_V4L2
+		driver_data->video_fd_daedalus,      /* LIBVA-1: H.264/VP9/AV1 */
+#else
+		-1,
+#endif
 	};
 	int i;
-	for (i = 0; i < 5; i++) {
+	for (i = 0; i < 6; i++) {
 		if (fds[i] < 0) continue;
 		if (v4l2_find_format(fds[i], V4L2_BUF_TYPE_VIDEO_OUTPUT, fmt))
 			return true;

src/h264.c

@@ -827,10 +827,63 @@ int h264_set_controls(struct request_data *driver_data,
 
 	dpb_update(context, &surface->params.h264.picture);
 
+	/*
+	 * Dump the raw VAAPI fields at the libva boundary so issue #8
+	 * follow-up can disambiguate "ffmpeg-vaapi didn't populate" from
+	 * "downstream consumer (daedalus_v4l2 wire protocol) corrupted the
+	 * value". One-line; safe to leave in — costs a single printf per frame.
+	 */
+	request_log("h264_set_controls: VAProfile=%d seq_fields=0x%08x pic_fields=0x%08x num_ref_frames=%u bit_depth_luma_m8=%u bit_depth_chroma_m8=%u w_mbs_m1=%u h_mbs_m1=%u\n",
+		    (int)profile,
+		    surface->params.h264.picture.seq_fields.value,
+		    surface->params.h264.picture.pic_fields.value,
+		    surface->params.h264.picture.num_ref_frames,
+		    surface->params.h264.picture.bit_depth_luma_minus8,
+		    surface->params.h264.picture.bit_depth_chroma_minus8,
+		    surface->params.h264.picture.picture_width_in_mbs_minus1,
+		    surface->params.h264.picture.picture_height_in_mbs_minus1);
+
 	h264_va_picture_to_v4l2(driver_data, context, surface,
 				&surface->params.h264.picture,
 				&decode, &pps, &sps);
 
+	/*
+	 * max_num_ref_frames fallback. Some VAAPI clients (older ffmpeg-vaapi
+	 * paths, some daedalus_v4l2 consumers) leave VAPicture->num_ref_frames
+	 * at zero. Hardware decoders tolerate; libavcodec-via-daedalus enforces
+	 * sps.max_num_ref_frames strictly and rejects every frame.
+	 *
+	 * Count valid DPB entries first (the bitstream-true reference count we
+	 * can see); fall back to a per-profile spec minimum if even that is 0.
+	 * See marfrit/libva-v4l2-request-fourier issue #8.
+	 */
+	if (sps.max_num_ref_frames == 0) {
+		unsigned int valid = 0;
+		unsigned int i;
+		for (i = 0; i < 16; i++) {
+			const VAPictureH264 *ref =
+				&surface->params.h264.picture.ReferenceFrames[i];
+			if (!(ref->flags & VA_PICTURE_H264_INVALID))
+				valid++;
+		}
+		if (valid > 0) {
+			sps.max_num_ref_frames = (uint8_t)valid;
+		} else {
+			switch (profile) {
+			case VAProfileH264ConstrainedBaseline:
+				sps.max_num_ref_frames = 1;
+				break;
+			case VAProfileH264Main:
+			case VAProfileH264High:
+			case VAProfileH264MultiviewHigh:
+			case VAProfileH264StereoHigh:
+			default:
+				sps.max_num_ref_frames = 4;
+				break;
+			}
+		}
+	}
+
 	/*
 	 * Populate the scaling matrix unconditionally: from VAAPI's
 	 * VAIQMatrixBufferH264 when the consumer sent one this frame

src/request.c

@@ -461,6 +461,44 @@ int request_switch_device_for_profile(struct request_data *driver_data,
 		kind = 'p';
 	}
 
+#ifdef HAVE_DAEDALUS_V4L2
+	/*
+	 * LIBVA-1: VP9/AV1/H.264 → daedalus_v4l2 when the daemon-backed
+	 * decoder fd is open.  Pi 5 has no rkvdec (those profiles map to
+	 * 'r' by default → video_fd_rkvdec = -1 → "stay on whatever's
+	 * active" fallback would put H.264 frames on rpi-hevc-dec's fd
+	 * and S_FMT would fail).  Re-route to the daedalus daemon instead.
+	 *
+	 * HEVC stays on 'p' (rpi-hevc-dec is HEVC-only — daedalus would
+	 * accept it via FFmpeg, but rpi-hevc-dec has the GPU-backed
+	 * hardware path so it's the right choice on this SoC).
+	 *
+	 * AV1 'a' kind (RK3588 vpu981) wins ONLY if vpu981 was probed.
+	 * On a Pi 5 the vpu981 slot stays -1, so we still route AV1 to
+	 * daedalus here.  Check video_fd_vpu981 to preserve the RK3588
+	 * priority for that case.
+	 */
+	if (driver_data->video_fd_daedalus >= 0 &&
+	    driver_data->media_fd_daedalus >= 0) {
+		switch (profile) {
+		case VAProfileH264Main:
+		case VAProfileH264High:
+		case VAProfileH264ConstrainedBaseline:
+		case VAProfileH264MultiviewHigh:
+		case VAProfileH264StereoHigh:
+		case VAProfileVP9Profile0:
+			kind = 'd';
+			break;
+		case VAProfileAV1Profile0:
+			if (driver_data->video_fd_vpu981 < 0)
+				kind = 'd';
+			break;
+		default:
+			break;
+		}
+	}
+#endif
+
 	if (kind == 'r') {
 		target_video = driver_data->video_fd_rkvdec;
 		target_media = driver_data->media_fd_rkvdec;
@@ -473,6 +511,11 @@ int request_switch_device_for_profile(struct request_data *driver_data,
 	} else if (kind == 'a') {
 		target_video = driver_data->video_fd_vpu981;
 		target_media = driver_data->media_fd_vpu981;
+#ifdef HAVE_DAEDALUS_V4L2
+	} else if (kind == 'd') {
+		target_video = driver_data->video_fd_daedalus;
+		target_media = driver_data->media_fd_daedalus;
+#endif
 	} else {
 		return -1;
 	}
@@ -697,26 +740,32 @@ VAStatus VA_DRIVER_INIT_FUNC(VADriverContextP context)
 				driver_data->video_fd_hantro = video_fd;
 				driver_data->media_fd_hantro = media_fd;
 			} else if (strcmp(info.driver, "rpi-hevc-dec") == 0) {
-				/* iter40: Pi 5 / CM5 — sole decoder is rpi-hevc-dec.
-				 * No alt driver to probe; the rkvdec / hantro slots
-				 * stay -1 and HEVC routes to 'p' via
-				 * request_device_kind_for_profile. */
+				/* iter40 + LIBVA-1: Pi 5 / CM5.  rpi-hevc-dec is
+				 * HEVC-only.  If daedalus_v4l2 is ALSO loaded (Pi 5
+				 * mixed deployment — out-of-tree daemon-backed
+				 * decoder for VP9/AV1/H264), pick it up as the alt
+				 * so VP9/AV1/H264 have somewhere to land. */
 				primary_driver = "rpi-hevc-dec";
+#ifdef HAVE_DAEDALUS_V4L2
+				alt_driver = "daedalus_v4l2";
+#else
 				alt_driver = NULL;
+#endif
 				driver_data->video_fd_rpi_hevc_dec = video_fd;
 				driver_data->media_fd_rpi_hevc_dec = media_fd;
 #ifdef HAVE_DAEDALUS_V4L2
 			} else if (strcmp(info.driver, "daedalus_v4l2") == 0) {
-				/* phase 8.10: Pi 5 daemon-backed decoder.  Sole
-				 * V4L2 stateless slot on this kernel; VP9 / AV1 /
-				 * H.264 all route through it.  Other slots stay -1.
-				 *
-				 * On a mixed-driver box (daedalus loaded ALONGSIDE
-				 * rpi-hevc-dec) HEVC would prefer rpi-hevc-dec via
-				 * the existing 'p' override; VP9/AV1/H264 prefer
-				 * daedalus_v4l2 since rpi-hevc-dec is HEVC-only. */
+				/* phase 8.10 + LIBVA-1: Pi 5 daemon-backed decoder.
+				 * VP9 / AV1 / H.264 route through it via the 'd'
+				 * kind below.  On a mixed-driver box where
+				 * rpi-hevc-dec is ALSO loaded, pick it up as the
+				 * alt so HEVC has somewhere to land too — find_
+				 * codec_device's known_decoder_drivers[] order
+				 * normally puts rpi-hevc-dec first (we hit the
+				 * other branch in practice), but symmetric handling
+				 * keeps us correct if probe order ever flips. */
 				primary_driver = "daedalus_v4l2";
-				alt_driver = NULL;
+				alt_driver = "rpi-hevc-dec";
 				driver_data->video_fd_daedalus = video_fd;
 				driver_data->media_fd_daedalus = media_fd;
 #endif
@@ -731,15 +780,38 @@ VAStatus VA_DRIVER_INIT_FUNC(VADriverContextP context)
 				int alt_v = open(alt_video, O_RDWR | O_NONBLOCK);
 				int alt_m = (alt_v >= 0) ? open(alt_media, O_RDWR | O_NONBLOCK) : -1;
 				if (alt_v >= 0 && alt_m >= 0) {
+					/* Dispatch into the matching per-driver slot.
+					 * iter38 only had rkvdec/hantro pairs; iter40 +
+					 * LIBVA-1 extended this to rpi-hevc-dec and
+					 * daedalus_v4l2 for the Pi 5 mixed-decoder
+					 * deployment. */
 					if (strcmp(alt_driver, "rkvdec") == 0) {
 						driver_data->video_fd_rkvdec = alt_v;
 						driver_data->media_fd_rkvdec = alt_m;
-					} else {
+					} else if (strcmp(alt_driver, "hantro-vpu") == 0) {
 						driver_data->video_fd_hantro = alt_v;
 						driver_data->media_fd_hantro = alt_m;
+					} else if (strcmp(alt_driver, "rpi-hevc-dec") == 0) {
+						driver_data->video_fd_rpi_hevc_dec = alt_v;
+						driver_data->media_fd_rpi_hevc_dec = alt_m;
+#ifdef HAVE_DAEDALUS_V4L2
+					} else if (strcmp(alt_driver, "daedalus_v4l2") == 0) {
+						driver_data->video_fd_daedalus = alt_v;
+						driver_data->media_fd_daedalus = alt_m;
+#endif
+					} else {
+						/* Shouldn't happen — primary_driver branches
+						 * above only set alt_driver to one of the
+						 * names handled here.  Close and move on. */
+						close(alt_v);
+						close(alt_m);
+						alt_v = -1;
+						alt_m = -1;
+					}
+					if (alt_v >= 0) {
+						request_log("iter38: also opened %s decoder at %s + %s\n",
+							    alt_driver, alt_video, alt_media);
 					}
-					request_log("iter38: also opened %s decoder at %s + %s\n",
-						    alt_driver, alt_video, alt_media);
 				} else {
 					if (alt_v >= 0) close(alt_v);
 					if (alt_m >= 0) close(alt_m);
@@ -890,6 +962,12 @@ VAStatus RequestTerminate(VADriverContextP context)
 		close(driver_data->video_fd_vpu981);
 	if (driver_data->media_fd_vpu981 >= 0)
 		close(driver_data->media_fd_vpu981);
+#ifdef HAVE_DAEDALUS_V4L2
+	if (driver_data->video_fd_daedalus >= 0)
+		close(driver_data->video_fd_daedalus);
+	if (driver_data->media_fd_daedalus >= 0)
+		close(driver_data->media_fd_daedalus);
+#endif
 	/* Fall back to direct close if neither alt fd captured the active
 	 * pair (env-override path). */
 	if (driver_data->video_fd_rkvdec < 0 && driver_data->video_fd_hantro < 0) {