iter28b DIAG: env-gated trim of HEVC slice_data trailing N bytes

src/picture.c

@@ -61,13 +61,32 @@ static VAStatus codec_store_buffer(struct request_data *driver_data,
 				   struct object_buffer *buffer_object)
 {
 	switch (buffer_object->type) {
-	case VASliceDataBufferType:
+	case VASliceDataBufferType: {
 		/*
 		 * Since there is no guarantee that the allocation
 		 * order is the same as the submission order (via
 		 * RenderPicture), we can't use a V4L2 buffer directly
 		 * and have to copy from a regular buffer.
 		 */
+		size_t slice_bytes = buffer_object->size * buffer_object->count;
+
+		/*
+		 * iter28b DIAG (env-gated): trim trailing N bytes from
+		 * VAAPI's slice_data buffer for HEVC. Tests the hypothesis
+		 * that ffmpeg-vaapi inflates slice_data_size vs
+		 * ffmpeg-v4l2request. Set LIBVA_HEVC_TRIM_TRAILING=N to
+		 * trim. For BBB frame 2 with libva 5549 vs kdirect 5509:
+		 * try N=40.
+		 */
+		if (profile == VAProfileHEVCMain) {
+			const char *trim_env = getenv("LIBVA_HEVC_TRIM_TRAILING");
+			if (trim_env) {
+				unsigned long t = strtoul(trim_env, NULL, 0);
+				if (t < slice_bytes)
+					slice_bytes -= t;
+			}
+		}
+
 		if (context->h264_start_code) {
 			static const char start_code[3] = { 0x00, 0x00, 0x01 };
 
@@ -79,11 +98,11 @@ static VAStatus codec_store_buffer(struct request_data *driver_data,
 		memcpy(surface_object->source_data +
 			       surface_object->slices_size,
 		       buffer_object->data,
-		       buffer_object->size * buffer_object->count);
-		surface_object->slices_size +=
-			buffer_object->size * buffer_object->count;
+		       slice_bytes);
+		surface_object->slices_size += slice_bytes;
 		surface_object->slices_count++;
 		break;
+	}
 
 	case VAPictureParameterBufferType:
 		switch (profile) {