picture: bounds-check codec_store_buffer slice writes against source_size

surface_object->source_data points at an OUTPUT-pool mmap of fixed
size source_size, negotiated by v4l2_query_buffer at request_pool_init
time (kernel sizeimage at S_FMT).  codec_store_buffer's
VASliceDataBufferType branch appended to it at three sites (H.264 Annex-B
start code, VP8 uncompressed-header pad, slice payload) without
consulting that capacity — a stream-level resolution upshift would walk
past the mmap and SIGSEGV inside the memcpy (mpv --hwdec=vaapi-copy on
the daedalus path, issue #13) or corrupt adjacent heap (Firefox RDD).

Add a check at each append site that fails the RenderPicture call with
VA_STATUS_ERROR_ALLOCATION_FAILED when slices_size+payload exceeds
source_size, and logs the over-budget request for postmortem.
libavcodec recreates the surface at the new dimensions on the next
BeginPicture, so a refused upshift slice is recoverable.

Doesn't address the root cause (surfaces should be re-created on
resolution change, or source_data should be grown on demand) but
removes the memory-safety hazard while the larger refactor waits.

Closes marfrit/libva-v4l2-request-fourier#13.
This commit is contained in:
2026-05-21 12:14:48 +02:00
parent 77f9236466
commit bfcb286031
+43 -7
View File
@@ -62,16 +62,37 @@ static VAStatus codec_store_buffer(struct request_data *driver_data,
struct object_buffer *buffer_object) struct object_buffer *buffer_object)
{ {
switch (buffer_object->type) { switch (buffer_object->type) {
case VASliceDataBufferType: case VASliceDataBufferType: {
/* /*
* Since there is no guarantee that the allocation * Since there is no guarantee that the allocation
* order is the same as the submission order (via * order is the same as the submission order (via
* RenderPicture), we can't use a V4L2 buffer directly * RenderPicture), we can't use a V4L2 buffer directly
* and have to copy from a regular buffer. * and have to copy from a regular buffer.
*
* Bounds check (issue #13): surface_object->source_data points
* at an OUTPUT-pool mmap of fixed size source_size, negotiated
* at S_FMT time. A stream-level resolution upshift can produce
* a slice larger than this allocation; without the guard, the
* memcpy walks past the mmap and SIGSEGVs (mpv --hwdec=vaapi-
* copy) or corrupts adjacent heap (Firefox RDD). Each append
* site below checks the running total against source_size and
* fails the RenderPicture call instead of corrupting memory;
* libavcodec re-creates the surface at the new resolution on
* the next BeginPicture.
*/ */
size_t cap = surface_object->source_size;
size_t need;
if (context->h264_start_code) { if (context->h264_start_code) {
static const char start_code[3] = { 0x00, 0x00, 0x01 }; static const char start_code[3] = { 0x00, 0x00, 0x01 };
need = (size_t)surface_object->slices_size +
sizeof(start_code);
if (need > cap) {
request_log("codec_store_buffer: H.264 start code would overflow OUTPUT buffer (%zu > %zu) — resolution upshift mid-stream?\n",
need, cap);
return VA_STATUS_ERROR_ALLOCATION_FAILED;
}
memcpy(surface_object->source_data + memcpy(surface_object->source_data +
surface_object->slices_size, surface_object->slices_size,
start_code, sizeof(start_code)); start_code, sizeof(start_code));
@@ -105,19 +126,34 @@ static VAStatus codec_store_buffer(struct request_data *driver_data,
unsigned int header_size = unsigned int header_size =
surface_object->params.vp8.picture.pic_fields.bits.key_frame == 0 ? surface_object->params.vp8.picture.pic_fields.bits.key_frame == 0 ?
10 : 3; 10 : 3;
need = (size_t)surface_object->slices_size + header_size;
if (need > cap) {
request_log("codec_store_buffer: VP8 header pad would overflow OUTPUT buffer (%zu > %zu)\n",
need, cap);
return VA_STATUS_ERROR_ALLOCATION_FAILED;
}
memset(surface_object->source_data + memset(surface_object->source_data +
surface_object->slices_size, surface_object->slices_size,
0, header_size); 0, header_size);
surface_object->slices_size += header_size; surface_object->slices_size += header_size;
} }
memcpy(surface_object->source_data + {
surface_object->slices_size, size_t payload = (size_t)buffer_object->size *
buffer_object->data, buffer_object->count;
buffer_object->size * buffer_object->count); need = (size_t)surface_object->slices_size + payload;
surface_object->slices_size += if (need > cap) {
buffer_object->size * buffer_object->count; request_log("codec_store_buffer: slice payload would overflow OUTPUT buffer (%zu > %zu) — resolution upshift mid-stream?\n",
need, cap);
return VA_STATUS_ERROR_ALLOCATION_FAILED;
}
memcpy(surface_object->source_data +
surface_object->slices_size,
buffer_object->data, payload);
surface_object->slices_size += payload;
}
surface_object->slices_count++; surface_object->slices_count++;
break; break;
}
case VAPictureParameterBufferType: case VAPictureParameterBufferType:
switch (profile) { switch (profile) {