Commit 1f5ad103 authored by François Grisez's avatar François Grisez

Prevent VTH264Decoder from crashing when receiving an i-frame with missing SPS or PPS

parent 48e77581
......@@ -469,7 +469,7 @@ typedef struct _VTH264DecCtx {
MSFilter *f;
} VTH264DecCtx;
static bool_t h264_dec_update_format_description(VTH264DecCtx *ctx, const MSList *parameter_sets) {
static CMFormatDescriptionRef format_desc_from_sps_pps(const MSList *parameter_sets) {
const MSList *it;
const size_t max_ps_count = 20;
size_t ps_count;
......@@ -478,13 +478,12 @@ static bool_t h264_dec_update_format_description(VTH264DecCtx *ctx, const MSList
int sps_count = 0, pps_count = 0;
int i;
OSStatus status;
CMFormatDescriptionRef format_desc;
if(ctx->format_desc) CFRelease(ctx->format_desc);
ctx->format_desc = NULL;
ps_count = ms_list_size(parameter_sets);
if(ps_count > max_ps_count) {
ms_error("VideoToolboxDec: too much SPS/PPS");
return FALSE;
return NULL;
}
for(it=parameter_sets,i=0; it; it=it->next,i++) {
mblk_t *m = (mblk_t *)it->data;
......@@ -495,18 +494,18 @@ static bool_t h264_dec_update_format_description(VTH264DecCtx *ctx, const MSList
}
if(sps_count==0) {
ms_error("VideoToolboxDec: no SPS");
return FALSE;
return NULL;
}
if(pps_count==0) {
ms_error("VideoToolboxDec: no PPS");
return FALSE;
return NULL;
}
status = CMVideoFormatDescriptionCreateFromH264ParameterSets(NULL, ps_count, ps_ptrs, ps_sizes, H264_NALU_HEAD_SIZE, &ctx->format_desc);
status = CMVideoFormatDescriptionCreateFromH264ParameterSets(NULL, ps_count, ps_ptrs, ps_sizes, H264_NALU_HEAD_SIZE, &format_desc);
if(status != noErr) {
ms_error("VideoToolboxDec: could not find out the input format: %d", (int)status);
return FALSE;
return NULL;
}
return TRUE;
return format_desc;
}
static void h264_dec_output_cb(VTH264DecCtx *ctx, void *sourceFrameRefCon,
......@@ -582,8 +581,8 @@ static bool_t h264_dec_init_decoder(VTH264DecCtx *ctx) {
static void h264_dec_uninit_decoder(VTH264DecCtx *ctx) {
ms_message("VideoToolboxDecoder: uninitializing decoder");
VTDecompressionSessionInvalidate(ctx->session);
if(ctx->session) CFRelease(ctx->session);
if(ctx->format_desc) CFRelease(ctx->format_desc);
CFRelease(ctx->session);
CFRelease(ctx->format_desc);
ctx->session = NULL;
ctx->format_desc = NULL;
}
......@@ -645,22 +644,19 @@ static void h264_dec_process(MSFilter *f) {
if(iframe_received) ms_message("VideoToolboxDecoder: I-frame received");
if(parameter_sets) {
CMFormatDescriptionRef last_format = ctx->format_desc ? CFRetain(ctx->format_desc) : NULL;
h264_dec_update_format_description(ctx, parameter_sets);
CMFormatDescriptionRef new_format_desc = format_desc_from_sps_pps(parameter_sets);
parameter_sets = ms_list_free_with_data(parameter_sets, (void (*)(void *))freemsg);
if(ctx->format_desc == NULL) goto fail;
if(last_format) {
CMVideoDimensions last_vsize = CMVideoFormatDescriptionGetDimensions(last_format);
CMVideoDimensions vsize = CMVideoFormatDescriptionGetDimensions(ctx->format_desc);
if(last_vsize.width != vsize.width || last_vsize.height != vsize.height) {
if(ctx->format_desc) {
CMVideoDimensions last_vsize = CMVideoFormatDescriptionGetDimensions(ctx->format_desc);
CMVideoDimensions new_vsize = CMVideoFormatDescriptionGetDimensions(new_format_desc);
if(last_vsize.width != new_vsize.width || last_vsize.height != new_vsize.height) {
ms_message("VideoToolboxDecoder: new encoded video size %dx%d -> %dx%d",
(int)last_vsize.width, (int)last_vsize.height, (int)vsize.width, (int)vsize.height);
ms_message("VideoToolboxDecoder: destroying decoding session");
VTDecompressionSessionInvalidate(ctx->session);
CFRelease(ctx->session);
ctx->session = NULL;
(int)last_vsize.width, (int)last_vsize.height, (int)new_vsize.width, (int)new_vsize.height);
h264_dec_uninit_decoder(ctx);
ctx->format_desc = new_format_desc;
}
CFRelease(last_format);
} else {
ctx->format_desc = new_format_desc;
}
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment