Commit eb5fdf10 authored by Simon Morlat's avatar Simon Morlat

enhance VideoToolbox decoder to be more reliable by using the new unpacker API

parent 4c4d1041
......@@ -617,121 +617,130 @@ static void h264_dec_process(MSFilter *f) {
MSPicture pixbuf_desc;
OSStatus status;
bctbx_list_t *parameter_sets = NULL;
bool_t unpacking_failed;
unsigned int res;
bool_t iframe_received = FALSE;
bool_t need_pli = FALSE;
ms_queue_init(&q_nalus);
ms_queue_init(&q_nalus2);
// unpack RTP packets
unpacking_failed = FALSE;
while((pkt = ms_queue_get(f->inputs[0]))) {
unpacking_failed |= (rfc3984_unpack(&ctx->unpacker, pkt, &q_nalus) != 0);
}
if (unpacking_failed) {
ms_warning("VideoToolboxDecoder: error while unpacking RTP packets");
if(ctx->freeze_on_error_enabled) {
ctx->freezed = TRUE;
goto fail;
}
}
// Pull SPSs and PPSs out and put them into the filter context if necessary
while((nalu = ms_queue_get(&q_nalus))) {
MSH264NaluType nalu_type = ms_h264_nalu_get_type(nalu);
if(nalu_type == MSH264NaluTypeSPS || nalu_type == MSH264NaluTypePPS) {
parameter_sets = bctbx_list_append(parameter_sets, nalu);
iframe_received = TRUE;
ctx->freezed = FALSE;
} else if(ctx->format_desc || parameter_sets) {
ms_queue_put(&q_nalus2, nalu);
} else {
freemsg(nalu);
}
}
if(iframe_received) ms_message("VideoToolboxDecoder: I-frame received");
if(ctx->freezed) goto put_frames_out;
if(parameter_sets) {
CMFormatDescriptionRef new_format_desc = format_desc_from_sps_pps(parameter_sets);
parameter_sets = bctbx_list_free_with_data(parameter_sets, (void (*)(void *))freemsg);
if(ctx->format_desc) {
CMVideoDimensions last_vsize = CMVideoFormatDescriptionGetDimensions(ctx->format_desc);
CMVideoDimensions new_vsize = CMVideoFormatDescriptionGetDimensions(new_format_desc);
if(last_vsize.width != new_vsize.width || last_vsize.height != new_vsize.height) {
ms_message("VideoToolboxDecoder: new encoded video size %dx%d -> %dx%d",
(int)last_vsize.width, (int)last_vsize.height, (int)new_vsize.width, (int)new_vsize.height);
h264_dec_uninit_decoder(ctx);
ctx->format_desc = new_format_desc;
ms_queue_flush(&q_nalus);
ms_queue_flush(&q_nalus2);
res = rfc3984_unpack2(&ctx->unpacker, pkt, &q_nalus);
if (res & Rfc3984FrameAvailable){
if ((res & Rfc3984FrameCorrupted) && ctx->freeze_on_error_enabled ){
ms_warning("VideoToolboxDecoder: corrupted frame received, skipping until next key-frame");
ctx->freezed = TRUE;
continue;
}
} else {
ctx->format_desc = new_format_desc;
}
}
/* Stops proccessing if no IDR has been received yet */
if(ctx->format_desc == NULL) {
ms_warning("VideoToolboxDecoder: no I-frame has been received yet");
goto fail;
}
/* Initializes the decoder if it has not be done yet or reconfigure it when
the size of the encoded video change */
if(ctx->session == NULL) {
if(!h264_dec_init_decoder(ctx)) {
ms_error("VideoToolboxDecoder: failed to initialized decoder");
goto fail;
}
}
// Pack all nalus in a VTBlockBuffer
CMBlockBufferCreateEmpty(NULL, 0, kCMBlockBufferAssureMemoryNowFlag, &stream);
while((nalu = ms_queue_get(&q_nalus2))) {
CMBlockBufferRef nalu_block;
size_t nalu_block_size = msgdsize(nalu) + H264_NALU_HEAD_SIZE;
uint32_t nalu_size = htonl(msgdsize(nalu));
CMBlockBufferCreateWithMemoryBlock(NULL, NULL, nalu_block_size, NULL, NULL, 0, nalu_block_size, kCMBlockBufferAssureMemoryNowFlag, &nalu_block);
CMBlockBufferReplaceDataBytes(&nalu_size, nalu_block, 0, H264_NALU_HEAD_SIZE);
CMBlockBufferReplaceDataBytes(nalu->b_rptr, nalu_block, H264_NALU_HEAD_SIZE, msgdsize(nalu));
CMBlockBufferAppendBufferReference(stream, nalu_block, 0, nalu_block_size, 0);
CFRelease(nalu_block);
freemsg(nalu);
}
if(!CMBlockBufferIsEmpty(stream)) {
timing_info.duration = kCMTimeInvalid;
timing_info.presentationTimeStamp = CMTimeMake(f->ticker->time, 1000);
timing_info.decodeTimeStamp = CMTimeMake(f->ticker->time, 1000);
CMSampleBufferCreate(
NULL, stream, TRUE, NULL, NULL,
ctx->format_desc, 1, 1, &timing_info,
0, NULL, &sample);
status = VTDecompressionSessionDecodeFrame(ctx->session, sample, 0, NULL, NULL);
CFRelease(sample);
if(status != noErr) {
CFRelease(stream);
ms_error("VideoToolboxDecoder: error while passing encoded frames to the decoder: %d", (int)status);
if(status == kVTInvalidSessionErr) {
h264_dec_uninit_decoder(ctx);
if (ctx->freezed){
if (res & Rfc3984IsKeyFrame){
ctx->freezed = FALSE; /*let's restart safely with a full i-frame*/
ms_message("VideoToolboxDecoder: receiving fresh i-frame to restart");
}else{
need_pli = TRUE;
continue;
}
}
// Pull SPSs and PPSs out and put them into the filter context if necessary
iframe_received = FALSE;
while((nalu = ms_queue_get(&q_nalus))) {
MSH264NaluType nalu_type = ms_h264_nalu_get_type(nalu);
if(nalu_type == MSH264NaluTypeSPS || nalu_type == MSH264NaluTypePPS) {
parameter_sets = bctbx_list_append(parameter_sets, nalu);
iframe_received = TRUE;
ctx->freezed = FALSE;
} else if(ctx->format_desc || parameter_sets) {
ms_queue_put(&q_nalus2, nalu);
} else {
freemsg(nalu);
}
}
goto fail;
if(iframe_received) ms_message("VideoToolboxDecoder: I-frame received");
if(parameter_sets) {
CMFormatDescriptionRef new_format_desc = format_desc_from_sps_pps(parameter_sets);
parameter_sets = bctbx_list_free_with_data(parameter_sets, (void (*)(void *))freemsg);
if(ctx->format_desc) {
CMVideoDimensions last_vsize = CMVideoFormatDescriptionGetDimensions(ctx->format_desc);
CMVideoDimensions new_vsize = CMVideoFormatDescriptionGetDimensions(new_format_desc);
if(last_vsize.width != new_vsize.width || last_vsize.height != new_vsize.height) {
ms_message("VideoToolboxDecoder: new encoded video size %dx%d -> %dx%d",
(int)last_vsize.width, (int)last_vsize.height, (int)new_vsize.width, (int)new_vsize.height);
h264_dec_uninit_decoder(ctx);
ctx->format_desc = new_format_desc;
}
} else {
ctx->format_desc = new_format_desc;
}
}
/* Stops proccessing if no IDR has been received yet */
if (ctx->format_desc == NULL) {
need_pli = TRUE;
ms_warning("VideoToolboxDecoder: no I-frame has been received yet");
continue;
}
/* Initializes the decoder if it has not be done yet or reconfigure it when
the size of the encoded video change */
if(ctx->session == NULL) {
if(!h264_dec_init_decoder(ctx)) {
ms_error("VideoToolboxDecoder: failed to initialize decoder");
continue;
}
}
// Pack all nalus in a VTBlockBuffer
CMBlockBufferCreateEmpty(NULL, 0, kCMBlockBufferAssureMemoryNowFlag, &stream);
while((nalu = ms_queue_get(&q_nalus2))) {
CMBlockBufferRef nalu_block;
size_t nalu_block_size = msgdsize(nalu) + H264_NALU_HEAD_SIZE;
uint32_t nalu_size = htonl(msgdsize(nalu));
CMBlockBufferCreateWithMemoryBlock(NULL, NULL, nalu_block_size, NULL, NULL, 0, nalu_block_size, kCMBlockBufferAssureMemoryNowFlag, &nalu_block);
CMBlockBufferReplaceDataBytes(&nalu_size, nalu_block, 0, H264_NALU_HEAD_SIZE);
CMBlockBufferReplaceDataBytes(nalu->b_rptr, nalu_block, H264_NALU_HEAD_SIZE, msgdsize(nalu));
CMBlockBufferAppendBufferReference(stream, nalu_block, 0, nalu_block_size, 0);
CFRelease(nalu_block);
freemsg(nalu);
}
if(!CMBlockBufferIsEmpty(stream)) {
timing_info.duration = kCMTimeInvalid;
timing_info.presentationTimeStamp = CMTimeMake(f->ticker->time, 1000);
timing_info.decodeTimeStamp = CMTimeMake(f->ticker->time, 1000);
CMSampleBufferCreate(
NULL, stream, TRUE, NULL, NULL,
ctx->format_desc, 1, 1, &timing_info,
0, NULL, &sample);
status = VTDecompressionSessionDecodeFrame(ctx->session, sample, 0, NULL, NULL);
CFRelease(sample);
if(status != noErr) {
CFRelease(stream);
ms_error("VideoToolboxDecoder: error while passing encoded frames to the decoder: %d", (int)status);
if(status == kVTInvalidSessionErr) {
h264_dec_uninit_decoder(ctx);
}
need_pli = TRUE;
continue;
}
}
CFRelease(stream);
}
}
CFRelease(stream);
goto put_frames_out;
fail:
ms_filter_lock(f);
if (ctx->enable_avpf) {
ms_message("VideoToolboxDecoder: sending PLI");
ms_filter_notify_no_arg(f, MS_VIDEO_DECODER_SEND_PLI);
}else ms_filter_notify_no_arg(f, MS_VIDEO_DECODER_DECODING_ERRORS);
ms_filter_unlock(f);
put_frames_out:
if (need_pli){
if (ctx->enable_avpf) {
ms_message("VideoToolboxDecoder: sending PLI");
ms_filter_notify_no_arg(f, MS_VIDEO_DECODER_SEND_PLI);
}else ms_filter_notify_no_arg(f, MS_VIDEO_DECODER_DECODING_ERRORS);
}
// Transfer decoded frames in the output queue
ms_mutex_lock(&ctx->mutex);
while((pixbuf = ms_queue_get(&ctx->queue))) {
......@@ -751,13 +760,6 @@ put_frames_out:
ms_mutex_lock(&ctx->mutex);
}
ms_mutex_unlock(&ctx->mutex);
// Cleaning
ms_queue_flush(&q_nalus);
ms_queue_flush(&q_nalus2);
ms_queue_flush(f->inputs[0]);
return;
}
static void h264_dec_uninit(MSFilter *f) {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment