h264dec.c 12.3 KB
Newer Older
1 2
/*
mediastreamer2 library - modular sound and video processing and streaming
3
Copyright (C) 2010  Belledonne Communications SARL
4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23
Author: Simon Morlat <simon.morlat@linphone.org>

This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; either version 2
of the License, or (at your option) any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
*/

#include "mediastreamer2/msfilter.h"
#include "mediastreamer2/rfc3984.h"
#include "mediastreamer2/msvideo.h"
24
#include "mediastreamer2/msticker.h"
25
#include "stream_regulator.h"
26 27 28 29 30

#include "ffmpeg-priv.h"

#include "ortp/b64.h"

31

32 33
typedef struct _DecData{
	mblk_t *sps,*pps;
34
	AVFrame* orig;
35
	Rfc3984Context unpacker;
36
	MSVideoSize vsize;
37
	struct SwsContext *sws_ctx;
Simon Morlat's avatar
Simon Morlat committed
38
	MSAverageFPS fps;
39 40 41 42
	AVCodecContext av_context;
	unsigned int packet_num;
	uint8_t *bitstream;
	int bitstream_size;
43
	MSStreamRegulator *regulator;
44
	MSYuvBufAllocator *buf_allocator;
45 46
	bool_t first_image_decoded;
	bool_t avpf_enabled;
47 48
}DecData;

49
static void ffmpeg_init(void){
50 51 52 53 54 55 56 57 58 59 60 61
	static bool_t done=FALSE;
	if (!done){
		avcodec_register_all();
		done=TRUE;
	}
}

static void dec_open(DecData *d){
	AVCodec *codec;
	int error;
	codec=avcodec_find_decoder(CODEC_ID_H264);
	if (codec==NULL) ms_fatal("Could not find H264 decoder in ffmpeg.");
62 63
	avcodec_get_context_defaults3(&d->av_context, NULL);
	error=avcodec_open2(&d->av_context,codec, NULL);
64 65 66 67 68 69
	if (error!=0){
		ms_fatal("avcodec_open() failed.");
	}
}

static void dec_init(MSFilter *f){
70
	DecData *d=ms_new0(DecData,1);
71 72 73 74 75 76 77
	ffmpeg_init();
	d->sps=NULL;
	d->pps=NULL;
	d->sws_ctx=NULL;
	rfc3984_init(&d->unpacker);
	d->packet_num=0;
	dec_open(d);
78 79
	d->vsize.width=0;
	d->vsize.height=0;
80 81
	d->bitstream_size=65536;
	d->bitstream=ms_malloc0(d->bitstream_size);
82
	d->orig = av_frame_alloc();
Simon Morlat's avatar
Simon Morlat committed
83
	ms_average_fps_init(&d->fps, "ffmpeg H264 decoder: FPS: %f");
84 85 86
	if (!d->orig) {
		ms_error("Could not allocate frame");
	}
87
	d->regulator = NULL;
88
	d->buf_allocator = ms_yuv_buf_allocator_new();
89 90 91
	f->data=d;
}

92
static void dec_preprocess(MSFilter* f) {
93 94
	DecData *s=(DecData*)f->data;
	s->first_image_decoded = FALSE;
95
	s->regulator = ms_stream_regulator_new(f->ticker, 90000);
96 97
}

98 99 100 101 102
static void dec_reinit(DecData *d){
	avcodec_close(&d->av_context);
	dec_open(d);
}

103 104 105 106 107
static void dec_postprocess(MSFilter *f) {
	DecData *s=(DecData*)f->data;
	ms_stream_regulator_free(s->regulator);
}

108 109 110 111 112 113
static void dec_uninit(MSFilter *f){
	DecData *d=(DecData*)f->data;
	rfc3984_uninit(&d->unpacker);
	avcodec_close(&d->av_context);
	if (d->sps) freemsg(d->sps);
	if (d->pps) freemsg(d->pps);
114
	if (d->orig) av_frame_free(&d->orig);
François Grisez's avatar
François Grisez committed
115
	if (d->sws_ctx) sws_freeContext(d->sws_ctx);
116
	ms_free(d->bitstream);
117
	ms_yuv_buf_allocator_free(d->buf_allocator);
118 119 120 121 122
	ms_free(d);
}

static mblk_t *get_as_yuvmsg(MSFilter *f, DecData *s, AVFrame *orig){
	AVCodecContext *ctx=&s->av_context;
123 124
	MSPicture pic = {0};
	mblk_t *yuv_msg;
125

126
	if (s->vsize.width!=ctx->width || s->vsize.height!=ctx->height){
127
		if (s->sws_ctx!=NULL){
128
			sws_freeContext(s->sws_ctx);
129 130 131
			s->sws_ctx=NULL;
		}
		ms_message("Getting yuv picture of %ix%i",ctx->width,ctx->height);
132 133
		s->vsize.width=ctx->width;
		s->vsize.height=ctx->height;
134
		s->sws_ctx=sws_getContext(ctx->width,ctx->height,ctx->pix_fmt,
135
			ctx->width,ctx->height,AV_PIX_FMT_YUV420P,SWS_FAST_BILINEAR,
136
                	NULL, NULL, NULL);
137
		ms_filter_notify_no_arg(f,MS_FILTER_OUTPUT_FMT_CHANGED);
138
	}
139
	yuv_msg=ms_yuv_buf_allocator_get(s->buf_allocator, &pic,ctx->width,ctx->height);
140
#if LIBSWSCALE_VERSION_INT >= AV_VERSION_INT(0,9,0)
141
	if (sws_scale(s->sws_ctx,(const uint8_t * const *)orig->data,orig->linesize, 0,
142
					ctx->height, pic.planes, pic.strides)<0){
143 144
#else
	if (sws_scale(s->sws_ctx,(uint8_t **)orig->data,orig->linesize, 0,
145
					ctx->height, pic.planes, pic.strides)<0){
146
#endif
147 148
		ms_error("%s: error in sws_scale().",f->desc->name);
	}
149
#if LIBAVUTIL_VERSION_INT >= AV_VERSION_INT(50,43,0) // backward compatibility with Debian Squeeze (6.0)
150
	mblk_set_timestamp_info(yuv_msg, orig->pkt_pts);
151
#endif
152
	return yuv_msg;
153 154 155 156 157 158 159 160 161 162 163 164 165 166 167
}

static void update_sps(DecData *d, mblk_t *sps){
	if (d->sps)
		freemsg(d->sps);
	d->sps=dupb(sps);
}

static void update_pps(DecData *d, mblk_t *pps){
	if (d->pps)
		freemsg(d->pps);
	if (pps) d->pps=dupb(pps);
	else d->pps=NULL;
}

168 169 170

static bool_t check_sps_change(DecData *d, mblk_t *sps){
	bool_t ret=FALSE;
171
	if (d->sps){
172 173
		ret=(msgdsize(sps)!=msgdsize(d->sps)) || (memcmp(d->sps->b_rptr,sps->b_rptr,msgdsize(sps))!=0);
		if (ret) {
174
			ms_message("SPS changed ! %i,%i",(int)msgdsize(sps),(int)msgdsize(d->sps));
175 176
			update_sps(d,sps);
			update_pps(d,NULL);
177
		}
178
	} else {
179 180 181
		ms_message("Receiving first SPS");
		update_sps(d,sps);
	}
182 183 184 185 186
	return ret;
}

static bool_t check_pps_change(DecData *d, mblk_t *pps){
	bool_t ret=FALSE;
187
	if (d->pps){
188 189
		ret=(msgdsize(pps)!=msgdsize(d->pps)) || (memcmp(d->pps->b_rptr,pps->b_rptr,msgdsize(pps))!=0);
		if (ret) {
190
			ms_message("PPS changed ! %i,%i",(int)msgdsize(pps),(int)msgdsize(d->pps));
191
			update_pps(d,pps);
192
		}
193
	}else {
194 195 196
		ms_message("Receiving first PPS");
		update_pps(d,pps);
	}
197
	return ret;
198 199
}

200

201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222
static void enlarge_bitstream(DecData *d, int new_size){
	d->bitstream_size=new_size;
	d->bitstream=ms_realloc(d->bitstream,d->bitstream_size);
}

static int nalusToFrame(DecData *d, MSQueue *naluq, bool_t *new_sps_pps){
	mblk_t *im;
	uint8_t *dst=d->bitstream,*src,*end;
	int nal_len;
	bool_t start_picture=TRUE;
	uint8_t nalu_type;
	*new_sps_pps=FALSE;
	end=d->bitstream+d->bitstream_size;
	while((im=ms_queue_get(naluq))!=NULL){
		src=im->b_rptr;
		nal_len=im->b_wptr-src;
		if (dst+nal_len+100>end){
			int pos=dst-d->bitstream;
			enlarge_bitstream(d, d->bitstream_size+nal_len+100);
			dst=d->bitstream+pos;
			end=d->bitstream+d->bitstream_size;
		}
223 224 225 226 227 228 229 230
		if (src[0]==0 && src[1]==0 && src[2]==0 && src[3]==1){
			int size=im->b_wptr-src;
			/*workaround for stupid RTP H264 sender that includes nal markers */
			memcpy(dst,src,size);
			dst+=size;
		}else{
			nalu_type=(*src) & ((1<<5)-1);
			if (nalu_type==7)
231
				*new_sps_pps=(check_sps_change(d,im) || *new_sps_pps);
232
			if (nalu_type==8)
233
				*new_sps_pps=(check_pps_change(d,im) || *new_sps_pps);
234
			if (start_picture || nalu_type==7/*SPS*/ || nalu_type==8/*PPS*/ ){
235
				*dst++=0;
236
				start_picture=FALSE;
237
			}
238

239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254
			/*prepend nal marker*/
			*dst++=0;
			*dst++=0;
			*dst++=1;
			*dst++=*src++;
			while(src<(im->b_wptr-3)){
				if (src[0]==0 && src[1]==0 && src[2]<3){
					*dst++=0;
					*dst++=0;
					*dst++=3;
					src+=2;
				}
				*dst++=*src++;
			}
			*dst++=*src++;
			*dst++=*src++;
255 256 257 258 259 260 261 262 263
			*dst++=*src++;
		}
		freemsg(im);
	}
	return dst-d->bitstream;
}

static void dec_process(MSFilter *f){
	DecData *d=(DecData*)f->data;
264
	mblk_t *im, *om;
265
	MSQueue nalus;
266
	bool_t requestPLI = FALSE;
267

268 269
	ms_queue_init(&nalus);
	while((im=ms_queue_get(f->inputs[0]))!=NULL){
270 271 272 273 274 275 276 277 278
		// Reset all contexts when an empty packet is received
		if(msgdsize(im) == 0) {
			rfc3984_uninit(&d->unpacker);
			rfc3984_init(&d->unpacker);
			dec_reinit(d);
			ms_stream_regulator_reset(d->regulator);
			freemsg(im);
			continue;
		}
279 280 281 282 283 284 285 286 287
		/*push the sps/pps given in sprop-parameter-sets if any*/
		if (d->packet_num==0 && d->sps && d->pps){
			mblk_set_timestamp_info(d->sps,mblk_get_timestamp_info(im));
			mblk_set_timestamp_info(d->pps,mblk_get_timestamp_info(im));
			rfc3984_unpack(&d->unpacker,d->sps,&nalus);
			rfc3984_unpack(&d->unpacker,d->pps,&nalus);
			d->sps=NULL;
			d->pps=NULL;
		}
288
		requestPLI |= (rfc3984_unpack(&d->unpacker,im,&nalus) != -0);
289 290 291 292
		if (!ms_queue_empty(&nalus)){
			int size;
			uint8_t *p,*end;
			bool_t need_reinit=FALSE;
François Grisez's avatar
François Grisez committed
293 294 295
#if LIBAVUTIL_VERSION_INT >= AV_VERSION_INT(50,43,0) // backward compatibility with Debian Squeeze (6.0)
			uint32_t frame_ts = mblk_get_timestamp_info(ms_queue_peek_first(&nalus));
#endif
296 297 298 299 300 301 302 303 304 305

			size=nalusToFrame(d,&nalus,&need_reinit);
			if (need_reinit)
				dec_reinit(d);
			p=d->bitstream;
			end=d->bitstream+size;
			while (end-p>0) {
				int len;
				int got_picture=0;
				AVPacket pkt;
306
				av_frame_unref(d->orig);
307 308 309
				av_init_packet(&pkt);
				pkt.data = p;
				pkt.size = end-p;
310
#if LIBAVUTIL_VERSION_INT >= AV_VERSION_INT(50,43,0) // backward compatibility with Debian Squeeze (6.0)
François Grisez's avatar
François Grisez committed
311
				pkt.pts = frame_ts;
312
#endif
313
				len=avcodec_decode_video2(&d->av_context,d->orig,&got_picture,&pkt);
314 315
				if (len<=0) {
					ms_warning("ms_AVdecoder_process: error %i.",len);
316
					ms_filter_notify_no_arg(f,MS_VIDEO_DECODER_DECODING_ERRORS);
317
					requestPLI = TRUE;
318 319 320
					break;
				}
				if (got_picture) {
321
					ms_stream_regulator_push(d->regulator, get_as_yuvmsg(f,d,d->orig));
322 323 324 325 326 327
				}
				p+=len;
			}
		}
		d->packet_num++;
	}
328 329 330 331 332 333 334
	while((om = ms_stream_regulator_get(d->regulator))) {
		ms_queue_put(f->outputs[0], om);
		if (!d->first_image_decoded) {
			d->first_image_decoded = TRUE;
			ms_filter_notify_no_arg(f,MS_VIDEO_DECODER_FIRST_IMAGE_DECODED);
		}
		if (ms_average_fps_update(&d->fps, f->ticker->time)) {
335
			ms_message("ffmpeg H264 decoder: Frame size: %dx%d", d->vsize.width,  d->vsize.height);
336 337
		}
	}
338 339 340
	if (d->avpf_enabled && requestPLI) {
		ms_filter_notify_no_arg(f, MS_VIDEO_DECODER_SEND_PLI);
	}
341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362
}

static int dec_add_fmtp(MSFilter *f, void *arg){
	DecData *d=(DecData*)f->data;
	const char *fmtp=(const char *)arg;
	char value[256];
	if (fmtp_get_value(fmtp,"sprop-parameter-sets",value,sizeof(value))){
		char * b64_sps=value;
		char * b64_pps=strchr(value,',');
		if (b64_pps){
			*b64_pps='\0';
			++b64_pps;
			ms_message("Got sprop-parameter-sets : sps=%s , pps=%s",b64_sps,b64_pps);
			d->sps=allocb(sizeof(value),0);
			d->sps->b_wptr+=b64_decode(b64_sps,strlen(b64_sps),d->sps->b_wptr,sizeof(value));
			d->pps=allocb(sizeof(value),0);
			d->pps->b_wptr+=b64_decode(b64_pps,strlen(b64_pps),d->pps->b_wptr,sizeof(value));
		}
	}
	return 0;
}

363 364 365 366
static int reset_first_image(MSFilter* f, void *data) {
	DecData *d=(DecData*)f->data;
	d->first_image_decoded = FALSE;
	return 0;
367 368
}

369 370 371 372
static int dec_get_vsize(MSFilter *f, void *data) {
	DecData *d = (DecData *)f->data;
	MSVideoSize *vsize = (MSVideoSize *)data;
	if (d->first_image_decoded == TRUE) {
373 374
		vsize->width = d->vsize.width;
		vsize->height = d->vsize.height;
375
	} else {
376 377
		vsize->width = MS_VIDEO_SIZE_UNKNOWN_W;
		vsize->height = MS_VIDEO_SIZE_UNKNOWN_H;
378 379 380 381
	}
	return 0;
}

Simon Morlat's avatar
Simon Morlat committed
382 383 384 385 386 387
static int dec_get_fps(MSFilter *f, void *data){
	DecData *s = (DecData *)f->data;
	*(float*)data= ms_average_fps_get(&s->fps);
	return 0;
}

388 389
static int dec_get_outfmt(MSFilter *f, void *data){
	DecData *s = (DecData *)f->data;
390
	((MSPinFormat*)data)->fmt=ms_factory_get_video_format(f->factory,"YUV420P",ms_video_size_make(s->vsize.width,s->vsize.height),0,NULL);
391 392 393
	return 0;
}

394 395 396 397 398 399
static int dec_enable_avpf(MSFilter *f, void *data){
	DecData *s = (DecData *)f->data;
	s->avpf_enabled = *(bool_t*)data;
	return 0;
}

400
static MSFilterMethod  h264_dec_methods[]={
401 402 403 404 405
	{	MS_FILTER_ADD_FMTP                                 ,	dec_add_fmtp      },
	{	MS_VIDEO_DECODER_RESET_FIRST_IMAGE_NOTIFICATION    ,	reset_first_image },
	{	MS_FILTER_GET_VIDEO_SIZE                           ,	dec_get_vsize     },
	{	MS_FILTER_GET_FPS                                  ,	dec_get_fps       },
	{	MS_FILTER_GET_OUTPUT_FMT                           ,	dec_get_outfmt    },
406
	{	MS_VIDEO_DECODER_ENABLE_AVPF                       ,	dec_enable_avpf   },
407
	{	0                                                  ,	NULL              }
408 409
};

Simon Morlat's avatar
Simon Morlat committed
410 411 412
#ifndef _MSC_VER

MSFilterDesc ms_h264_dec_desc={
413 414 415 416 417 418 419 420
	.id=MS_H264_DEC_ID,
	.name="MSH264Dec",
	.text="A H264 decoder based on ffmpeg project.",
	.category=MS_FILTER_DECODER,
	.enc_fmt="H264",
	.ninputs=1,
	.noutputs=1,
	.init=dec_init,
421
	.preprocess=dec_preprocess,
422
	.process=dec_process,
423
	.postprocess=dec_postprocess,
424
	.uninit=dec_uninit,
425 426
	.methods=h264_dec_methods,
	.flags=MS_FILTER_IS_PUMP
427 428
};

Simon Morlat's avatar
Simon Morlat committed
429 430 431 432 433 434 435 436 437 438 439 440
#else


MSFilterDesc ms_h264_dec_desc={
	MS_H264_DEC_ID,
	"MSH264Dec",
	"A H264 decoder based on ffmpeg project.",
	MS_FILTER_DECODER,
	"H264",
	1,
	1,
	dec_init,
441
	dec_preprocess,
Simon Morlat's avatar
Simon Morlat committed
442
	dec_process,
443
	dec_postprocess,
Simon Morlat's avatar
Simon Morlat committed
444
	dec_uninit,
445 446
	h264_dec_methods,
	MS_FILTER_IS_PUMP
Simon Morlat's avatar
Simon Morlat committed
447 448 449 450
};

#endif

451 452 453 454 455 456
void __register_ffmpeg_h264_decoder_if_possible(MSFactory *obj) {
	ms_ffmpeg_check_init();
	if (avcodec_find_decoder(CODEC_ID_H264) && HAVE_NON_FREE_CODECS) {
		ms_factory_register_filter(obj, &ms_h264_dec_desc);
	}
}