h264dec.c 11.5 KB
Newer Older
1 2
/*
mediastreamer2 library - modular sound and video processing and streaming
3
Copyright (C) 2010  Belledonne Communications SARL
4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23
Author: Simon Morlat <simon.morlat@linphone.org>

This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; either version 2
of the License, or (at your option) any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
*/

#include "mediastreamer2/msfilter.h"
#include "mediastreamer2/rfc3984.h"
#include "mediastreamer2/msvideo.h"
24
#include "mediastreamer2/msticker.h"
25
#include "stream_regulator.h"
26 27 28 29 30

#include "ffmpeg-priv.h"

#include "ortp/b64.h"

31

32 33
typedef struct _DecData{
	mblk_t *sps,*pps;
34
	AVFrame* orig;
35
	Rfc3984Context unpacker;
36
	MSVideoSize vsize;
37
	struct SwsContext *sws_ctx;
Simon Morlat's avatar
Simon Morlat committed
38
	MSAverageFPS fps;
39 40 41 42
	AVCodecContext av_context;
	unsigned int packet_num;
	uint8_t *bitstream;
	int bitstream_size;
43
	bool_t first_image_decoded;
44
	MSStreamRegulator *regulator;
45 46 47 48 49
}DecData;

static void ffmpeg_init(){
	static bool_t done=FALSE;
	if (!done){
50
#ifdef FF_API_AVCODEC_INIT
51
		avcodec_init();
52
#endif
53 54 55 56 57 58 59 60 61 62
		avcodec_register_all();
		done=TRUE;
	}
}

static void dec_open(DecData *d){
	AVCodec *codec;
	int error;
	codec=avcodec_find_decoder(CODEC_ID_H264);
	if (codec==NULL) ms_fatal("Could not find H264 decoder in ffmpeg.");
63 64
	avcodec_get_context_defaults3(&d->av_context, NULL);
	error=avcodec_open2(&d->av_context,codec, NULL);
65 66 67 68 69 70
	if (error!=0){
		ms_fatal("avcodec_open() failed.");
	}
}

static void dec_init(MSFilter *f){
71
	DecData *d=ms_new0(DecData,1);
72 73 74 75 76 77 78
	ffmpeg_init();
	d->sps=NULL;
	d->pps=NULL;
	d->sws_ctx=NULL;
	rfc3984_init(&d->unpacker);
	d->packet_num=0;
	dec_open(d);
79 80
	d->vsize.width=0;
	d->vsize.height=0;
81 82
	d->bitstream_size=65536;
	d->bitstream=ms_malloc0(d->bitstream_size);
83
	d->orig = av_frame_alloc();
Simon Morlat's avatar
Simon Morlat committed
84
	ms_average_fps_init(&d->fps, "ffmpeg H264 decoder: FPS: %f");
85 86 87
	if (!d->orig) {
		ms_error("Could not allocate frame");
	}
88
	d->regulator = NULL;
89 90 91
	f->data=d;
}

92
static void dec_preprocess(MSFilter* f) {
93 94
	DecData *s=(DecData*)f->data;
	s->first_image_decoded = FALSE;
95
	s->regulator = ms_stream_regulator_new(f->ticker, 90000);
96 97
}

98 99 100 101 102
static void dec_reinit(DecData *d){
	avcodec_close(&d->av_context);
	dec_open(d);
}

103 104 105 106 107
static void dec_postprocess(MSFilter *f) {
	DecData *s=(DecData*)f->data;
	ms_stream_regulator_free(s->regulator);
}

108 109 110 111 112 113
static void dec_uninit(MSFilter *f){
	DecData *d=(DecData*)f->data;
	rfc3984_uninit(&d->unpacker);
	avcodec_close(&d->av_context);
	if (d->sps) freemsg(d->sps);
	if (d->pps) freemsg(d->pps);
114
	if (d->orig) av_frame_free(&d->orig);
François Grisez's avatar
François Grisez committed
115
	if (d->sws_ctx) sws_freeContext(d->sws_ctx);
116 117 118 119 120 121
	ms_free(d->bitstream);
	ms_free(d);
}

static mblk_t *get_as_yuvmsg(MSFilter *f, DecData *s, AVFrame *orig){
	AVCodecContext *ctx=&s->av_context;
122 123
	MSPicture pic = {0};
	mblk_t *yuv_msg;
124

125
	if (s->vsize.width!=ctx->width || s->vsize.height!=ctx->height){
126
		if (s->sws_ctx!=NULL){
127
			sws_freeContext(s->sws_ctx);
128 129 130
			s->sws_ctx=NULL;
		}
		ms_message("Getting yuv picture of %ix%i",ctx->width,ctx->height);
131 132
		s->vsize.width=ctx->width;
		s->vsize.height=ctx->height;
133
		s->sws_ctx=sws_getContext(ctx->width,ctx->height,ctx->pix_fmt,
134 135
			ctx->width,ctx->height,PIX_FMT_YUV420P,SWS_FAST_BILINEAR,
                	NULL, NULL, NULL);
136
		ms_filter_notify_no_arg(f,MS_FILTER_OUTPUT_FMT_CHANGED);
137
	}
138
	yuv_msg=ms_yuv_buf_alloc(&pic,ctx->width,ctx->height);
139
#if LIBSWSCALE_VERSION_INT >= AV_VERSION_INT(0,9,0)
140
	if (sws_scale(s->sws_ctx,(const uint8_t * const *)orig->data,orig->linesize, 0,
141
					ctx->height, pic.planes, pic.strides)<0){
142 143
#else
	if (sws_scale(s->sws_ctx,(uint8_t **)orig->data,orig->linesize, 0,
144
					ctx->height, pic.planes, pic.strides)<0){
145
#endif
146 147
		ms_error("%s: error in sws_scale().",f->desc->name);
	}
148
#if LIBAVUTIL_VERSION_INT >= AV_VERSION_INT(50,43,0) // backward compatibility with Debian Squeeze (6.0)
149
	mblk_set_timestamp_info(yuv_msg, orig->pkt_pts);
150
#endif
151
	return yuv_msg;
152 153 154 155 156 157 158 159 160 161 162 163 164 165 166
}

static void update_sps(DecData *d, mblk_t *sps){
	if (d->sps)
		freemsg(d->sps);
	d->sps=dupb(sps);
}

static void update_pps(DecData *d, mblk_t *pps){
	if (d->pps)
		freemsg(d->pps);
	if (pps) d->pps=dupb(pps);
	else d->pps=NULL;
}

167 168 169

static bool_t check_sps_change(DecData *d, mblk_t *sps){
	bool_t ret=FALSE;
170
	if (d->sps){
171 172
		ret=(msgdsize(sps)!=msgdsize(d->sps)) || (memcmp(d->sps->b_rptr,sps->b_rptr,msgdsize(sps))!=0);
		if (ret) {
173
			ms_message("SPS changed ! %i,%i",(int)msgdsize(sps),(int)msgdsize(d->sps));
174 175
			update_sps(d,sps);
			update_pps(d,NULL);
176
		}
177
	} else {
178 179 180
		ms_message("Receiving first SPS");
		update_sps(d,sps);
	}
181 182 183 184 185
	return ret;
}

static bool_t check_pps_change(DecData *d, mblk_t *pps){
	bool_t ret=FALSE;
186
	if (d->pps){
187 188
		ret=(msgdsize(pps)!=msgdsize(d->pps)) || (memcmp(d->pps->b_rptr,pps->b_rptr,msgdsize(pps))!=0);
		if (ret) {
189
			ms_message("PPS changed ! %i,%i",(int)msgdsize(pps),(int)msgdsize(d->pps));
190
			update_pps(d,pps);
191
		}
192
	}else {
193 194 195
		ms_message("Receiving first PPS");
		update_pps(d,pps);
	}
196
	return ret;
197 198
}

199

200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221
static void enlarge_bitstream(DecData *d, int new_size){
	d->bitstream_size=new_size;
	d->bitstream=ms_realloc(d->bitstream,d->bitstream_size);
}

static int nalusToFrame(DecData *d, MSQueue *naluq, bool_t *new_sps_pps){
	mblk_t *im;
	uint8_t *dst=d->bitstream,*src,*end;
	int nal_len;
	bool_t start_picture=TRUE;
	uint8_t nalu_type;
	*new_sps_pps=FALSE;
	end=d->bitstream+d->bitstream_size;
	while((im=ms_queue_get(naluq))!=NULL){
		src=im->b_rptr;
		nal_len=im->b_wptr-src;
		if (dst+nal_len+100>end){
			int pos=dst-d->bitstream;
			enlarge_bitstream(d, d->bitstream_size+nal_len+100);
			dst=d->bitstream+pos;
			end=d->bitstream+d->bitstream_size;
		}
222 223 224 225 226 227 228 229 230 231 232 233
		if (src[0]==0 && src[1]==0 && src[2]==0 && src[3]==1){
			int size=im->b_wptr-src;
			/*workaround for stupid RTP H264 sender that includes nal markers */
			memcpy(dst,src,size);
			dst+=size;
		}else{
			nalu_type=(*src) & ((1<<5)-1);
			if (nalu_type==7)
				*new_sps_pps=check_sps_change(d,im) || *new_sps_pps;
			if (nalu_type==8)
				*new_sps_pps=check_pps_change(d,im) || *new_sps_pps;
			if (start_picture || nalu_type==7/*SPS*/ || nalu_type==8/*PPS*/ ){
234
				*dst++=0;
235
				start_picture=FALSE;
236
			}
237

238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253
			/*prepend nal marker*/
			*dst++=0;
			*dst++=0;
			*dst++=1;
			*dst++=*src++;
			while(src<(im->b_wptr-3)){
				if (src[0]==0 && src[1]==0 && src[2]<3){
					*dst++=0;
					*dst++=0;
					*dst++=3;
					src+=2;
				}
				*dst++=*src++;
			}
			*dst++=*src++;
			*dst++=*src++;
254 255 256 257 258 259 260 261 262
			*dst++=*src++;
		}
		freemsg(im);
	}
	return dst-d->bitstream;
}

static void dec_process(MSFilter *f){
	DecData *d=(DecData*)f->data;
263
	mblk_t *im, *om;
264
	MSQueue nalus;
265

266 267
	ms_queue_init(&nalus);
	while((im=ms_queue_get(f->inputs[0]))!=NULL){
268 269 270 271 272 273 274 275 276
		// Reset all contexts when an empty packet is received
		if(msgdsize(im) == 0) {
			rfc3984_uninit(&d->unpacker);
			rfc3984_init(&d->unpacker);
			dec_reinit(d);
			ms_stream_regulator_reset(d->regulator);
			freemsg(im);
			continue;
		}
277 278 279 280 281 282 283 284 285 286 287 288 289 290
		/*push the sps/pps given in sprop-parameter-sets if any*/
		if (d->packet_num==0 && d->sps && d->pps){
			mblk_set_timestamp_info(d->sps,mblk_get_timestamp_info(im));
			mblk_set_timestamp_info(d->pps,mblk_get_timestamp_info(im));
			rfc3984_unpack(&d->unpacker,d->sps,&nalus);
			rfc3984_unpack(&d->unpacker,d->pps,&nalus);
			d->sps=NULL;
			d->pps=NULL;
		}
		rfc3984_unpack(&d->unpacker,im,&nalus);
		if (!ms_queue_empty(&nalus)){
			int size;
			uint8_t *p,*end;
			bool_t need_reinit=FALSE;
François Grisez's avatar
François Grisez committed
291 292 293
#if LIBAVUTIL_VERSION_INT >= AV_VERSION_INT(50,43,0) // backward compatibility with Debian Squeeze (6.0)
			uint32_t frame_ts = mblk_get_timestamp_info(ms_queue_peek_first(&nalus));
#endif
294 295 296 297 298 299 300 301 302 303

			size=nalusToFrame(d,&nalus,&need_reinit);
			if (need_reinit)
				dec_reinit(d);
			p=d->bitstream;
			end=d->bitstream+size;
			while (end-p>0) {
				int len;
				int got_picture=0;
				AVPacket pkt;
304
				av_frame_unref(d->orig);
305 306 307
				av_init_packet(&pkt);
				pkt.data = p;
				pkt.size = end-p;
308
#if LIBAVUTIL_VERSION_INT >= AV_VERSION_INT(50,43,0) // backward compatibility with Debian Squeeze (6.0)
François Grisez's avatar
François Grisez committed
309
				pkt.pts = frame_ts;
310
#endif
311
				len=avcodec_decode_video2(&d->av_context,d->orig,&got_picture,&pkt);
312 313
				if (len<=0) {
					ms_warning("ms_AVdecoder_process: error %i.",len);
314
					ms_filter_notify_no_arg(f,MS_VIDEO_DECODER_DECODING_ERRORS);
315 316 317
					break;
				}
				if (got_picture) {
318
					ms_stream_regulator_push(d->regulator, get_as_yuvmsg(f,d,d->orig));
319 320 321 322 323 324
				}
				p+=len;
			}
		}
		d->packet_num++;
	}
325 326 327 328 329 330 331
	while((om = ms_stream_regulator_get(d->regulator))) {
		ms_queue_put(f->outputs[0], om);
		if (!d->first_image_decoded) {
			d->first_image_decoded = TRUE;
			ms_filter_notify_no_arg(f,MS_VIDEO_DECODER_FIRST_IMAGE_DECODED);
		}
		if (ms_average_fps_update(&d->fps, f->ticker->time)) {
332
			ms_message("ffmpeg H264 decoder: Frame size: %dx%d", d->vsize.width,  d->vsize.height);
333 334
		}
	}
335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356
}

static int dec_add_fmtp(MSFilter *f, void *arg){
	DecData *d=(DecData*)f->data;
	const char *fmtp=(const char *)arg;
	char value[256];
	if (fmtp_get_value(fmtp,"sprop-parameter-sets",value,sizeof(value))){
		char * b64_sps=value;
		char * b64_pps=strchr(value,',');
		if (b64_pps){
			*b64_pps='\0';
			++b64_pps;
			ms_message("Got sprop-parameter-sets : sps=%s , pps=%s",b64_sps,b64_pps);
			d->sps=allocb(sizeof(value),0);
			d->sps->b_wptr+=b64_decode(b64_sps,strlen(b64_sps),d->sps->b_wptr,sizeof(value));
			d->pps=allocb(sizeof(value),0);
			d->pps->b_wptr+=b64_decode(b64_pps,strlen(b64_pps),d->pps->b_wptr,sizeof(value));
		}
	}
	return 0;
}

357 358 359 360
static int reset_first_image(MSFilter* f, void *data) {
	DecData *d=(DecData*)f->data;
	d->first_image_decoded = FALSE;
	return 0;
361 362
}

363 364 365 366
static int dec_get_vsize(MSFilter *f, void *data) {
	DecData *d = (DecData *)f->data;
	MSVideoSize *vsize = (MSVideoSize *)data;
	if (d->first_image_decoded == TRUE) {
367 368
		vsize->width = d->vsize.width;
		vsize->height = d->vsize.height;
369
	} else {
370 371
		vsize->width = MS_VIDEO_SIZE_UNKNOWN_W;
		vsize->height = MS_VIDEO_SIZE_UNKNOWN_H;
372 373 374 375
	}
	return 0;
}

Simon Morlat's avatar
Simon Morlat committed
376 377 378 379 380 381
static int dec_get_fps(MSFilter *f, void *data){
	DecData *s = (DecData *)f->data;
	*(float*)data= ms_average_fps_get(&s->fps);
	return 0;
}

382 383
static int dec_get_outfmt(MSFilter *f, void *data){
	DecData *s = (DecData *)f->data;
384
	((MSPinFormat*)data)->fmt=ms_factory_get_video_format(f->factory,"YUV420P",ms_video_size_make(s->vsize.width,s->vsize.height),0,NULL);
385 386 387
	return 0;
}

388 389
static MSFilterMethod  h264_dec_methods[]={
	{	MS_FILTER_ADD_FMTP	,	dec_add_fmtp	},
Simon Morlat's avatar
Simon Morlat committed
390
	{	MS_VIDEO_DECODER_RESET_FIRST_IMAGE_NOTIFICATION, reset_first_image },
391
	{	MS_FILTER_GET_VIDEO_SIZE,	dec_get_vsize	},
Simon Morlat's avatar
Simon Morlat committed
392
	{	MS_FILTER_GET_FPS	,	dec_get_fps	},
393
	{	MS_FILTER_GET_OUTPUT_FMT,	dec_get_outfmt	},
394 395 396
	{	0			,	NULL	}
};

Simon Morlat's avatar
Simon Morlat committed
397 398 399
#ifndef _MSC_VER

MSFilterDesc ms_h264_dec_desc={
400 401 402 403 404 405 406 407
	.id=MS_H264_DEC_ID,
	.name="MSH264Dec",
	.text="A H264 decoder based on ffmpeg project.",
	.category=MS_FILTER_DECODER,
	.enc_fmt="H264",
	.ninputs=1,
	.noutputs=1,
	.init=dec_init,
408
	.preprocess=dec_preprocess,
409
	.process=dec_process,
410
	.postprocess=dec_postprocess,
411
	.uninit=dec_uninit,
412 413
	.methods=h264_dec_methods,
	.flags=MS_FILTER_IS_PUMP
414 415
};

Simon Morlat's avatar
Simon Morlat committed
416 417 418 419 420 421 422 423 424 425 426 427
#else


MSFilterDesc ms_h264_dec_desc={
	MS_H264_DEC_ID,
	"MSH264Dec",
	"A H264 decoder based on ffmpeg project.",
	MS_FILTER_DECODER,
	"H264",
	1,
	1,
	dec_init,
428
	dec_preprocess,
Simon Morlat's avatar
Simon Morlat committed
429
	dec_process,
430
	dec_postprocess,
Simon Morlat's avatar
Simon Morlat committed
431
	dec_uninit,
432 433
	h264_dec_methods,
	MS_FILTER_IS_PUMP
Simon Morlat's avatar
Simon Morlat committed
434 435 436 437 438
};

#endif

MS_FILTER_DESC_EXPORT(ms_h264_dec_desc)
439