msopenh264dec.cpp 9.91 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16
/*
H.264 encoder/decoder plugin for mediastreamer2 based on the openh264 library.
Copyright (C) 2006-2012 Belledonne Communications, Grenoble

This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; either version 2
of the License, or (at your option) any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
17
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
18 19 20 21
*/


#include "msopenh264dec.h"
22 23
#include "mediastreamer2/msticker.h"
#include "ortp/b64.h"
24 25 26 27 28
#include "wels/codec_ver.h"

static void decoder_log(void* context, int level, const char* message){
	ms_message("OpenH264 decoder: %s", message);
}
29

Ghislain MARY's avatar
Ghislain MARY committed
30
MSOpenH264Decoder::MSOpenH264Decoder(MSFilter *f)
31
	: mFilter(f), mDecoder(0), mUnpacker(0), mSPS(0), mPPS(0), mYUVMsg(0),
Ghislain MARY's avatar
Ghislain MARY committed
32
	mBitstream(0), mBitstreamSize(65536), mLastErrorReportTime(0),
33 34
	mWidth(MS_VIDEO_SIZE_UNKNOWN_W), mHeight(MS_VIDEO_SIZE_UNKNOWN_H),
	mInitialized(false), mFirstImageDecoded(false)
35 36 37
{
	long ret = WelsCreateDecoder(&mDecoder);
	if (ret != 0) {
Simon Morlat's avatar
Simon Morlat committed
38
		ms_error("OpenH264 decoder: Failed to create decoder: %li", ret);
Ghislain MARY's avatar
Ghislain MARY committed
39 40
	} else {
		mBitstream = static_cast<uint8_t *>(ms_malloc0(mBitstreamSize));
41 42 43 44
		WelsTraceCallback cb = &decoder_log;
		mDecoder->SetOption(DECODER_OPTION_TRACE_CALLBACK, (void*)&cb);
		int logLevel = WELS_LOG_WARNING;
		mDecoder->SetOption(DECODER_OPTION_TRACE_LEVEL, &logLevel);
45 46 47 48 49
	}
}

MSOpenH264Decoder::~MSOpenH264Decoder()
{
Ghislain MARY's avatar
Ghislain MARY committed
50 51 52
	if (mBitstream != 0) {
		ms_free(mBitstream);
	}
53 54 55 56 57 58 59
	if (mDecoder != 0) {
		WelsDestroyDecoder(mDecoder);
	}
}

void MSOpenH264Decoder::initialize()
{
60 61 62 63 64
	if (!mInitialized) {
		mFirstImageDecoded = false;
		mUnpacker=rfc3984_new();
		if (mDecoder != 0) {
			SDecodingParam params = { 0 };
65 66 67 68
#if (OPENH264_MAJOR == 1) && (OPENH264_MINOR >=6)
#else
			params.eOutputColorFormat = videoFormatI420;
#endif
69
			params.uiTargetDqLayer = (unsigned char) -1;
70
			params.eEcActiveIdc = ERROR_CON_FRAME_COPY_CROSS_IDR;
71
			params.sVideoProperty.size = sizeof(params.sVideoProperty);
72
			params.sVideoProperty.eVideoBsType = VIDEO_BITSTREAM_AVC;
73 74 75 76 77 78 79
			long ret = mDecoder->Initialize(&params);
			if (ret != 0) {
				ms_error("OpenH264 decoder: Failed to initialize: %li", ret);
			} else {
				ms_average_fps_init(&mFPS, "OpenH264 decoder: FPS=%f");
				mInitialized = true;
			}
80 81 82 83
		}
	}
}

Ghislain MARY's avatar
Ghislain MARY committed
84
void MSOpenH264Decoder::feed()
85 86
{
	if (!isInitialized()){
87
		ms_error("MSOpenH264Decoder::feed(): not initialized");
Ghislain MARY's avatar
Ghislain MARY committed
88
		ms_queue_flush(mFilter->inputs[0]);
89 90 91 92 93 94 95
		return;
	}

	MSQueue nalus;
	ms_queue_init(&nalus);

	mblk_t *im;
96
	bool requestPLI = false;
Ghislain MARY's avatar
Ghislain MARY committed
97
	while ((im = ms_queue_get(mFilter->inputs[0])) != NULL) {
Ghislain MARY's avatar
Ghislain MARY committed
98
		if ((getIDRPicId() == 0) && (mSPS != 0) && (mPPS != 0)) {
99 100 101
			// Push the sps/pps given in sprop-parameter-sets if any
			mblk_set_timestamp_info(mSPS, mblk_get_timestamp_info(im));
			mblk_set_timestamp_info(mPPS, mblk_get_timestamp_info(im));
102 103
			requestPLI |= (rfc3984_unpack(mUnpacker, mSPS, &nalus) < 0);
			requestPLI |= (rfc3984_unpack(mUnpacker, mPPS, &nalus) < 0);
104 105 106
			mSPS = 0;
			mPPS = 0;
		}
107
		requestPLI |= (rfc3984_unpack(mUnpacker, im, &nalus) < 0);
Ghislain MARY's avatar
Ghislain MARY committed
108
		if (!ms_queue_empty(&nalus)) {
Ghislain MARY's avatar
Ghislain MARY committed
109 110
			void * pData[3] = { 0 };
			SBufferInfo sDstBufInfo = { 0 };
Ghislain MARY's avatar
Ghislain MARY committed
111
			int len = nalusToFrame(&nalus);
112
			
113
			DECODING_STATE state = mDecoder->DecodeFrame2(mBitstream, len, (uint8_t**)pData, &sDstBufInfo);
Ghislain MARY's avatar
Ghislain MARY committed
114
			if (state != dsErrorFree) {
115
				ms_error("OpenH264 decoder: DecodeFrame2 failed: 0x%x", (int)state);
116 117 118
				if (mAVPFEnabled) {
					requestPLI = true;
				} else if (((mFilter->ticker->time - mLastErrorReportTime) > 5000) || (mLastErrorReportTime == 0)) {
Ghislain MARY's avatar
Ghislain MARY committed
119 120
					mLastErrorReportTime = mFilter->ticker->time;
					ms_filter_notify_no_arg(mFilter, MS_VIDEO_DECODER_DECODING_ERRORS);
Ghislain MARY's avatar
Ghislain MARY committed
121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137
				}
			}
			if (sDstBufInfo.iBufferStatus == 1) {
				uint8_t * pDst[3] = { 0 };
				pDst[0] = (uint8_t *)pData[0];
				pDst[1] = (uint8_t *)pData[1];
				pDst[2] = (uint8_t *)pData[2];

				// Update video size and (re)allocate YUV buffer if needed
				if ((mWidth != sDstBufInfo.UsrData.sSystemBuffer.iWidth)
					|| (mHeight != sDstBufInfo.UsrData.sSystemBuffer.iHeight)) {
					if (mYUVMsg) {
						freemsg(mYUVMsg);
					}
					mWidth = sDstBufInfo.UsrData.sSystemBuffer.iWidth;
					mHeight = sDstBufInfo.UsrData.sSystemBuffer.iHeight;
					mYUVMsg = ms_yuv_buf_alloc(&mOutbuf, mWidth, mHeight);
138
					ms_filter_notify_no_arg(mFilter,MS_FILTER_OUTPUT_FMT_CHANGED);
Ghislain MARY's avatar
Ghislain MARY committed
139 140 141 142 143 144 145 146 147 148 149 150 151 152
				}

				// Scale/copy frame to destination mblk_t
				for (int i = 0; i < 3; i++) {
					uint8_t *dst = mOutbuf.planes[i];
					uint8_t *src = pDst[i];
					int h = mHeight >> (( i > 0) ? 1 : 0);

					for(int j = 0; j < h; j++) {
						memcpy(dst, src, mOutbuf.strides[i]);
						dst += mOutbuf.strides[i];
						src += sDstBufInfo.UsrData.sSystemBuffer.iStride[(i == 0) ? 0 : 1];
					}
				}
Ghislain MARY's avatar
Ghislain MARY committed
153
				ms_queue_put(mFilter->outputs[0], dupmsg(mYUVMsg));
Ghislain MARY's avatar
Ghislain MARY committed
154 155

				// Update average FPS
Simon Morlat's avatar
Simon Morlat committed
156
				if (ms_average_fps_update(&mFPS, mFilter->ticker->time)) {
Ghislain MARY's avatar
Ghislain MARY committed
157 158 159 160 161 162
					ms_message("OpenH264 decoder: Frame size: %dx%d", mWidth, mHeight);
				}

				// Notify first decoded image
				if (!mFirstImageDecoded) {
					mFirstImageDecoded = true;
Ghislain MARY's avatar
Ghislain MARY committed
163
					ms_filter_notify_no_arg(mFilter, MS_VIDEO_DECODER_FIRST_IMAGE_DECODED);
Ghislain MARY's avatar
Ghislain MARY committed
164
				}
Ghislain MARY's avatar
Ghislain MARY committed
165 166 167 168

#if MSOPENH264_DEBUG
				ms_message("OpenH264 decoder: IDR pic id: %d, Frame num: %d, Temporal id: %d, VCL NAL: %d", getIDRPicId(), getFrameNum(), getTemporalId(), getVCLNal());
#endif
Ghislain MARY's avatar
Ghislain MARY committed
169
			}
170 171
		}
	}
172 173 174 175

	if (mAVPFEnabled && requestPLI) {
		ms_filter_notify_no_arg(mFilter, MS_VIDEO_DECODER_SEND_PLI);
	}
176 177 178 179
}

void MSOpenH264Decoder::uninitialize()
{
180 181
	if (mSPS != 0) {
		freemsg(mSPS);
182
		mSPS=NULL;
183 184 185
	}
	if (mPPS != 0) {
		freemsg(mPPS);
186
		mPPS=NULL;
187
	}
Ghislain MARY's avatar
Ghislain MARY committed
188 189
	if (mYUVMsg != 0) {
		freemsg(mYUVMsg);
190
		mYUVMsg=NULL;
Ghislain MARY's avatar
Ghislain MARY committed
191
	}
192 193 194
	if (mDecoder != 0) {
		mDecoder->Uninitialize();
	}
195 196 197 198
	if (mUnpacker){
		rfc3984_destroy(mUnpacker);
		mUnpacker=NULL;
	}
199 200
	mInitialized = false;
}
Ghislain MARY's avatar
Ghislain MARY committed
201

202 203 204 205 206 207 208 209 210 211 212 213 214 215 216
void MSOpenH264Decoder::provideSpropParameterSets(char *value, int valueSize)
{
	char *b64_sps = value;
	char *b64_pps = strchr(value, ',');
	if (b64_pps) {
		*b64_pps = '\0';
		++b64_pps;
		ms_message("OpenH264 decoder: Got sprop-parameter-sets sps=%s, pps=%s", b64_sps, b64_pps);
		mSPS = allocb(valueSize, 0);
		mSPS->b_wptr += b64::b64_decode(b64_sps, strlen(b64_sps), mSPS->b_wptr, valueSize);
		mPPS = allocb(valueSize, 0);
		mPPS->b_wptr += b64::b64_decode(b64_pps, strlen(b64_pps), mPPS->b_wptr, valueSize);
	}
}

Ghislain MARY's avatar
Ghislain MARY committed
217 218 219 220 221 222 223 224 225 226 227 228 229 230 231
void MSOpenH264Decoder::resetFirstImageDecoded()
{
	mFirstImageDecoded = false;
	mWidth = MS_VIDEO_SIZE_UNKNOWN_W;
	mHeight = MS_VIDEO_SIZE_UNKNOWN_H;
}

MSVideoSize MSOpenH264Decoder::getSize() const
{
	MSVideoSize size;
	size.width = mWidth;
	size.height = mHeight;
	return size;
}

Simon Morlat's avatar
Simon Morlat committed
232 233 234 235
float MSOpenH264Decoder::getFps()const{
	return ms_average_fps_get(&mFPS);
}

236 237 238 239 240
const MSFmtDescriptor * MSOpenH264Decoder::getOutFmt()const{
	MSVideoSize vsize={mWidth,mHeight};
	return ms_factory_get_video_format(mFilter->factory,"YUV420P",vsize,0,NULL);
}

Ghislain MARY's avatar
Ghislain MARY committed
241
int MSOpenH264Decoder::nalusToFrame(MSQueue *nalus)
Ghislain MARY's avatar
Ghislain MARY committed
242
{
Ghislain MARY's avatar
Ghislain MARY committed
243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265
	mblk_t *im;
	uint8_t *dst = mBitstream;
	uint8_t *end = mBitstream + mBitstreamSize;
	bool startPicture = true;

	while ((im = ms_queue_get(nalus)) != NULL) {
		uint8_t *src = im->b_rptr;
		int nalLen = im->b_wptr - src;
		if ((dst + nalLen + 128) > end) {
			int pos = dst - mBitstream;
			enlargeBitstream(mBitstreamSize + nalLen + 128);
			dst = mBitstream + pos;
			end = mBitstream + mBitstreamSize;
		}
		if ((src[0] == 0) && (src[1] == 0) && (src[2] == 0) && (src[3] == 1)) {
			// Workaround for stupid RTP H264 sender that includes nal markers
#if MSOPENH264_DEBUG
			ms_warning("OpenH264 decoder: stupid RTP H264 encoder");
#endif
			int size = im->b_wptr - src;
			memcpy(dst, src, size);
			dst += size;
		} else {
Ghislain MARY's avatar
Ghislain MARY committed
266
			uint8_t naluType = *src & 0x1f;
Ghislain MARY's avatar
Ghislain MARY committed
267 268 269 270 271 272 273 274 275 276 277
#if MSOPENH264_DEBUG
			if ((naluType != 1) && (naluType != 7) && (naluType != 8)) {
				ms_message("OpenH264 decoder: naluType=%d", naluType);
			}
			if (naluType == 7) {
				ms_message("OpenH264 decoder: Got SPS");
			}
			if (naluType == 8) {
				ms_message("OpenH264 decoder: Got PPS");
			}
#endif
Ghislain MARY's avatar
Ghislain MARY committed
278 279 280 281 282
			if (startPicture
				|| (naluType == 6) // SEI
				|| (naluType == 7) // SPS
				|| (naluType == 8) // PPS
				|| ((naluType >= 14) && (naluType <= 18))) { // Reserved
Ghislain MARY's avatar
Ghislain MARY committed
283
				*dst++ = 0;
Ghislain MARY's avatar
Ghislain MARY committed
284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299
				startPicture = false;
			}

			// Prepend nal marker
			*dst++ = 0;
			*dst++ = 0;
			*dst++ = 1;
			*dst++ = *src++;
			while (src < (im->b_wptr - 3)) {
				if ((src[0] == 0) && (src[1] == 0) && (src[2] < 3)) {
					*dst++ = 0;
					*dst++ = 0;
					*dst++ = 3;
					src += 2;
				}
				*dst++ = *src++;
Ghislain MARY's avatar
Ghislain MARY committed
300
			}
Ghislain MARY's avatar
Ghislain MARY committed
301 302 303
			while (src < im->b_wptr) {
				*dst++ = *src++;
			}
Ghislain MARY's avatar
Ghislain MARY committed
304
		}
Ghislain MARY's avatar
Ghislain MARY committed
305 306 307 308
		freemsg(im);
	}
	return dst - mBitstream;
}
Ghislain MARY's avatar
Ghislain MARY committed
309

Ghislain MARY's avatar
Ghislain MARY committed
310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351
void MSOpenH264Decoder::enlargeBitstream(int newSize)
{
	mBitstreamSize = newSize;
	mBitstream = static_cast<uint8_t *>(ms_realloc(mBitstream, mBitstreamSize));
}

int32_t MSOpenH264Decoder::getFrameNum()
{
	int32_t frameNum = -1;
	int ret = mDecoder->GetOption(DECODER_OPTION_FRAME_NUM, &frameNum);
	if (ret != 0) {
		ms_error("OpenH264 decoder: Failed getting frame number: %d", ret);
	}
	return frameNum;
}

int32_t MSOpenH264Decoder::getIDRPicId()
{
	int32_t IDRPicId = -1;
	int ret = mDecoder->GetOption(DECODER_OPTION_IDR_PIC_ID, &IDRPicId);
	if (ret != 0) {
		ms_error("OpenH264 decoder: Failed getting IDR pic id: %d", ret);
	}
	return IDRPicId;
}

int32_t MSOpenH264Decoder::getTemporalId()
{
	int32_t temporalId = -1;
	int ret = mDecoder->GetOption(DECODER_OPTION_TEMPORAL_ID, &temporalId);
	if (ret != 0) {
		ms_error("OpenH264 decoder: Failed getting temporal id: %d", ret);
	}
	return temporalId;
}

int32_t MSOpenH264Decoder::getVCLNal()
{
	int32_t vclNal = -1;
	int ret = mDecoder->GetOption(DECODER_OPTION_VCL_NAL, &vclNal);
	if (ret != 0) {
		ms_error("OpenH264 decoder: Failed getting VCL NAL: %d", ret);
Ghislain MARY's avatar
Ghislain MARY committed
352
	}
Ghislain MARY's avatar
Ghislain MARY committed
353
	return vclNal;
Ghislain MARY's avatar
Ghislain MARY committed
354
}