media-codec-decoder.cpp 10.4 KB
Newer Older
1
/*
2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17
 Mediastreamer2 media-codec-decoder.cpp
 Copyright (C) 2018 Belledonne Communications SARL

 This program is free software; you can redistribute it and/or
 modify it under the terms of the GNU General Public License
 as published by the Free Software Foundation; either version 2
 of the License, or (at your option) any later version.

 This program is distributed in the hope that it will be useful,
 but WITHOUT ANY WARRANTY; without even the implied warranty of
 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 GNU General Public License for more details.

 You should have received a copy of the GNU General Public License
 along with this program; if not, write to the Free Software
 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
18 19
*/

20 21 22 23 24 25
#include <algorithm>
#include <cstdint>
#include <cstring>
#include <sstream>
#include <stdexcept>

François Grisez's avatar
François Grisez committed
26 27 28
#include <jni.h>
#include <media/NdkMediaFormat.h>
#include <ortp/b64.h>
29
#include <ortp/str_utils.h>
François Grisez's avatar
François Grisez committed
30

31
#include "mediastreamer2/formats.h"
32 33
#include "mediastreamer2/msfilter.h"
#include "mediastreamer2/msticker.h"
François Grisez's avatar
François Grisez committed
34

35
#include "android_mediacodec.h"
36 37
#include "h26x-utils.h"
#include "media-codec-decoder.h"
38 39
#include "media-codec-h264-decoder.h"
#include "media-codec-h265-decoder.h"
40

François Grisez's avatar
François Grisez committed
41
using namespace b64;
42
using namespace mediastreamer;
43
using namespace std;
44

45 46
namespace mediastreamer {

47
MediaCodecDecoder::MediaCodecDecoder(const std::string &mime) {
48
	try {
49 50 51 52 53 54 55
		_impl = AMediaCodec_createDecoderByType(mime.c_str());
		if (_impl == nullptr) {
			ostringstream msg;
			msg << "could not create MediaCodec for '" << mime << "'";
			throw runtime_error(msg.str());
		}
		_format = createFormat(mime);
56
		_bufAllocator = ms_yuv_buf_allocator_new();
57
		_naluHeader.reset(H26xToolFactory::get(mime).createNaluHeader());
58
		_psStore.reset(H26xToolFactory::get(mime).createParameterSetsStore());
59
		startImpl();
60
	} catch (const runtime_error &e) {
61 62 63 64
		if (_impl) AMediaCodec_delete(_impl);
		if (_format) AMediaFormat_delete(_format);
		if (_bufAllocator) ms_yuv_buf_allocator_free(_bufAllocator);
		throw e;
65 66 67 68
	}
}

MediaCodecDecoder::~MediaCodecDecoder() {
69
	AMediaCodec_delete(_impl);
70 71 72
	ms_yuv_buf_allocator_free(_bufAllocator);
}

73
bool MediaCodecDecoder::setParameterSets(MSQueue *parameterSets, uint64_t timestamp) {
74
	if (!feed(parameterSets, timestamp, true)) {
75
		ms_error("MediaCodecDecoder: parameter sets has been refused by the decoder.");
76
		return false;
77
	}
78
	_needParameters = false;
79
	return true;
80
}
81

82
bool MediaCodecDecoder::feed(MSQueue *encodedFrame, uint64_t timestamp) {
83
	bool status = false;
84 85 86

	_psStore->extractAllPs(encodedFrame);
	if (_psStore->hasNewParameters()) {
87
		ms_message("MediaCodecDecoder: new paramter sets received");
88 89 90 91
		_needParameters = true;
		_psStore->acknowlege();
	}

92
	if (_needParameters) {
93 94 95 96
		MSQueue parameters;
		ms_queue_init(&parameters);
		_psStore->fetchAllPs(&parameters);
		if (!setParameterSets(&parameters, timestamp)) {
97
			ms_error("MediaCodecDecoder: waiting for parameter sets.");
98 99
			goto clean;
		}
100 101 102 103
	}

	if (_needKeyFrame) {
		if (!isKeyFrame(encodedFrame)) {
104
			ms_error("MediaCodecDecoder: waiting for key frame.");
105 106 107
			goto clean;
		}
		_needKeyFrame = false;
108
	}
109 110 111 112 113

	if (!feed(encodedFrame, timestamp, false)) {
		goto clean;
	}

114
	_pendingFrames++;
115 116 117
	status = true;

clean:
118
	ms_queue_flush(encodedFrame);
119
	return status;
120 121 122 123 124 125 126 127 128 129 130 131 132 133
}

mblk_t *MediaCodecDecoder::fetch() {
	mblk_t *om = nullptr;
	AMediaImage image = {0};
	int dst_pix_strides[4] = {1, 1, 1, 1};
	MSRect dst_roi = {0};
	AMediaCodecBufferInfo info;
	ssize_t oBufidx = -1;

	if (_impl == nullptr || _pendingFrames <= 0) goto end;

	oBufidx = AMediaCodec_dequeueOutputBuffer(_impl, &info, _timeoutUs);
	if (oBufidx == AMEDIACODEC_INFO_OUTPUT_FORMAT_CHANGED) {
134
		ms_message("MediaCodecDecoder: output format has changed.");
135 136 137 138 139
		oBufidx = AMediaCodec_dequeueOutputBuffer(_impl, &info, _timeoutUs);
	}

	if (oBufidx < 0) {
		if (oBufidx == AMEDIA_ERROR_UNKNOWN) {
140
			ms_error("MediaCodecDecoder: AMediaCodec_dequeueOutputBuffer() had an exception");
141
		} else if (oBufidx != AMEDIACODEC_INFO_TRY_AGAIN_LATER) {
142
			ms_error("MediaCodecDecoder: unknown error while dequeueing an output buffer (oBufidx=%zd)", oBufidx);
143 144 145 146 147 148 149
		}
		goto end;
	}

	_pendingFrames--;

	if (AMediaCodec_getOutputImage(_impl, oBufidx, &image) <= 0) {
150
		ms_error("MediaCodecDecoder: AMediaCodec_getOutputImage() failed");
151 152 153 154
		goto end;
	}

	MSPicture pic;
155
	om = ms_yuv_buf_allocator_get(_bufAllocator, &pic, image.crop_rect.w, image.crop_rect.h);
156 157 158 159 160 161 162 163 164
	ms_yuv_buf_copy_with_pix_strides(image.buffers, image.row_strides, image.pixel_strides, image.crop_rect,
										pic.planes, pic.strides, dst_pix_strides, dst_roi);
	AMediaImage_close(&image);

end:
	if (oBufidx >= 0) AMediaCodec_releaseOutputBuffer(_impl, oBufidx, FALSE);
	return om;
}

165 166 167 168 169 170
MediaCodecDecoder *MediaCodecDecoder::createDecoder(const std::string &mime) {
	if (mime == "video/avc") return new MediaCodecH264Decoder();
	else if (mime == "video/hevc") return new MediaCodecH265Decoder();
	else throw invalid_argument(mime);
}

171
AMediaFormat *MediaCodecDecoder::createFormat(const std::string &mime) const {
172 173 174 175 176
	AMediaFormat *format = AMediaFormat_new();
	AMediaFormat_setString(format, "mime", mime.c_str());
	AMediaFormat_setInt32(format, "color-format", 0x7f420888);
	AMediaFormat_setInt32(format, "max-width", 1920);
	AMediaFormat_setInt32(format, "max-height", 1920);
177
	AMediaFormat_setInt32(format, "priority", 0);
178 179
	return format;
}
180

181 182 183
void MediaCodecDecoder::startImpl() {
	media_status_t status = AMEDIA_OK;
	ostringstream errMsg;
184
	ms_message("MediaCodecDecoder: starting decoder");
185
	if ((status = AMediaCodec_configure(_impl, _format, nullptr, nullptr, 0)) != AMEDIA_OK) {
186
		errMsg << "configuration failure: " << int(status);
187
		throw runtime_error(errMsg.str());
188 189 190 191 192 193 194 195
	}

	if ((status = AMediaCodec_start(_impl)) != AMEDIA_OK) {
		errMsg << "starting failure: " << int(status);
		throw runtime_error(errMsg.str());
	}
}

196
void MediaCodecDecoder::stopImpl() {
197
	ms_message("MediaCodecDecoder: stopping decoder");
198 199 200
	AMediaCodec_stop(_impl);
}

201
bool MediaCodecDecoder::feed(MSQueue *encodedFrame, uint64_t timestamp, bool isPs) {
202 203 204 205 206 207 208
	unique_ptr<H26xNaluHeader> header;
	header.reset(H26xToolFactory::get("video/avc").createNaluHeader());
	for(mblk_t *m = ms_queue_peek_first(encodedFrame); !ms_queue_end(encodedFrame, m); m = ms_queue_next(encodedFrame, m)) {
		header->parse(m->b_rptr);
		ms_message("MediaCodecDecoder: nalu type %d", int(header->getAbsType()));
	}

209 210
	H26xUtils::nalusToByteStream(encodedFrame, _bitstream);

211 212 213 214
	if (_impl == nullptr) return false;

	ssize_t iBufidx = AMediaCodec_dequeueInputBuffer(_impl, _timeoutUs);
	if (iBufidx < 0) {
215
		ms_error("MediaCodecDecoder: %s.", iBufidx == -1 ? "no buffer available for queuing this frame ! Decoder is too slow" : "AMediaCodec_dequeueInputBuffer() had an exception");
216 217 218 219 220 221
		return false;
	}

	size_t bufsize;
	uint8_t *buf = AMediaCodec_getInputBuffer(_impl, iBufidx, &bufsize);
	if (buf == nullptr) {
222
		ms_error("MediaCodecDecoder: AMediaCodec_getInputBuffer() returned NULL");
223 224 225
		return false;
	}

226
	size_t size = _bitstream.size();
227 228 229 230
	if (size > bufsize) {
		ms_error("Cannot copy the all the bitstream into the input buffer size : %zu and bufsize %zu", size, bufsize);
		size = min(size, bufsize);
	}
231
	memcpy(buf, _bitstream.data(), size);
232 233 234

	uint32_t flags = isPs ? BufferFlag::CodecConfig : BufferFlag::None;
	if (AMediaCodec_queueInputBuffer(_impl, iBufidx, 0, size, timestamp * 1000ULL, flags) != 0) {
235
		ms_error("MediaCodecDecoder: AMediaCodec_queueInputBuffer() had an exception");
236 237 238 239 240 241
		return false;
	}

	return true;
}

242 243
bool MediaCodecDecoder::isKeyFrame(const MSQueue *frame) const {
	for (const mblk_t *nalu = ms_queue_peek_first(frame); !ms_queue_end(frame, nalu); nalu = ms_queue_next(frame, nalu)) {
244 245 246 247 248 249
		_naluHeader->parse(nalu->b_rptr);
		if (_naluHeader->getAbsType().isKeyFramePart()) return true;
	}
	return false;
}

250
MediaCodecDecoderFilterImpl::MediaCodecDecoderFilterImpl(MSFilter *f, const std::string &mime):
251
	DecodingFilterImpl(f),
252
	_vsize({0, 0}),
253
	_unpacker(H26xToolFactory::get(mime).createNalUnpacker()) {
254

255
	try {
256
		_codec.reset(MediaCodecDecoder::createDecoder(mime));
257 258
		ms_message("MediaCodecDecoder: initialization");
		ms_average_fps_init(&_fps, " H26x decoder: FPS: %f");
259
	} catch (const runtime_error &e) {
260
		ms_error("MediaCodecDecoder: %s", e.what());
261 262
		_codec.reset(nullptr);
	}
263 264 265 266
}

void MediaCodecDecoderFilterImpl::preprocess() {
	_firstImageDecoded = false;
267
	if (_codec) _codec->waitForKeyFrame();
268 269 270
}

void MediaCodecDecoderFilterImpl::process() {
271
	bool requestPli = false;
François Grisez's avatar
François Grisez committed
272
	MSQueue frame;
273

274
	if (_codec == nullptr) {
275
		ms_queue_flush(getInput(0));
276 277 278
		return;
	}

279
	ms_queue_init(&frame);
280

281
	while (mblk_t *im = ms_queue_get(getInput(0))) {
282
		NalUnpacker::Status unpacking_ret = _unpacker->unpack(im, &frame);
283 284 285 286

		if (!unpacking_ret.frameAvailable) continue;

		if (unpacking_ret.frameCorrupted) {
287
			ms_warning("MediaCodecDecoder: corrupted frame");
288
			requestPli = true;
289
			if (_freezeOnError) {
290
				ms_queue_flush(&frame);
291
				_codec->waitForKeyFrame();
292
				continue;
293
			}
294
		}
295

296 297
		struct timespec ts;
		clock_gettime(CLOCK_MONOTONIC, &ts);
298
		uint64_t tsMs = (ts.tv_sec * 1000ULL) + (ts.tv_nsec / 1000000ULL) + 10ULL;
299

300
		requestPli = !_codec->feed(&frame, tsMs);
301 302

		ms_queue_flush(&frame);
303
	}
304

305
	mblk_t *om;
306
	while ((om = _codec->fetch()) != nullptr) {
307 308 309 310 311 312
		MSPicture pic;
		ms_yuv_buf_init_from_mblk(&pic, om);
		_vsize.width = pic.w;
		_vsize.height = pic.h;

		if (!_firstImageDecoded) {
313
			ms_message("MediaCodecDecoder: first frame decoded %ix%i", _vsize.width, _vsize.height);
314
			_firstImageDecoded = true;
315
			notify(MS_VIDEO_DECODER_FIRST_IMAGE_DECODED);
316
		}
317

318 319
		ms_average_fps_update(&_fps, getTime());
		ms_queue_put(getOutput(0), om);
320 321
	}

322
	if (_avpfEnabled && requestPli) {
323
		notify(MS_VIDEO_DECODER_SEND_PLI);
324 325 326 327
	}
}

void MediaCodecDecoderFilterImpl::postprocess() {
328
	_unpacker->reset();
329 330 331 332 333 334 335 336 337 338 339 340 341 342 343
}

void MediaCodecDecoderFilterImpl::resetFirstImage() {
	_firstImageDecoded = false;
}

MSVideoSize MediaCodecDecoderFilterImpl::getVideoSize() const {
	return _firstImageDecoded ? _vsize : MS_VIDEO_SIZE_UNKNOWN;
}

float MediaCodecDecoderFilterImpl::getFps() const {
	return ms_average_fps_get(&_fps);
}

const MSFmtDescriptor *MediaCodecDecoderFilterImpl::getOutFmt() const {
344
	return ms_factory_get_video_format(getFactory(), "YUV420P", ms_video_size_make(_vsize.width, _vsize.height), 0, nullptr);
345 346 347 348 349 350 351 352
}

void MediaCodecDecoderFilterImpl::enableAvpf(bool enable) {
	_avpfEnabled = enable;
}

void MediaCodecDecoderFilterImpl::enableFreezeOnError(bool enable) {
	_freezeOnError = enable;
353
	ms_message("MediaCodecDecoder: freeze on error %s", _freezeOnError ? "enabled" : "disabled");
354 355 356
}

} // namespace mediastreamer