media-codec-encoder.cpp 12.9 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16
/*
mediastreamer2 mediacodech264enc.c
Copyright (C) 2015 Belledonne Communications SARL

This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; either version 2
of the License, or (at your option) any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
17
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
18 19
*/

François Grisez's avatar
François Grisez committed
20
#include <cstdint>
21
#include <cstring>
François Grisez's avatar
François Grisez committed
22
#include <sstream>
23 24
#include <stdexcept>

25
#include <jni.h>
26 27 28 29
#include <ortp/b64.h>

#include "mediastreamer2/msjava.h"
#include "mediastreamer2/msticker.h"
30

31
#include "android_mediacodec.h"
32 33
#include "h26x-utils.h"
#include "media-codec-encoder.h"
34

35
using namespace mediastreamer;
36
using namespace std;
37

38 39
namespace mediastreamer {

40 41 42
MediaCodecEncoder::MediaCodecEncoder(const std::string &mime):
	_mime(mime),
	_psInserter(H26xToolFactory::get(mime).createParameterSetsInserter()) {
43 44 45 46 47 48 49 50 51 52 53

	try {
		_vsize.width = 0;
		_vsize.height = 0;
		// We shall allocate the MediaCodec encoder the sooner as possible and before the decoder, because
		// some phones hardware encoder and decoders can't be allocated at the same time.
		createImpl();
	} catch (const runtime_error &e) {
		ms_error("MSMediaCodecH264Enc: %s", e.what());
	}
}
54

55 56 57
MediaCodecEncoder::~MediaCodecEncoder() {
	if (_impl) AMediaCodec_delete(_impl);
}
58

59 60 61 62 63 64 65 66 67
void MediaCodecEncoder::setBitrate(int bitrate) {
	_bitrate = bitrate;
	if (isRunning() && _impl) {
		AMediaFormat *afmt = AMediaFormat_new();
		// update the output bitrate
		AMediaFormat_setInt32(afmt, "video-bitrate", (_bitrate * 9)/10);
		AMediaCodec_setParams(_impl, afmt);
		AMediaFormat_delete(afmt);
	}
68
}
69

70 71 72 73 74 75 76 77 78 79 80 81 82 83 84
void MediaCodecEncoder::start() {
	try {
		if (_isRunning) {
			ms_warning("MediaCodecEncoder: encoder already started");
			return;
		}
		configureImpl();
		if (AMediaCodec_start(_impl) != AMEDIA_OK) {
			throw runtime_error("could not start encoder.");
		}
		_isRunning = true;
		ms_message("MSMediaCodecH264Enc: encoder successfully started");
	} catch (const runtime_error &e) {
		ms_error("MSMediaCodecH264Enc: %s", e.what());
	}
85
}
86

87 88 89 90 91 92 93 94 95 96 97 98 99 100
void MediaCodecEncoder::stop() {
	if (!_isRunning) {
		ms_warning("MediaCodecEncoder: encoder already stopped");
		return;
	}
	if (_impl) {
		AMediaCodec_flush(_impl);
		AMediaCodec_stop(_impl);
		// It is preferable to reset the encoder, otherwise it may not accept a new configuration while returning in preprocess().
		// This was observed at least on Moto G2, with qualcomm encoder.
		AMediaCodec_reset(_impl);
	}
	_psInserter->flush();
	_isRunning = false;
101
	_pendingFrames = 0;
102
}
103

104
void MediaCodecEncoder::feed(mblk_t *rawData, uint64_t time, bool requestIFrame) {
105 106 107 108 109 110
	if (!_isRunning) {
		ms_error("MSMediaCodecH264Enc: encoder not running. Dropping buffer.");
		freemsg(rawData);
		return;
	}

François Grisez's avatar
François Grisez committed
111
	if (_recoveryMode && time % 5000 == 0) {
112 113 114 115 116 117
		try {
			if (_impl == nullptr) createImpl();
			configureImpl();
			_recoveryMode = false;
		} catch (const runtime_error &e) {
			ms_error("MSMediaCodecH264Enc: %s", e.what());
118
			ms_error("MSMediaCodecH264Enc: AMediaCodec_reset() was not sufficient, will recreate the encoder in a moment...");
119
			AMediaCodec_delete(_impl);
120
		}
121 122
	}

123 124 125
	MSPicture pic;
	ms_yuv_buf_init_from_mblk(&pic, rawData);

126 127 128 129 130 131 132 133 134
	if (requestIFrame) {
		AMediaFormat *afmt = AMediaFormat_new();
		/*Force a key-frame*/
		AMediaFormat_setInt32(afmt, "request-sync", 0);
		AMediaCodec_setParams(_impl, afmt);
		AMediaFormat_delete(afmt);
		ms_error("MSMediaCodecH264Enc: I-frame requested to MediaCodec");
	}

135 136
	ssize_t ibufidx = AMediaCodec_dequeueInputBuffer(_impl, _timeoutUs);
	if (ibufidx < 0) {
137 138 139
		if (ibufidx == AMEDIA_ERROR_UNKNOWN) {
			ms_error("MSMediaCodecH264Enc: AMediaCodec_dequeueInputBuffer() had an exception");
		} else if (ibufidx == -1) {
140
			ms_error("MSMediaCodecH264Enc: no input buffer available.");
141 142 143
		} else {
			ms_error("MSMediaCodecH264Enc: unknown error while requesting an input buffer (%zd)", ibufidx);
		}
144 145 146
		return;
	}

147
	size_t bufsize;
François Grisez's avatar
François Grisez committed
148 149
	uint8_t *buf = AMediaCodec_getInputBuffer(_impl, ibufidx, &bufsize);
	if (buf == nullptr) {
150 151 152 153 154 155 156 157 158 159 160 161 162
		ms_error("MSMediaCodecH264Enc: obtained InputBuffer, but no address.");
		return;
	}

	AMediaImage image;
	if (AMediaCodec_getInputImage(_impl, ibufidx, &image)) {
		if (image.format == 35 /* YUV_420_888 */) {
			MSRect src_roi = {0, 0, pic.w, pic.h};
			int src_pix_strides[4] = {1, 1, 1, 1};
			ms_yuv_buf_copy_with_pix_strides(pic.planes, pic.strides, src_pix_strides, src_roi, image.buffers, image.row_strides, image.pixel_strides, image.crop_rect);
			bufsize = image.row_strides[0] * image.height * 3 / 2;
		} else {
			ms_error("MSMediaCodecH264Enc: encoder requires non YUV420 format");
163
		}
164
		AMediaImage_close(&image);
165
	}
166

167
	if (AMediaCodec_queueInputBuffer(_impl, ibufidx, 0, bufsize, time * 1000, 0) == AMEDIA_ERROR_BASE) {
François Grisez's avatar
François Grisez committed
168 169 170
		ms_error("MSMediaCodecH264Enc: error while queuing input buffer");
		return;
	}
171 172

	_pendingFrames++;
François Grisez's avatar
François Grisez committed
173
	freemsg(rawData);
174 175
}

François Grisez's avatar
François Grisez committed
176
bool MediaCodecEncoder::fetch(MSQueue *encodedData) {
177
	MSQueue outq;
François Grisez's avatar
François Grisez committed
178 179 180 181
	AMediaCodecBufferInfo info;
	uint8_t *buf;
	size_t bufsize;

182
	if (!_isRunning || _recoveryMode || _pendingFrames <= 0) return false;
François Grisez's avatar
François Grisez committed
183

184
	ms_queue_init(&outq);
185

François Grisez's avatar
François Grisez committed
186
	ssize_t obufidx = AMediaCodec_dequeueOutputBuffer(_impl, &info, _timeoutUs);
187 188 189 190
	if (obufidx == AMEDIACODEC_INFO_OUTPUT_FORMAT_CHANGED) {
		ms_message("MSMediaCodecH264Enc: output format has changed.");
		obufidx = AMediaCodec_dequeueOutputBuffer(_impl, &info, _timeoutUs);
	}
191
	if (obufidx < 0) {
François Grisez's avatar
François Grisez committed
192
		if (obufidx == AMEDIA_ERROR_UNKNOWN) {
193 194 195 196
			ms_error("MSMediaCodecH264Enc: AMediaCodec_dequeueOutputBuffer() had an exception, MediaCodec is lost");
			// MediaCodec need to be reset  at this point because it may have become irrevocably crazy.
			AMediaCodec_reset(_impl);
			_recoveryMode = true;
197
		} else if (obufidx != AMEDIACODEC_INFO_TRY_AGAIN_LATER) {
198 199
			ms_error("MSMediaCodecH264Enc: unknown error while requesting an output buffer (%zd)", obufidx);
		}
François Grisez's avatar
François Grisez committed
200 201 202
		return false;
	}

203 204
	_pendingFrames--;

François Grisez's avatar
François Grisez committed
205 206 207 208
	if ((buf = AMediaCodec_getOutputBuffer(_impl, obufidx, &bufsize)) == nullptr) {
		ms_error("MSMediaCodecH264Enc: AMediaCodec_getOutputBuffer() returned nullptr");
		AMediaCodec_releaseOutputBuffer(_impl, obufidx, FALSE);
		return false;
209
	}
210

211
	H26xUtils::byteStreamToNalus(buf + info.offset, info.size, &outq);
212
	_psInserter->process(&outq, encodedData);
François Grisez's avatar
François Grisez committed
213 214

	AMediaCodec_releaseOutputBuffer(_impl, obufidx, FALSE);
François Grisez's avatar
François Grisez committed
215
	return true;
216
}
217

218 219 220 221 222 223
void MediaCodecEncoder::createImpl() {
	_impl = AMediaCodec_createEncoderByType(_mime.c_str());
	if (_impl == nullptr) {
		throw runtime_error("could not create MediaCodec");
	}
}
224

225
void MediaCodecEncoder::configureImpl() {
226
	AMediaFormat *format = createMediaFormat();
François Grisez's avatar
François Grisez committed
227 228

	ms_message("configuring MediaCodec with the following parameters:");
229
	ms_message("%s", getMediaForamtAsString().str().c_str());
230 231 232 233 234 235

	media_status_t status = AMediaCodec_configure(_impl, format, nullptr, nullptr, AMEDIACODEC_CONFIGURE_FLAG_ENCODE);
	AMediaFormat_delete(format);

	if (status != 0) {
		throw runtime_error("could not configure encoder.");
236
	}
237

François Grisez's avatar
François Grisez committed
238 239 240
	ms_message("MSMediaCodecH264Enc: encoder successfully configured.");
}

241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256
AMediaFormat *MediaCodecEncoder::createMediaFormat() const {
	AMediaFormat *format = AMediaFormat_new();
	AMediaFormat_setString(format, "mime", _mime.c_str());
	AMediaFormat_setInt32(format, "color-format", _colorFormat);
	AMediaFormat_setInt32(format, "width", _vsize.width);
	AMediaFormat_setInt32(format, "height", _vsize.height);
	AMediaFormat_setInt32(format, "frame-rate", _fps);
	AMediaFormat_setInt32(format, "bitrate", (_bitrate * 9)/10); // take a margin
	AMediaFormat_setInt32(format, "bitrate-mode", _bitrateMode);
	AMediaFormat_setInt32(format, "i-frame-interval", _iFrameInterval);
	AMediaFormat_setInt32(format, "latency", _encodingLatency);
	AMediaFormat_setInt32(format, "priority", _priority);
	return format;
}

std::ostringstream MediaCodecEncoder::getMediaForamtAsString() const {
François Grisez's avatar
François Grisez committed
257
	ostringstream os;
258
	os << "\tmime: " << _mime << endl;
François Grisez's avatar
François Grisez committed
259 260 261 262

	os.unsetf(ios::basefield);
	os.setf(ios::hex);
	os.setf(ios::showbase);
263
	os << "\tcolor-format: " << _colorFormat << endl;
François Grisez's avatar
François Grisez committed
264 265 266 267
	os.unsetf(ios::basefield);
	os.setf(ios::dec);
	os.unsetf(ios::showbase);

268
	os << "\tvideo size: " << _vsize.width << "x" << _vsize.height << endl;
269 270
	os << "\tframe-rate: " << _fps << " fps" << endl;
	os << "\tbitrate: " << _bitrate << " b/s" << endl;
271 272
	os << "\tbitrate-mode: " << _bitrateMode << endl;
	os << "\ti-frame-intervale: " << _iFrameInterval << endl;
273
	os << "\tlatency: " << _encodingLatency << endl;
274
	os << "\tpriority: " << _priority << endl;
275
	return os;
276 277 278
}

// Public methods
279 280
MediaCodecEncoderFilterImpl::MediaCodecEncoderFilterImpl(MSFilter *f, MediaCodecEncoder *encoder, NalPacker *packer, const MSVideoConfiguration *defaultVConfList):
	_f(f), _encoder(encoder), _packer(packer), _defaultVConfList(defaultVConfList) {
281

282
	setVideoConfigurations(nullptr);
283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298
	_vconf = ms_video_find_best_configuration_for_size(_vconfList, MS_VIDEO_SIZE_CIF, ms_factory_get_cpu_count(f->factory));
	ms_video_starter_init(&_starter);

	_packer->setPacketizationMode(NalPacker::NonInterleavedMode);
	_packer->enableAggregation(false);
}

void MediaCodecEncoderFilterImpl::preprocess() {
	_encoder->start();
	ms_video_starter_init(&_starter);
	ms_iframe_requests_limiter_init(&_iframeLimiter, 1000);
}

void MediaCodecEncoderFilterImpl::process() {
	/*First queue input image*/
	if (mblk_t *im = ms_queue_peek_last(_f->inputs[0])) {
299 300 301
		bool requestIFrame = false;
		if (ms_iframe_requests_limiter_iframe_requested(&_iframeLimiter, _f->ticker->time) ||
		        (!_avpfEnabled && ms_video_starter_need_i_frame(&_starter, _f->ticker->time))) {
302
			ms_message("MSMediaCodecH264Enc: requesting I-frame to the encoder.");
303 304 305 306
			requestIFrame = true;
			ms_iframe_requests_limiter_notify_iframe_sent(&_iframeLimiter, _f->ticker->time);
		}
		_encoder->feed(dupmsg(im), _f->ticker->time, requestIFrame);
307 308 309 310
	}
	ms_queue_flush(_f->inputs[0]);

	/*Second, dequeue possibly pending encoded frames*/
François Grisez's avatar
François Grisez committed
311 312
	MSQueue nalus;
	ms_queue_init(&nalus);
François Grisez's avatar
François Grisez committed
313
	while (_encoder->fetch(&nalus)) {
314 315 316 317
		if (!_firstFrameDecoded) {
			_firstFrameDecoded = true;
			ms_video_starter_first_frame(&_starter, _f->ticker->time);
		}
318
		_packer->pack(&nalus, _f->outputs[0], static_cast<uint32_t>(_f->ticker->time * 90LL));
François Grisez's avatar
François Grisez committed
319
	}
320 321 322 323 324
}

void MediaCodecEncoderFilterImpl::postprocess() {
	_packer->flush();
	_encoder->stop();
325
	_firstFrameDecoded = false;
326
}
327

328 329 330
int MediaCodecEncoderFilterImpl::getBitrate() const {
	return _vconf.required_bitrate;
}
331

332
void MediaCodecEncoderFilterImpl::setBitrate(int br) {
333
	if (_encoder->isRunning()) {
334 335 336
		/* Encoding is already ongoing, do not change video size, only bitrate. */
		_vconf.required_bitrate = br;
		/* apply the new bitrate request to the running MediaCodec*/
337
		setVideoConfiguration(&_vconf);
338 339 340
	} else {
		MSVideoConfiguration best_vconf = ms_video_find_best_configuration_for_size_and_bitrate(_vconfList, _vconf.vsize, ms_factory_get_cpu_count(_f->factory),  br);
		setVideoConfiguration(&best_vconf);
341
	}
342
}
343

344 345
int MediaCodecEncoderFilterImpl::setVideoConfiguration(const MSVideoConfiguration *vconf) {
	if (vconf != &_vconf) memcpy(&_vconf, vconf, sizeof(MSVideoConfiguration));
346

347
	ms_message("Video configuration set: bitrate=%d bits/s, fps=%f, vsize=%dx%d", _vconf.required_bitrate, _vconf.fps, _vconf.vsize.width, _vconf.vsize.height);
348

349 350 351 352
	_encoder->setVideoSize(_vconf.vsize);
	_encoder->setFps(_vconf.fps);
	_encoder->setBitrate(_vconf.required_bitrate);

353 354
	return 0;
}
355

356 357 358 359
void MediaCodecEncoderFilterImpl::setFps(float  fps) {
	_vconf.fps = fps;
	setVideoConfiguration(&_vconf);
}
360

361 362 363
float MediaCodecEncoderFilterImpl::getFps() const {
	return _vconf.fps;
}
364

365 366 367
MSVideoSize MediaCodecEncoderFilterImpl::getVideoSize() const {
	return _vconf.vsize;
}
368

369
void MediaCodecEncoderFilterImpl::enableAvpf(bool enable) {
370
	ms_message("MSMediaCodecH264Enc: AVPF %s", enable ? "enabled" : "disabled");
371 372
	_avpfEnabled = enable;
}
373

374
void MediaCodecEncoderFilterImpl::setVideoSize(const MSVideoSize &vsize) {
375
	MSVideoConfiguration best_vconf = ms_video_find_best_configuration_for_size_and_bitrate(_vconfList, vsize, ms_factory_get_cpu_count(_f->factory), _vconf.required_bitrate);
376 377 378 379 380
	_vconf.vsize = vsize;
	_vconf.fps = best_vconf.fps;
	_vconf.bitrate_limit = best_vconf.bitrate_limit;
	setVideoConfiguration(&_vconf);
}
381

382 383 384 385
void MediaCodecEncoderFilterImpl::notifyPli() {
	ms_message("MSMediaCodecH264Enc: PLI requested");
	ms_iframe_requests_limiter_request_iframe(&_iframeLimiter);
}
386

387 388 389 390
void MediaCodecEncoderFilterImpl::notifyFir() {
	ms_message("MSMediaCodecH264Enc: FIR requested");
	ms_iframe_requests_limiter_request_iframe(&_iframeLimiter);
}
391

392 393 394
const MSVideoConfiguration *MediaCodecEncoderFilterImpl::getVideoConfiguratons() const {
	return _vconfList;
}
395

396
void MediaCodecEncoderFilterImpl::setVideoConfigurations(const MSVideoConfiguration *vconfs) {
397
	_vconfList = vconfs ? vconfs : _defaultVConfList;
398
}
399

400
}