Making an abstraction for Media Codec decoding filters.

src/CMakeLists.txt

@@ -382,6 +382,7 @@ if(ENABLE_VIDEO)
 		list(APPEND VOIP_SOURCE_FILES_CXX
 			android/media-codec-decoder.cpp
 			android/media-codec-encoder.cpp
+			android/media-codec-h264-decoder.cpp
 			android/media-codec-h264-encoder.cpp
 		)
 		if(CMAKE_SYSTEM_PROCESSOR STREQUAL "armv7-a")

src/android/media-codec-decoder.h

+/*
+ Mediastreamer2 media-codec-decoder.h
+ Copyright (C) 2018 Belledonne Communications SARL
+
+ This program is free software; you can redistribute it and/or
+ modify it under the terms of the GNU General Public License
+ as published by the Free Software Foundation; either version 2
+ of the License, or (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+
+#pragma once
+
+#include <cstdint>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include <media/NdkMediaCodec.h>
+
+#include "mediastreamer2/msvideo.h"
+
+#include "nal-unpacker.h"
+
+namespace mediastreamer {
+
+class MediaCodecDecoderFilterImpl {
+public:
+	MediaCodecDecoderFilterImpl(MSFilter *f, const std::string &mimeType, NalUnpacker *unpacker);
+	~MediaCodecDecoderFilterImpl();
+
+	void preprocess();
+	void process();
+	void postprocess();
+
+	MSVideoSize getVideoSize() const;
+	float getFps() const;
+	const MSFmtDescriptor *getOutFmt() const;
+
+	void enableAvpf(bool enable);
+	void enableFreezeOnError(bool enable);
+	void resetFirstImage();
+
+protected:
+	media_status_t initMediaCodec();
+	void flush(bool with_reset);
+
+	MSVideoSize _vsize;
+	MSAverageFPS _fps;
+	bool _avpfEnabled = false;
+	bool _freezeOnError = true;
+
+	MSFilter *_f = nullptr;
+	std::string _mimeType;
+	std::unique_ptr<NalUnpacker> _unpacker;
+	AMediaCodec *_codec = nullptr;
+	unsigned int _packetNum = 0;
+	std::vector<uint8_t> bitstream;
+	MSYuvBufAllocator *_bufAllocator = nullptr;
+	bool _bufferQueued = false;
+	bool _firstImageDecoded = false;
+	bool _needKeyFrame = false;
+
+	static const unsigned int _timeoutUs = 0;
+};
+
+}

src/android/media-codec-encoder.cpp

@@ -44,11 +44,14 @@ static const MSVideoConfiguration mediaCodecH264_conf_list[] = {
 };
 
 using namespace mediastreamer;
+using namespace std;
 
 namespace mediastreamer {
 
 // Public methods
-MediaCodecEncoderFilterImpl::MediaCodecEncoderFilterImpl(MSFilter *f, NalPacker *packer): _f(f), _packer(packer) {
+MediaCodecEncoderFilterImpl::MediaCodecEncoderFilterImpl(MSFilter *f, const string &mime, NalPacker *packer):
+	_f(f), _mime(mime), _packer(packer) {
+
 	_vconfList = mediaCodecH264_conf_list;
 	_vconf = ms_video_find_best_configuration_for_size(_vconfList, MS_VIDEO_SIZE_CIF, ms_factory_get_cpu_count(f->factory));
 	ms_video_starter_init(&_starter);

src/android/media-codec-encoder.h

@@ -20,6 +20,7 @@
 #pragma once
 
 #include <memory>
+#include <string>
 
 #include <media/NdkMediaCodec.h>
 
@@ -33,7 +34,7 @@ namespace mediastreamer {
 
 class MediaCodecEncoderFilterImpl {
 public:
-	MediaCodecEncoderFilterImpl(MSFilter *f, NalPacker *packer);
+	MediaCodecEncoderFilterImpl(MSFilter *f, const std::string &mime, NalPacker *packer);
 	virtual ~MediaCodecEncoderFilterImpl();
 
 	virtual void preprocess();
@@ -64,15 +65,17 @@ protected:
 	media_status_t tryColorFormat(AMediaFormat *format, unsigned value);
 	int encConfigure();
 
-	MSFilter *_f = nullptr;
-	AMediaCodec *_codec = nullptr;
 	const MSVideoConfiguration *_vconfList;
 	MSVideoConfiguration _vconf;
+	bool _avpfEnabled = false;
+
+	MSFilter *_f = nullptr;
+	std::string _mime;
 	std::unique_ptr<NalPacker> _packer;
+	AMediaCodec *_codec = nullptr;
 	uint64_t _framenum = 0;
 	MSVideoStarter _starter;
 	MSIFrameRequestsLimiterCtx _iframeLimiter;
-	bool _avpfEnabled = false;
 	bool _firstBufferQueued = false;
 	bool _codecStarted = false;
 	bool _codecLost = false;

src/android/media-codec-h264-decoder.cpp

+/*
+ Mediastreamer2 media-codec-h264-decoder.cpp
+ Copyright (C) 2015 Belledonne Communications SARL
+
+ This program is free software; you can redistribute it and/or
+ modify it under the terms of the GNU General Public License
+ as published by the Free Software Foundation; either version 2
+ of the License, or (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+*/
+
+#include <ortp/b64.h>
+
+#include "h264utils.h"
+#include "h264-nal-unpacker.h"
+#include "media-codec-decoder.h"
+
+using namespace b64;
+
+namespace mediastreamer {
+
+class MediaCodecH264DecoderFilterImpl: public MediaCodecDecoderFilterImpl {
+public:
+	MediaCodecH264DecoderFilterImpl(MSFilter *f): MediaCodecDecoderFilterImpl(f, "video/avc", new H264NalUnpacker()) {}
+	~MediaCodecH264DecoderFilterImpl() {
+		if (_sps) freemsg(_sps);
+		if (_pps) freemsg(_pps);
+	}
+
+	void process() {
+		if (_packetNum == 0 && _sps && _pps) {
+			static_cast<H264NalUnpacker &>(*_unpacker).setOutOfBandSpsPps(_sps, _pps);
+			_sps = nullptr;
+			_pps = nullptr;
+		}
+		MediaCodecDecoderFilterImpl::process();
+	}
+
+	void addFmtp(const char *fmtp) {
+		char value[256];
+		if (fmtp_get_value(fmtp, "sprop-parameter-sets", value, sizeof(value))) {
+			char *b64_sps = value;
+			char *b64_pps = strchr(value, ',');
+
+			if (b64_pps) {
+				*b64_pps = '\0';
+				++b64_pps;
+				ms_message("Got sprop-parameter-sets : sps=%s , pps=%s", b64_sps, b64_pps);
+				_sps = allocb(sizeof(value), 0);
+				_sps->b_wptr += b64_decode(b64_sps, strlen(b64_sps), _sps->b_wptr, sizeof(value));
+				_pps = allocb(sizeof(value), 0);
+				_pps->b_wptr += b64_decode(b64_pps, strlen(b64_pps), _pps->b_wptr, sizeof(value));
+			}
+		}
+	}
+
+	static void onFilterInit(MSFilter *f) {
+		f->data = new MediaCodecH264DecoderFilterImpl(f);
+	}
+
+	static void onFilterUninit(MSFilter *f) {
+		delete static_cast<MediaCodecH264DecoderFilterImpl *>(f->data);
+	}
+
+	static void onFilterPreProcess(MSFilter *f) {
+		static_cast<MediaCodecH264DecoderFilterImpl *>(f->data)->preprocess();
+	}
+
+	static void onFilterPostProcess(MSFilter *f) {
+		static_cast<MediaCodecH264DecoderFilterImpl *>(f->data)->postprocess();
+	}
+
+	static void onFilterProcces(MSFilter *f) {
+		static_cast<MediaCodecH264DecoderFilterImpl *>(f->data)->process();
+	}
+
+	static int onAddFmtpCall(MSFilter *f, void *arg) {
+		const char *fmtp = static_cast<const char *>(arg);
+		static_cast<MediaCodecH264DecoderFilterImpl *>(f->data)->addFmtp(fmtp);
+		return 0;
+	}
+
+	static int onResetFirstImageCall(MSFilter *f, void *arg) {
+		static_cast<MediaCodecH264DecoderFilterImpl *>(f->data)->resetFirstImage();
+		return 0;
+	}
+
+	static int onGetVideoSizeCall(MSFilter *f, void *arg) {
+		MSVideoSize *vsize = static_cast<MSVideoSize *>(arg);
+		*vsize = static_cast<MediaCodecH264DecoderFilterImpl *>(f->data)->getVideoSize();
+		return 0;
+	}
+
+	static int onGetFpsCall(MSFilter *f, void *arg) {
+		float *fps = static_cast<float *>(arg);
+		*fps = static_cast<MediaCodecH264DecoderFilterImpl *>(f->data)->getFps();
+		return 0;
+	}
+
+	static int onGetOutFmtCall(MSFilter *f, void *arg) {
+		MSPinFormat *pinFormat = static_cast<MSPinFormat *>(arg);
+		pinFormat->fmt = static_cast<MediaCodecH264DecoderFilterImpl *>(f->data)->getOutFmt();
+		return 0;
+	}
+
+	static int onEnableAvpfCall(MSFilter *f, void *arg) {
+		const bool_t *enable = static_cast<bool_t *>(arg);
+		static_cast<MediaCodecH264DecoderFilterImpl *>(f->data)->enableAvpf(enable);
+		return 0;
+	}
+
+	static int onEnableFreezeOnErrorCall(MSFilter *f, void *arg) {
+		const bool_t *enable = static_cast<bool_t *>(arg);
+		static_cast<MediaCodecH264DecoderFilterImpl *>(f->data)->enableFreezeOnError(enable);
+		return 0;
+	}
+
+private:
+	void updateSps(mblk_t *sps) {
+		if (_sps) freemsg(_sps);
+		_sps = dupb(sps);
+	}
+
+	void updatePps(mblk_t *pps) {
+		if (_pps) freemsg(_pps);
+		if (pps) _pps = dupb(pps);
+		else _pps = nullptr;
+	}
+
+	bool checkSpsChange(mblk_t *sps) {
+		bool ret = false;
+		if (_sps) {
+			ret = (msgdsize(sps) != msgdsize(_sps)) || (memcmp(_sps->b_rptr, sps->b_rptr, msgdsize(sps)) != 0);
+
+			if (ret) {
+				ms_message("SPS changed ! %i,%i", (int)msgdsize(sps), (int)msgdsize(_sps));
+				updateSps(sps);
+				updatePps(nullptr);
+			}
+		} else {
+			ms_message("Receiving first SPS");
+			updateSps(sps);
+		}
+		return ret;
+	}
+
+	bool checkPpsChange(mblk_t *pps) {
+		bool ret = false;
+		if (_pps) {
+			ret = (msgdsize(pps) != msgdsize(_pps)) || (memcmp(_pps->b_rptr, pps->b_rptr, msgdsize(pps)) != 0);
+
+			if (ret) {
+				ms_message("PPS changed ! %i,%i", (int)msgdsize(pps), (int)msgdsize(_pps));
+				updatePps(pps);
+			}
+		} else {
+			ms_message("Receiving first PPS");
+			updatePps(pps);
+		}
+		return ret;
+	}
+
+	mblk_t *_sps = nullptr;
+	mblk_t *_pps = nullptr;
+};
+
+}
+
+using namespace mediastreamer;
+
+static MSFilterMethod  mediacodec_h264_dec_methods[] = {
+	{	MS_FILTER_ADD_FMTP                                 , MediaCodecH264DecoderFilterImpl::onAddFmtpCall             },
+	{	MS_VIDEO_DECODER_RESET_FIRST_IMAGE_NOTIFICATION    , MediaCodecH264DecoderFilterImpl::onResetFirstImageCall     },
+	{	MS_FILTER_GET_VIDEO_SIZE                           , MediaCodecH264DecoderFilterImpl::onGetVideoSizeCall        },
+	{	MS_FILTER_GET_FPS                                  , MediaCodecH264DecoderFilterImpl::onGetFpsCall              },
+	{	MS_FILTER_GET_OUTPUT_FMT                           , MediaCodecH264DecoderFilterImpl::onGetOutFmtCall           },
+	{ 	MS_VIDEO_DECODER_ENABLE_AVPF                       , MediaCodecH264DecoderFilterImpl::onEnableAvpfCall          },
+	{	MS_VIDEO_DECODER_FREEZE_ON_ERROR                   , MediaCodecH264DecoderFilterImpl::onEnableFreezeOnErrorCall },
+	{	0                                                  , nullptr                                                    }
+};
+
+
+extern "C" MSFilterDesc ms_mediacodec_h264_dec_desc = {
+	.id = MS_MEDIACODEC_H264_DEC_ID,
+	.name = "MSMediaCodecH264Dec",
+	.text = "A H264 decoder based on MediaCodec API.",
+	.category = MS_FILTER_DECODER,
+	.enc_fmt = "H264",
+	.ninputs = 1,
+	.noutputs = 1,
+	.init = MediaCodecH264DecoderFilterImpl::onFilterInit,
+	.preprocess = MediaCodecH264DecoderFilterImpl::onFilterPreProcess,
+	.process = MediaCodecH264DecoderFilterImpl::onFilterProcces,
+	.postprocess = MediaCodecH264DecoderFilterImpl::onFilterPostProcess,
+	.uninit = MediaCodecH264DecoderFilterImpl::onFilterUninit,
+	.methods = mediacodec_h264_dec_methods,
+	.flags = MS_FILTER_IS_PUMP
+};

src/android/media-codec-h264-encoder.cpp

@@ -26,13 +26,14 @@ namespace mediastreamer {
 class MediaCodecH264EncoderFilterImpl: public MediaCodecEncoderFilterImpl {
 public:
 
-	MediaCodecH264EncoderFilterImpl(MSFilter *f): MediaCodecEncoderFilterImpl(f, new H264NalPacker()) {}
+	MediaCodecH264EncoderFilterImpl(MSFilter *f): MediaCodecEncoderFilterImpl(f, "video/avc", new H264NalPacker()) {}
 	~MediaCodecH264EncoderFilterImpl() {
 		if (_sps) freemsg(_sps);
 		if (_pps) freemsg(_pps);
 	}
 
 	void postprocess() override {
+		MediaCodecEncoderFilterImpl::postprocess();
 		setMblk(&_sps, nullptr);
 		setMblk(&_pps, nullptr);
 	}

src/voip/h26x-utils.cpp

@@ -100,4 +100,45 @@ void byteSTreamToNalus(const uint8_t *byteStream, size_t size, MSQueue *out) {
 	if (begin) push_nalu(begin, p, out);
 }
 
+void nalusToByteStream(MSQueue *nalus, std::vector<uint8_t> &bytestream) {
+	bool start_picture = true;
+	bytestream.resize(0);
+	while (mblk_t *im = ms_queue_get(nalus)) {
+		const uint8_t *src = im->b_rptr;
+		if (src[0] == 0 && src[1] == 0 && src[2] == 0 && src[3] == 1) {
+			while (src != im->b_wptr) {
+				bytestream.push_back(*src++);
+			}
+		} else {
+			if (start_picture) {
+				bytestream.push_back(0);
+				start_picture = false;
+			}
+
+			/*prepend nal marker*/
+			bytestream.push_back(0);
+			bytestream.push_back(0);
+			bytestream.push_back(1);
+			bytestream.push_back(*src++);
+
+			while (src < (im->b_wptr - 3)) {
+				if (src[0] == 0 && src[1] == 0 && src[2] < 3) {
+					bytestream.push_back(0);
+					bytestream.push_back(0);
+					bytestream.push_back(3);
+					src += 2;
+				}
+
+				bytestream.push_back(*src++);
+			}
+
+			bytestream.push_back(*src++);
+			bytestream.push_back(*src++);
+			bytestream.push_back(*src++);
+		}
+
+		freemsg(im);
+	}
+}
+
 }

src/voip/h26x-utils.h

@@ -32,4 +32,6 @@ void naluStreamToNalus(const uint8_t *byteStream, size_t size, MSQueue *out);
 void byteStreamToNalus(const std::vector<uint8_t> &byteStream, MSQueue *out);
 void byteStreamToNalus(const uint8_t *byteStream, size_t size, MSQueue *out);
 
+void nalusToByteStream(MSQueue *nalus, std::vector<uint8_t> &bytestream);
+
 }