Makes an abstract class for MediaCodec based filters.

src/CMakeLists.txt

@@ -306,9 +306,10 @@ if(ENABLE_VIDEO)
 		utils/h264utils.cpp
 		utils/h265-utils.cpp
 		voip/h264-nal-packer.cpp
-		voip/h265-nal-packer.cpp
 		voip/h264-nal-unpacker.cpp
+		voip/h265-nal-packer.cpp
 		voip/h265-nal-unpacker.cpp
+		voip/h26x-utils.cpp
 		voip/nal-packer.cpp
 		voip/nal-unpacker.cpp
 		voip/rfc3984.cpp
@@ -381,6 +382,7 @@ if(ENABLE_VIDEO)
 		list(APPEND VOIP_SOURCE_FILES_CXX
 			android/mediacodech264dec.cpp
 			android/media-codec-encoder.cpp
+			android/media-codec-h264-encoder.cpp
 		)
 		if(CMAKE_SYSTEM_PROCESSOR STREQUAL "armv7-a")
 			list(APPEND VOIP_SOURCE_FILES_ASM

src/android/media-codec-encoder.h

+/*
+ Mediastreamer2 media-codec-encoder.h
+ Copyright (C) 2018 Belledonne Communications SARL
+
+ This program is free software; you can redistribute it and/or
+ modify it under the terms of the GNU General Public License
+ as published by the Free Software Foundation; either version 2
+ of the License, or (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+*/
+
+#pragma once
+
+#include <memory>
+
+#include <media/NdkMediaCodec.h>
+
+#include "mediastreamer2/mscodecutils.h"
+#include "mediastreamer2/msfilter.h"
+#include "mediastreamer2/msvideo.h"
+
+#include "nal-packer.h"
+
+namespace mediastreamer {
+
+class MediaCodecEncoderFilterImpl {
+public:
+	MediaCodecEncoderFilterImpl(MSFilter *f, NalPacker *packer);
+	virtual ~MediaCodecEncoderFilterImpl();
+
+	virtual void preprocess();
+	virtual void process();
+	virtual void postprocess();
+
+	int getBitrate() const;
+	void setBitrate(int br);
+
+	const MSVideoConfiguration *getVideoConfiguratons() const;
+	void setVideoConfigurations(const MSVideoConfiguration *vconfs);
+	int setVideoConfiguration(const MSVideoConfiguration *vconf);
+
+	float getFps() const;
+	void setFps(float  fps);
+
+	MSVideoSize getVideoSize() const;
+	void setVideoSize(const MSVideoSize &vsize);
+
+	void enableAvpf(bool enable);
+	void notifyPli();
+	void notifyFir();
+
+protected:
+	virtual void onFrameEncodedHook(MSQueue *frame) {};
+
+	media_status_t allocEncoder();
+	media_status_t tryColorFormat(AMediaFormat *format, unsigned value);
+	int encConfigure();
+
+	MSFilter *_f = nullptr;
+	AMediaCodec *_codec = nullptr;
+	const MSVideoConfiguration *_vconfList;
+	MSVideoConfiguration _vconf;
+	std::unique_ptr<NalPacker> _packer;
+	uint64_t _framenum = 0;
+	MSVideoStarter _starter;
+	MSIFrameRequestsLimiterCtx _iframeLimiter;
+	bool _avpfEnabled = false;
+	bool _firstBufferQueued = false;
+	bool _codecStarted = false;
+	bool _codecLost = false;
+
+	static const int _timeoutUs = 0;
+};
+
+}

src/android/media-codec-h264-encoder.cpp

+/*
+ Mediastreamer2 media-codec-h264-encoder.cpp
+ Copyright (C) 2018 Belledonne Communications SARL
+
+ This program is free software; you can redistribute it and/or
+ modify it under the terms of the GNU General Public License
+ as published by the Free Software Foundation; either version 2
+ of the License, or (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+*/
+
+#include "h264-nal-packer.h"
+#include "h264utils.h"
+#include "media-codec-encoder.h"
+
+namespace mediastreamer {
+
+class MediaCodecH264EncoderFilterImpl: public MediaCodecEncoderFilterImpl {
+public:
+
+	MediaCodecH264EncoderFilterImpl(MSFilter *f): MediaCodecEncoderFilterImpl(f, new H264NalPacker()) {}
+	~MediaCodecH264EncoderFilterImpl() {
+		if (_sps) freemsg(_sps);
+		if (_pps) freemsg(_pps);
+	}
+
+	void postprocess() override {
+		setMblk(&_sps, nullptr);
+		setMblk(&_pps, nullptr);
+	}
+
+	static void onFilterInit(MSFilter *f) {
+		f->data = new MediaCodecH264EncoderFilterImpl(f);
+	}
+
+	static void onFilterPreprocess(MSFilter *f) {
+		static_cast<MediaCodecH264EncoderFilterImpl *>(f->data)->preprocess();
+	}
+
+	static void onFilterPostprocess(MSFilter *f) {
+		static_cast<MediaCodecH264EncoderFilterImpl *>(f->data)->postprocess();
+	}
+
+	static void onFilterUninit(MSFilter *f) {
+		delete static_cast<MediaCodecH264EncoderFilterImpl *>(f->data);
+	}
+
+	static void onFilterProcess(MSFilter *f) {
+		static_cast<MediaCodecH264EncoderFilterImpl *>(f->data)->process();
+	}
+
+	static int onGetBitrateCall(MSFilter *f, void *arg) {
+		int *bitrate = static_cast<int *>(arg);
+		*bitrate = static_cast<MediaCodecH264EncoderFilterImpl *>(f->data)->getBitrate();
+		return 0;
+	}
+
+	static int onSetConfigurationCall(MSFilter *f, void *arg) {
+		const MSVideoConfiguration *vconf = static_cast<MSVideoConfiguration *>(arg);
+		static_cast<MediaCodecH264EncoderFilterImpl *>(f->data)->setVideoConfiguration(vconf);
+		return 0;
+	}
+
+	static int onSetBitrateCall(MSFilter *f, void *arg) {
+		int br = *static_cast<int *>(arg);
+		static_cast<MediaCodecH264EncoderFilterImpl *>(f->data)->setBitrate(br);
+		return 0;
+	}
+
+	static int onSetFpsCall(MSFilter *f, void *arg) {
+		float fps = *static_cast<float *>(arg);
+		static_cast<MediaCodecH264EncoderFilterImpl *>(f->data)->setFps(fps);
+		return 0;
+	}
+
+	static int onGetFpsCall(MSFilter *f, void *arg) {
+		float *fps = static_cast<float *>(arg);
+		*fps = static_cast<MediaCodecH264EncoderFilterImpl *>(f->data)->getFps();
+		return 0;
+	}
+
+	static int onGetVideoSizeCall(MSFilter *f, void *arg) {
+		MSVideoSize *vsize = static_cast<MSVideoSize *>(arg);
+		*vsize = static_cast<MediaCodecH264EncoderFilterImpl *>(f->data)->getVideoSize();
+		return 0;
+	}
+
+	static int onEnableAvpfCall(MSFilter *f, void *data) {
+		bool_t enable = *static_cast<bool_t *>(data);
+		static_cast<MediaCodecH264EncoderFilterImpl *>(f->data)->enableAvpf(enable);
+		return 0;
+	}
+
+	static int onSetVideoSizeCall(MSFilter *f, void *arg) {
+		const MSVideoSize *vsize = static_cast<MSVideoSize *>(arg);
+		static_cast<MediaCodecH264EncoderFilterImpl *>(f->data)->setVideoSize(*vsize);
+		return 0;
+	}
+
+	static int onNotifyPliCall(MSFilter *f, void *data) {
+		static_cast<MediaCodecH264EncoderFilterImpl *>(f->data)->notifyPli();
+		return 0;
+	}
+
+	static int onNotifyFirCall(MSFilter *f, void *data) {
+		static_cast<MediaCodecH264EncoderFilterImpl *>(f->data)->notifyFir();
+		return 0;
+	}
+
+	static int onGetVideoConfigurationsCall(MSFilter *f, void *data) {
+		const MSVideoConfiguration **vconfs = static_cast<const MSVideoConfiguration **>(data);
+		*vconfs = static_cast<MediaCodecH264EncoderFilterImpl *>(f->data)->getVideoConfiguratons();
+		return 0;
+	}
+
+	static int onSetVideoConfigurationsCall(MSFilter *f, void *data) {
+		const MSVideoConfiguration *vconfs = static_cast<const MSVideoConfiguration *>(data);
+		static_cast<MediaCodecH264EncoderFilterImpl *>(f->data)->setVideoConfigurations(vconfs);
+		return 0;
+	}
+
+private:
+	void onFrameEncodedHook(MSQueue *frame) override {
+		bool have_seen_sps_pps = false;
+		mblk_t *m = ms_queue_peek_first(frame);
+		switch (ms_h264_nalu_get_type(m)) {
+			case MSH264NaluTypeIDR:
+				if (!have_seen_sps_pps) {
+					ms_message("MSMediaCodecH264Enc: seeing IDR without prior SPS/PPS, so manually adding them.");
+
+					if (_sps && _pps) {
+						ms_queue_insert(frame, m, copyb(_sps));
+						ms_queue_insert(frame, m, copyb(_pps));
+					} else {
+						ms_error("MSMediaCodecH264Enc: SPS or PPS are not known !");
+					}
+				}
+				break;
+
+			case MSH264NaluTypeSPS:
+				ms_message("MSMediaCodecH264Enc: seeing SPS");
+				have_seen_sps_pps = true;
+				setMblk(&_sps, m);
+				m = ms_queue_next(&nalus, m);
+
+				if (!ms_queue_end(frame, m) && ms_h264_nalu_get_type(m) == MSH264NaluTypePPS) {
+					ms_message("MSMediaCodecH264Enc: seeing PPS");
+					setMblk(&_pps, m);
+				}
+				break;
+
+			case MSH264NaluTypePPS:
+				ms_warning("MSMediaCodecH264Enc: unexpecting starting PPS");
+				break;
+			default:
+				break;
+		}
+	}
+
+	static void setMblk(mblk_t **packet, mblk_t *newone) {
+		if (newone) newone = copyb(newone);
+		if (*packet) freemsg(*packet);
+		*packet = newone;
+	}
+
+	mblk_t *_sps = nullptr, *_pps = nullptr; /*lastly generated SPS, PPS, in case we need to repeat them*/
+};
+
+} // mamespace mediastreamer
+
+using namespace mediastreamer;
+
+static MSFilterMethod  mediacodec_h264_enc_methods[] = {
+	{ MS_FILTER_SET_FPS                       , MediaCodecH264EncoderFilterImpl::onSetFpsCall                 },
+	{ MS_FILTER_SET_BITRATE                   , MediaCodecH264EncoderFilterImpl::onSetBitrateCall             },
+	{ MS_FILTER_GET_BITRATE                   , MediaCodecH264EncoderFilterImpl::onGetBitrateCall             },
+	{ MS_FILTER_GET_FPS                       , MediaCodecH264EncoderFilterImpl::onGetFpsCall                 },
+	{ MS_FILTER_GET_VIDEO_SIZE                , MediaCodecH264EncoderFilterImpl::onGetVideoSizeCall           },
+	{ MS_VIDEO_ENCODER_NOTIFY_PLI             , MediaCodecH264EncoderFilterImpl::onNotifyPliCall              },
+	{ MS_VIDEO_ENCODER_NOTIFY_FIR             , MediaCodecH264EncoderFilterImpl::onNotifyFirCall              },
+	{ MS_FILTER_SET_VIDEO_SIZE                , MediaCodecH264EncoderFilterImpl::onSetVideoSizeCall           },
+	{ MS_VIDEO_ENCODER_ENABLE_AVPF            , MediaCodecH264EncoderFilterImpl::onEnableAvpfCall             },
+	{ MS_VIDEO_ENCODER_GET_CONFIGURATION_LIST , MediaCodecH264EncoderFilterImpl::onGetVideoConfigurationsCall },
+	{ MS_VIDEO_ENCODER_SET_CONFIGURATION_LIST , MediaCodecH264EncoderFilterImpl::onSetVideoConfigurationsCall },
+	{ MS_VIDEO_ENCODER_SET_CONFIGURATION      , MediaCodecH264EncoderFilterImpl::onSetConfigurationCall       },
+	{ 0                                       , nullptr                                                       }
+};
+
+
+extern "C" MSFilterDesc ms_mediacodec_h264_enc_desc = {
+	.id = MS_MEDIACODEC_H264_ENC_ID,
+	.name = "MSMediaCodecH264Enc",
+	.text = "A H264 encoder based on MediaCodec API.",
+	.category = MS_FILTER_ENCODER,
+	.enc_fmt = "H264",
+	.ninputs = 1,
+	.noutputs = 1,
+	.init = MediaCodecH264EncoderFilterImpl::onFilterInit,
+	.preprocess = MediaCodecH264EncoderFilterImpl::onFilterPreprocess,
+	.process = MediaCodecH264EncoderFilterImpl::onFilterProcess,
+	.postprocess = MediaCodecH264EncoderFilterImpl::onFilterPostprocess,
+	.uninit = MediaCodecH264EncoderFilterImpl::onFilterUninit,
+	.methods = mediacodec_h264_enc_methods,
+	.flags = MS_FILTER_IS_PUMP
+};

src/utils/h264utils.cpp

@@ -26,60 +26,14 @@
 #include "mediastreamer2/rfc3984.h"
 
 #include "h264utils.h"
+#include "h26x-utils.h"
 
-extern "C" {
-
-static void  push_nalu(const uint8_t *begin, const uint8_t *end, MSQueue *nalus) {
-	unsigned ecount = 0;
-	const uint8_t *src = begin;
-	size_t nalu_len = (end - begin);
-	uint8_t nalu_byte  = *src++;
-
-	mblk_t *m = allocb(nalu_len, 0);
-
-	// Removal of the 3 in a 003x sequence
-	// This emulation prevention byte is normally part of a NAL unit.
-	/* H.264 standard sys in par 7.4.1 page 58
-	 emulation_prevention_three_byte is a byte equal to 0x03.
-	 When an emulation_prevention_three_byte is present in a NAL unit, it shall be discarded by the decoding process.
-	 Within the NAL unit, the following three-byte sequence shall not occur at any byte-aligned position: 0x000000, 0x000001, 0x00002
-	 */
-	*m->b_wptr++ = nalu_byte;
-	while (src < end - 3) {
-		if (src[0] == 0 && src[1] == 0 && src[2] == 3) {
-			*m->b_wptr++ = 0;
-			*m->b_wptr++ = 0;
-			// drop the emulation_prevention_three_byte
-			src += 3;
-			++ecount;
-			continue;
-		}
-		*m->b_wptr++ = *src++;
-	}
-	*m->b_wptr++ = *src++;
-	*m->b_wptr++ = *src++;
-	*m->b_wptr++ = *src++;
+using namespace mediastreamer;
 
-	ms_queue_put(nalus, m);
-}
+extern "C" {
 
 void ms_h264_bitstream_to_nalus(const uint8_t *bitstream, size_t size, MSQueue *nalus) {
-	size_t i;
-	const uint8_t *p, *begin = NULL;
-	int zeroes = 0;
-
-	for (i = 0, p = bitstream; i < size; ++i) {
-		if (*p == 0) {
-			++zeroes;
-		} else if (zeroes >= 2 && *p == 1) {
-			if (begin) {
-				push_nalu(begin, p - zeroes, nalus);
-			}
-			begin = p + 1;
-		} else zeroes = 0;
-		++p;
-	}
-	if (begin) push_nalu(begin, p, nalus);
+	byteStreamToNalus(bitstream, size, nalus);
 }
 
 uint8_t ms_h264_nalu_get_nri(const mblk_t *nalu) {

src/utils/h264utils.h

@@ -83,14 +83,17 @@ unsigned int ms_h264_pps_get_id(const mblk_t *pps);
 void ms_h264_bitstream_to_nalus(const uint8_t *bitstream, size_t size, MSQueue *nalus);
 
 /**
- * Slices a frame into several nal units.
+ * @brief Slices a frame into several nal units.
  *
  * Same as ms_h264_bitstream_to_nalus() except that the stream must be in size-prefixed
  * format i.e. each nalu in the stream must be prefixed by its size encoded on 4 bytes big-endian.
+ *
  * @param frame Buffer containing the stream to slice
  * @param size Size of the buffer
  * @param nalus The queue where produced nal units will be pushed into
  * @param idr_count If not NULL, use the pointer to store the number of IDR nalus found in the stream.
+ *
+ * @deprecated Use mediastreamer::naluStreamToNalus() instead. Deprecated since 2018-05-21.
  */
 void ms_h264_stream_to_nalus(const uint8_t *frame, size_t size, MSQueue *nalus, int *idr_count);
 

src/voip/h26x-utils.cpp

+/*
+ Mediastreamer2 h26x-utils.h
+ Copyright (C) 2018 Belledonne Communications SARL
+
+ This program is free software; you can redistribute it and/or
+ modify it under the terms of the GNU General Public License
+ as published by the Free Software Foundation; either version 2
+ of the License, or (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+*/
+
+#include "h26x-utils.h"
+
+using namespace std;
+
+namespace mediastreamer {
+
+void naluStreamToNalus(const std::vector<uint8_t> &byteStream, MSQueue *out) {
+	naluStreamToNalus(byteStream.data(), byteStream.size(), out);
+}
+
+void naluStreamToNalus(const uint8_t *bytestream, size_t size, MSQueue *out) {
+	const uint8_t *ptr = bytestream;
+	while (ptr < bytestream + size) {
+		uint32_t nalu_size;
+		memcpy(&nalu_size, ptr, 4);
+		nalu_size = ntohl(nalu_size);
+
+		mblk_t *nalu = allocb(nalu_size, 0);
+		memcpy(nalu->b_wptr, ptr + 4, nalu_size);
+		ptr += nalu_size + 4;
+		nalu->b_wptr += nalu_size;
+
+		ms_queue_put(out, nalu);
+	}
+}
+
+void byteStreamToNalus(const std::vector<uint8_t> &byteStream, MSQueue *out) {
+	byteStreamToNalus(byteStream.data(), byteStream.size(), out);
+}
+
+static void  push_nalu(const uint8_t *begin, const uint8_t *end, MSQueue *nalus) {
+	unsigned ecount = 0;
+	const uint8_t *src = begin;
+	size_t nalu_len = (end - begin);
+	uint8_t nalu_byte  = *src++;
+
+	mblk_t *m = allocb(nalu_len, 0);
+
+	// Removal of the 3 in a 003x sequence
+	// This emulation prevention byte is normally part of a NAL unit.
+	/* H.264 standard sys in par 7.4.1 page 58
+	 * emulation_prevention_three_byte is a byte equal to 0x03.
+	 * When an emulation_prevention_three_byte is present in a NAL unit, it shall be discarded by the decoding process.
+	 * Within the NAL unit, the following three-byte sequence shall not occur at any byte-aligned position: 0x000000, 0x000001, 0x00002
+	 */
+	*m->b_wptr++ = nalu_byte;
+	while (src < end - 3) {
+		if (src[0] == 0 && src[1] == 0 && src[2] == 3) {
+			*m->b_wptr++ = 0;
+			*m->b_wptr++ = 0;
+			// drop the emulation_prevention_three_byte
+			src += 3;
+			++ecount;
+			continue;
+		}
+		*m->b_wptr++ = *src++;
+	}
+	*m->b_wptr++ = *src++;
+	*m->b_wptr++ = *src++;
+	*m->b_wptr++ = *src++;
+
+	ms_queue_put(nalus, m);
+}
+
+void byteSTreamToNalus(const uint8_t *byteStream, size_t size, MSQueue *out) {
+	size_t i;
+	const uint8_t *p, *begin = NULL;
+	int zeroes = 0;
+
+	for (i = 0, p = byteStream; i < size; ++i) {
+		if (*p == 0) {
+			++zeroes;
+		} else if (zeroes >= 2 && *p == 1) {
+			if (begin) {
+				push_nalu(begin, p - zeroes, out);
+			}
+			begin = p + 1;
+		} else zeroes = 0;
+		++p;
+	}
+	if (begin) push_nalu(begin, p, out);
+}
+
+}

src/voip/h26x-utils.h

+/*
+ Mediastreamer2 h26x-utils.h
+ Copyright (C) 2018 Belledonne Communications SARL
+
+ This program is free software; you can redistribute it and/or
+ modify it under the terms of the GNU General Public License
+ as published by the Free Software Foundation; either version 2
+ of the License, or (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+*/
+
+#pragma once
+
+#include <cstdint>
+#include <vector>
+
+#include "mediastreamer2/msqueue.h"
+
+namespace mediastreamer {
+
+void naluStreamToNalus(const std::vector<uint8_t> &byteStream, MSQueue *out);
+void naluStreamToNalus(const uint8_t *byteStream, size_t size, MSQueue *out);
+
+void byteStreamToNalus(const std::vector<uint8_t> &byteStream, MSQueue *out);
+void byteStreamToNalus(const uint8_t *byteStream, size_t size, MSQueue *out);
+
+}