Commit 77659558 authored by François Grisez's avatar François Grisez
Browse files

Making an abstraction for Media Codec decoding filters.

parent 8f7d33a1
......@@ -382,6 +382,7 @@ if(ENABLE_VIDEO)
list(APPEND VOIP_SOURCE_FILES_CXX
android/media-codec-decoder.cpp
android/media-codec-encoder.cpp
android/media-codec-h264-decoder.cpp
android/media-codec-h264-encoder.cpp
)
if(CMAKE_SYSTEM_PROCESSOR STREQUAL "armv7-a")
......
This diff is collapsed.
/*
Mediastreamer2 media-codec-decoder.h
Copyright (C) 2018 Belledonne Communications SARL
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; either version 2
of the License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
#pragma once
#include <cstdint>
#include <memory>
#include <string>
#include <vector>
#include <media/NdkMediaCodec.h>
#include "mediastreamer2/msvideo.h"
#include "nal-unpacker.h"
namespace mediastreamer {
class MediaCodecDecoderFilterImpl {
public:
MediaCodecDecoderFilterImpl(MSFilter *f, const std::string &mimeType, NalUnpacker *unpacker);
~MediaCodecDecoderFilterImpl();
void preprocess();
void process();
void postprocess();
MSVideoSize getVideoSize() const;
float getFps() const;
const MSFmtDescriptor *getOutFmt() const;
void enableAvpf(bool enable);
void enableFreezeOnError(bool enable);
void resetFirstImage();
protected:
media_status_t initMediaCodec();
void flush(bool with_reset);
MSVideoSize _vsize;
MSAverageFPS _fps;
bool _avpfEnabled = false;
bool _freezeOnError = true;
MSFilter *_f = nullptr;
std::string _mimeType;
std::unique_ptr<NalUnpacker> _unpacker;
AMediaCodec *_codec = nullptr;
unsigned int _packetNum = 0;
std::vector<uint8_t> bitstream;
MSYuvBufAllocator *_bufAllocator = nullptr;
bool _bufferQueued = false;
bool _firstImageDecoded = false;
bool _needKeyFrame = false;
static const unsigned int _timeoutUs = 0;
};
}
......@@ -44,11 +44,14 @@ static const MSVideoConfiguration mediaCodecH264_conf_list[] = {
};
using namespace mediastreamer;
using namespace std;
namespace mediastreamer {
// Public methods
MediaCodecEncoderFilterImpl::MediaCodecEncoderFilterImpl(MSFilter *f, NalPacker *packer): _f(f), _packer(packer) {
MediaCodecEncoderFilterImpl::MediaCodecEncoderFilterImpl(MSFilter *f, const string &mime, NalPacker *packer):
_f(f), _mime(mime), _packer(packer) {
_vconfList = mediaCodecH264_conf_list;
_vconf = ms_video_find_best_configuration_for_size(_vconfList, MS_VIDEO_SIZE_CIF, ms_factory_get_cpu_count(f->factory));
ms_video_starter_init(&_starter);
......
......@@ -20,6 +20,7 @@
#pragma once
#include <memory>
#include <string>
#include <media/NdkMediaCodec.h>
......@@ -33,7 +34,7 @@ namespace mediastreamer {
class MediaCodecEncoderFilterImpl {
public:
MediaCodecEncoderFilterImpl(MSFilter *f, NalPacker *packer);
MediaCodecEncoderFilterImpl(MSFilter *f, const std::string &mime, NalPacker *packer);
virtual ~MediaCodecEncoderFilterImpl();
virtual void preprocess();
......@@ -64,15 +65,17 @@ protected:
media_status_t tryColorFormat(AMediaFormat *format, unsigned value);
int encConfigure();
MSFilter *_f = nullptr;
AMediaCodec *_codec = nullptr;
const MSVideoConfiguration *_vconfList;
MSVideoConfiguration _vconf;
bool _avpfEnabled = false;
MSFilter *_f = nullptr;
std::string _mime;
std::unique_ptr<NalPacker> _packer;
AMediaCodec *_codec = nullptr;
uint64_t _framenum = 0;
MSVideoStarter _starter;
MSIFrameRequestsLimiterCtx _iframeLimiter;
bool _avpfEnabled = false;
bool _firstBufferQueued = false;
bool _codecStarted = false;
bool _codecLost = false;
......
/*
Mediastreamer2 media-codec-h264-decoder.cpp
Copyright (C) 2015 Belledonne Communications SARL
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; either version 2
of the License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
#include <ortp/b64.h>
#include "h264utils.h"
#include "h264-nal-unpacker.h"
#include "media-codec-decoder.h"
using namespace b64;
namespace mediastreamer {
class MediaCodecH264DecoderFilterImpl: public MediaCodecDecoderFilterImpl {
public:
MediaCodecH264DecoderFilterImpl(MSFilter *f): MediaCodecDecoderFilterImpl(f, "video/avc", new H264NalUnpacker()) {}
~MediaCodecH264DecoderFilterImpl() {
if (_sps) freemsg(_sps);
if (_pps) freemsg(_pps);
}
void process() {
if (_packetNum == 0 && _sps && _pps) {
static_cast<H264NalUnpacker &>(*_unpacker).setOutOfBandSpsPps(_sps, _pps);
_sps = nullptr;
_pps = nullptr;
}
MediaCodecDecoderFilterImpl::process();
}
void addFmtp(const char *fmtp) {
char value[256];
if (fmtp_get_value(fmtp, "sprop-parameter-sets", value, sizeof(value))) {
char *b64_sps = value;
char *b64_pps = strchr(value, ',');
if (b64_pps) {
*b64_pps = '\0';
++b64_pps;
ms_message("Got sprop-parameter-sets : sps=%s , pps=%s", b64_sps, b64_pps);
_sps = allocb(sizeof(value), 0);
_sps->b_wptr += b64_decode(b64_sps, strlen(b64_sps), _sps->b_wptr, sizeof(value));
_pps = allocb(sizeof(value), 0);
_pps->b_wptr += b64_decode(b64_pps, strlen(b64_pps), _pps->b_wptr, sizeof(value));
}
}
}
static void onFilterInit(MSFilter *f) {
f->data = new MediaCodecH264DecoderFilterImpl(f);
}
static void onFilterUninit(MSFilter *f) {
delete static_cast<MediaCodecH264DecoderFilterImpl *>(f->data);
}
static void onFilterPreProcess(MSFilter *f) {
static_cast<MediaCodecH264DecoderFilterImpl *>(f->data)->preprocess();
}
static void onFilterPostProcess(MSFilter *f) {
static_cast<MediaCodecH264DecoderFilterImpl *>(f->data)->postprocess();
}
static void onFilterProcces(MSFilter *f) {
static_cast<MediaCodecH264DecoderFilterImpl *>(f->data)->process();
}
static int onAddFmtpCall(MSFilter *f, void *arg) {
const char *fmtp = static_cast<const char *>(arg);
static_cast<MediaCodecH264DecoderFilterImpl *>(f->data)->addFmtp(fmtp);
return 0;
}
static int onResetFirstImageCall(MSFilter *f, void *arg) {
static_cast<MediaCodecH264DecoderFilterImpl *>(f->data)->resetFirstImage();
return 0;
}
static int onGetVideoSizeCall(MSFilter *f, void *arg) {
MSVideoSize *vsize = static_cast<MSVideoSize *>(arg);
*vsize = static_cast<MediaCodecH264DecoderFilterImpl *>(f->data)->getVideoSize();
return 0;
}
static int onGetFpsCall(MSFilter *f, void *arg) {
float *fps = static_cast<float *>(arg);
*fps = static_cast<MediaCodecH264DecoderFilterImpl *>(f->data)->getFps();
return 0;
}
static int onGetOutFmtCall(MSFilter *f, void *arg) {
MSPinFormat *pinFormat = static_cast<MSPinFormat *>(arg);
pinFormat->fmt = static_cast<MediaCodecH264DecoderFilterImpl *>(f->data)->getOutFmt();
return 0;
}
static int onEnableAvpfCall(MSFilter *f, void *arg) {
const bool_t *enable = static_cast<bool_t *>(arg);
static_cast<MediaCodecH264DecoderFilterImpl *>(f->data)->enableAvpf(enable);
return 0;
}
static int onEnableFreezeOnErrorCall(MSFilter *f, void *arg) {
const bool_t *enable = static_cast<bool_t *>(arg);
static_cast<MediaCodecH264DecoderFilterImpl *>(f->data)->enableFreezeOnError(enable);
return 0;
}
private:
void updateSps(mblk_t *sps) {
if (_sps) freemsg(_sps);
_sps = dupb(sps);
}
void updatePps(mblk_t *pps) {
if (_pps) freemsg(_pps);
if (pps) _pps = dupb(pps);
else _pps = nullptr;
}
bool checkSpsChange(mblk_t *sps) {
bool ret = false;
if (_sps) {
ret = (msgdsize(sps) != msgdsize(_sps)) || (memcmp(_sps->b_rptr, sps->b_rptr, msgdsize(sps)) != 0);
if (ret) {
ms_message("SPS changed ! %i,%i", (int)msgdsize(sps), (int)msgdsize(_sps));
updateSps(sps);
updatePps(nullptr);
}
} else {
ms_message("Receiving first SPS");
updateSps(sps);
}
return ret;
}
bool checkPpsChange(mblk_t *pps) {
bool ret = false;
if (_pps) {
ret = (msgdsize(pps) != msgdsize(_pps)) || (memcmp(_pps->b_rptr, pps->b_rptr, msgdsize(pps)) != 0);
if (ret) {
ms_message("PPS changed ! %i,%i", (int)msgdsize(pps), (int)msgdsize(_pps));
updatePps(pps);
}
} else {
ms_message("Receiving first PPS");
updatePps(pps);
}
return ret;
}
mblk_t *_sps = nullptr;
mblk_t *_pps = nullptr;
};
}
using namespace mediastreamer;
static MSFilterMethod mediacodec_h264_dec_methods[] = {
{ MS_FILTER_ADD_FMTP , MediaCodecH264DecoderFilterImpl::onAddFmtpCall },
{ MS_VIDEO_DECODER_RESET_FIRST_IMAGE_NOTIFICATION , MediaCodecH264DecoderFilterImpl::onResetFirstImageCall },
{ MS_FILTER_GET_VIDEO_SIZE , MediaCodecH264DecoderFilterImpl::onGetVideoSizeCall },
{ MS_FILTER_GET_FPS , MediaCodecH264DecoderFilterImpl::onGetFpsCall },
{ MS_FILTER_GET_OUTPUT_FMT , MediaCodecH264DecoderFilterImpl::onGetOutFmtCall },
{ MS_VIDEO_DECODER_ENABLE_AVPF , MediaCodecH264DecoderFilterImpl::onEnableAvpfCall },
{ MS_VIDEO_DECODER_FREEZE_ON_ERROR , MediaCodecH264DecoderFilterImpl::onEnableFreezeOnErrorCall },
{ 0 , nullptr }
};
extern "C" MSFilterDesc ms_mediacodec_h264_dec_desc = {
.id = MS_MEDIACODEC_H264_DEC_ID,
.name = "MSMediaCodecH264Dec",
.text = "A H264 decoder based on MediaCodec API.",
.category = MS_FILTER_DECODER,
.enc_fmt = "H264",
.ninputs = 1,
.noutputs = 1,
.init = MediaCodecH264DecoderFilterImpl::onFilterInit,
.preprocess = MediaCodecH264DecoderFilterImpl::onFilterPreProcess,
.process = MediaCodecH264DecoderFilterImpl::onFilterProcces,
.postprocess = MediaCodecH264DecoderFilterImpl::onFilterPostProcess,
.uninit = MediaCodecH264DecoderFilterImpl::onFilterUninit,
.methods = mediacodec_h264_dec_methods,
.flags = MS_FILTER_IS_PUMP
};
......@@ -26,13 +26,14 @@ namespace mediastreamer {
class MediaCodecH264EncoderFilterImpl: public MediaCodecEncoderFilterImpl {
public:
MediaCodecH264EncoderFilterImpl(MSFilter *f): MediaCodecEncoderFilterImpl(f, new H264NalPacker()) {}
MediaCodecH264EncoderFilterImpl(MSFilter *f): MediaCodecEncoderFilterImpl(f, "video/avc", new H264NalPacker()) {}
~MediaCodecH264EncoderFilterImpl() {
if (_sps) freemsg(_sps);
if (_pps) freemsg(_pps);
}
void postprocess() override {
MediaCodecEncoderFilterImpl::postprocess();
setMblk(&_sps, nullptr);
setMblk(&_pps, nullptr);
}
......
......@@ -100,4 +100,45 @@ void byteSTreamToNalus(const uint8_t *byteStream, size_t size, MSQueue *out) {
if (begin) push_nalu(begin, p, out);
}
void nalusToByteStream(MSQueue *nalus, std::vector<uint8_t> &bytestream) {
bool start_picture = true;
bytestream.resize(0);
while (mblk_t *im = ms_queue_get(nalus)) {
const uint8_t *src = im->b_rptr;
if (src[0] == 0 && src[1] == 0 && src[2] == 0 && src[3] == 1) {
while (src != im->b_wptr) {
bytestream.push_back(*src++);
}
} else {
if (start_picture) {
bytestream.push_back(0);
start_picture = false;
}
/*prepend nal marker*/
bytestream.push_back(0);
bytestream.push_back(0);
bytestream.push_back(1);
bytestream.push_back(*src++);
while (src < (im->b_wptr - 3)) {
if (src[0] == 0 && src[1] == 0 && src[2] < 3) {
bytestream.push_back(0);
bytestream.push_back(0);
bytestream.push_back(3);
src += 2;
}
bytestream.push_back(*src++);
}
bytestream.push_back(*src++);
bytestream.push_back(*src++);
bytestream.push_back(*src++);
}
freemsg(im);
}
}
}
......@@ -32,4 +32,6 @@ void naluStreamToNalus(const uint8_t *byteStream, size_t size, MSQueue *out);
void byteStreamToNalus(const std::vector<uint8_t> &byteStream, MSQueue *out);
void byteStreamToNalus(const uint8_t *byteStream, size_t size, MSQueue *out);
void nalusToByteStream(MSQueue *nalus, std::vector<uint8_t> &bytestream);
}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment