diff --git a/CMakeLists.txt b/CMakeLists.txt index 1cc1fdae54437b602be1b9fde1789f68f3e33e5d..0e92ec47cb60f4446417ab21b1f594308858b677 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -59,9 +59,12 @@ endif() if(LINPHONE_BUILDER_GROUP_EXTERNAL_SOURCE_PATH_BUILDERS) include("${EP_ms2_CONFIG_DIR}/Mediastreamer2Config.cmake") include("${EP_ortp_CONFIG_DIR}/ORTPConfig.cmake") + set(BcToolbox_FIND_COMPONENTS tester) + include("${EP_bctoolbox_CONFIG_DIR}/BcToolboxConfig.cmake") else() find_package(Mediastreamer2 REQUIRED) find_package(ORTP REQUIRED) + find_package(BcToolbox REQUIRED) endif() find_library(LIBM NAMES m) @@ -123,6 +126,8 @@ if(FIXED_POINT_PROCESSOR) add_definitions(-D__ARMEL__) endif() +set(AEC_CXX11_FLAGS "-std=c++11") + set(WEBRTC_SIGNAL_PROCESSING_DIR ${WEBRTC_SRC_DIR}/common_audio/signal_processing) set(WEBRTC_SIGNAL_PROCESSING_SRCS ${WEBRTC_SIGNAL_PROCESSING_DIR}/spl_sqrt.c @@ -169,14 +174,26 @@ if(ENABLE_AEC) set(AEC_SRC_DIR "${WEBRTC_SRC_DIR}/modules/audio_processing/aec") set(AEC_SRCS aec.c + aec_splitting_filter.cc ${AEC_SRC_DIR}/aec_core.c ${AEC_SRC_DIR}/aec_rdft.c ${AEC_SRC_DIR}/aec_resampler.c ${AEC_SRC_DIR}/echo_cancellation.c + ${WEBRTC_SRC_DIR}/base/checks.cc + ${WEBRTC_SRC_DIR}/common_audio/audio_util.cc + ${WEBRTC_SRC_DIR}/common_audio/sparse_fir_filter.cc ${WEBRTC_SRC_DIR}/system_wrappers/source/cpu_features.cc + ${WEBRTC_SRC_DIR}/modules/audio_processing/three_band_filter_bank.cc ${WEBRTC_SRC_DIR}/modules/audio_processing/utility/delay_estimator.c ${WEBRTC_SRC_DIR}/modules/audio_processing/utility/delay_estimator_wrapper.c ) + set(AEC_CXX11_SRCS + aec_splitting_filter.cc + ${WEBRTC_SRC_DIR}/common_audio/audio_util.cc + ${WEBRTC_SRC_DIR}/common_audio/sparse_fir_filter.cc + ${WEBRTC_SRC_DIR}/modules/audio_processing/three_band_filter_bank.cc + ) + bc_apply_compile_flags(AEC_CXX11_SRCS AEC_CXX11_FLAGS) if(SYSTEM_PROCESSOR MATCHES "^arm.*" OR SYSTEM_PROCESSOR MATCHES "^aarch64.*") list(APPEND AEC_SRCS ${AEC_SRC_DIR}/aec_core_neon.c @@ -190,6 +207,8 @@ if(ENABLE_AEC) endif() include_directories( ${AEC_SRC_DIR}/include + ${WEBRTC_SRC_DIR}/common_audio/include + ${WEBRTC_SRC_DIR}/modules/audio_processing ) endif() if(ENABLE_AECM) @@ -370,6 +389,7 @@ endif() if(ENABLE_STATIC) add_library(mswebrtc-static STATIC ${SOURCE_FILES}) set_target_properties(mswebrtc-static PROPERTIES OUTPUT_NAME mswebrtc) + set_target_properties(mswebrtc-static PROPERTIES LINKER_LANGUAGE CXX) target_link_libraries(mswebrtc-static ${LIBS}) install(TARGETS mswebrtc-static ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}/mediastreamer/plugins @@ -383,6 +403,7 @@ if(ENABLE_SHARED) add_library(mswebrtc SHARED ${SOURCE_FILES}) endif() target_link_libraries(mswebrtc ${LIBS}) + set_target_properties(mswebrtc PROPERTIES LINKER_LANGUAGE CXX) if(IOS) set(MIN_OS ${LINPHONE_IOS_DEPLOYMENT_TARGET}) set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/build/osx/") diff --git a/aec.c b/aec.c index 14fdd8b9f11045776d8caec72bb6b8855401ff45..c25ba8c2d3f0df179127bc43ef3496743bbda51e 100644 --- a/aec.c +++ b/aec.c @@ -25,7 +25,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. #include "mediastreamer2/msticker.h" #ifdef BUILD_AEC #include "echo_cancellation.h" -#include "signal_processing_library.h" +#include "aec_splitting_filter.h" #endif #ifdef BUILD_AECM #include "echo_control_mobile.h" @@ -74,12 +74,7 @@ typedef struct WebRTCAECState { bool_t using_zeroes; WebRTCAECType aec_type; #ifdef BUILD_AEC - int ref_state1[6]; - int ref_state2[6]; - int echo_state1[6]; - int echo_state2[6]; - int oecho_state1[6]; - int oecho_state2[6]; + MSWebRtcAecSplittingFilter *splitting_filter; #endif } WebRTCAECState; @@ -98,14 +93,6 @@ static void webrtc_aecgeneric_init(MSFilter *f, WebRTCAECType aec_type) { s->echostarted = FALSE; s->bypass_mode = FALSE; s->aec_type = aec_type; -#ifdef BUILD_AEC - memset(s->ref_state1, 0, sizeof(s->ref_state1)); - memset(s->ref_state2, 0, sizeof(s->ref_state2)); - memset(s->echo_state1, 0, sizeof(s->echo_state1)); - memset(s->echo_state2, 0, sizeof(s->echo_state2)); - memset(s->oecho_state1, 0, sizeof(s->oecho_state1)); - memset(s->oecho_state2, 0, sizeof(s->oecho_state2)); -#endif #ifdef EC_DUMP { @@ -225,22 +212,6 @@ static void webrtc_aec_preprocess(MSFilter *f) { s->nominal_ref_samples = delay_samples; } -#ifdef BUILD_AEC -void intbuf2floatbuf(int16_t *intbuf, float *floatbuf, int framesize) { - int i; - for (i = 0; i < framesize; i++) { - floatbuf[i] = (float)intbuf[i]; - } -} - -void floatbuf2intbuf(float *floatbuf, int16_t *intbuf, int framesize) { - int i; - for (i = 0; i < framesize; i++) { - intbuf[i] = (int16_t)floatbuf[i]; - } -} -#endif - /* inputs[0]= reference signal from far end (sent to soundcard) * inputs[1]= near speech & echo signal (read from soundcard) * outputs[0]= is a copy of inputs[0] to be sent to soundcard @@ -251,14 +222,8 @@ static void webrtc_aec_process(MSFilter *f) { int nbytes = s->framesize * sizeof(int16_t); mblk_t *refm; int16_t *ref, *echo; -#ifdef BUILD_AEC - float *fref = NULL, *fecho = NULL, *foecho = NULL; - const float * fecho_bands_array[3]; - float * foecho_bands_array[3]; - int16_t *bandsref, *bandsecho, *bandsoecho; int nbands = 1; int bandsize = s->framesize; -#endif if (s->bypass_mode) { while ((refm = ms_queue_get(f->inputs[0])) != NULL) { @@ -289,19 +254,12 @@ static void webrtc_aec_process(MSFilter *f) { echo = (int16_t *) alloca(nbytes); #ifdef BUILD_AEC if (s->aec_type == WebRTCAECTypeNormal) { - int nfbytes = s->framesize * sizeof(float); - fref = (float *)alloca(nfbytes); - fecho = (float *)alloca(nfbytes); - foecho = (float *)alloca(nfbytes); if (s->samplerate > 16000) { nbands = s->samplerate / 16000; bandsize = 160; } - if (nbands > 1) { - int bandsbufsize = nbands * bandsize * sizeof(int16_t); - bandsref = (int16_t *) alloca(bandsbufsize); - bandsecho = (int16_t *) alloca(bandsbufsize); - bandsoecho = (int16_t *) alloca(bandsbufsize); + if (!s->splitting_filter) { + s->splitting_filter = mswebrtc_aec_splitting_filter_create(nbands, bandsize); } } #endif @@ -351,31 +309,18 @@ static void webrtc_aec_process(MSFilter *f) { #endif #ifdef BUILD_AEC if (s->aec_type == WebRTCAECTypeNormal) { - if (nbands == 2) { - WebRtcSpl_AnalysisQMF(ref, s->framesize, bandsref, bandsref + bandsize, s->ref_state1, s->ref_state2); - intbuf2floatbuf(bandsref, fref, s->framesize); - WebRtcSpl_AnalysisQMF(echo, s->framesize, bandsecho, bandsecho + bandsize, s->echo_state1, s->echo_state2); - intbuf2floatbuf(bandsecho, fecho, s->framesize); - fecho_bands_array[0] = fecho; - fecho_bands_array[1] = fecho + bandsize; - foecho_bands_array[0] = foecho; - foecho_bands_array[1] = foecho + bandsize; - } else { - intbuf2floatbuf(ref, fref, s->framesize); - intbuf2floatbuf(echo, fecho, s->framesize); - fecho_bands_array[0] = fecho; - foecho_bands_array[0] = foecho; - } - if (WebRtcAec_BufferFarend(s->aecInst, (const float*)fref, (size_t)bandsize) != 0) + mswebrtc_aec_splitting_filter_analysis(s->splitting_filter, ref, echo); + if (WebRtcAec_BufferFarend(s->aecInst, + mswebrtc_aec_splitting_filter_get_ref(s->splitting_filter), + (size_t)mswebrtc_aec_splitting_filter_get_bandsize(s->splitting_filter)) != 0) ms_error("WebRtcAec_BufferFarend() failed."); - if (WebRtcAec_Process(s->aecInst, fecho_bands_array, nbands, foecho_bands_array, (size_t)bandsize, 0, 0) != 0) + if (WebRtcAec_Process(s->aecInst, + mswebrtc_aec_splitting_filter_get_echo_bands(s->splitting_filter), + mswebrtc_aec_splitting_filter_get_number_of_bands(s->splitting_filter), + mswebrtc_aec_splitting_filter_get_output_bands(s->splitting_filter), + (size_t)mswebrtc_aec_splitting_filter_get_bandsize(s->splitting_filter), 0, 0) != 0) ms_error("WebRtcAec_Process() failed."); - if (nbands == 2) { - floatbuf2intbuf(foecho, bandsoecho, s->framesize); - WebRtcSpl_SynthesisQMF(bandsoecho, bandsoecho + bandsize, bandsize, (int16_t *)oecho->b_wptr, s->oecho_state1, s->oecho_state2); - } else { - floatbuf2intbuf(foecho, (int16_t *)oecho->b_wptr, s->framesize); - } + mswebrtc_aec_splitting_filter_synthesis(s->splitting_filter, (int16_t *)oecho->b_wptr); } #endif #ifdef BUILD_AECM @@ -401,6 +346,10 @@ static void webrtc_aec_postprocess(MSFilter *f) { ms_bufferizer_flush(&s->delayed_ref); ms_bufferizer_flush(&s->echo); ms_flow_controlled_bufferizer_flush(&s->ref); + if (s->splitting_filter) { + mswebrtc_aec_splitting_filter_destroy(s->splitting_filter); + s->splitting_filter = NULL; + } if (s->aecInst != NULL) { #ifdef BUILD_AEC if (s->aec_type == WebRTCAECTypeNormal) { @@ -421,8 +370,10 @@ static int webrtc_aec_set_sr(MSFilter *f, void *arg) { int requested_sr = *(int *) arg; int sr = requested_sr; - if ((requested_sr != 8000) && (requested_sr != 16000) && (requested_sr != 32000)) { - if ((s->aec_type == WebRTCAECTypeNormal) && (requested_sr > 32000)) { + if ((requested_sr != 8000) && (requested_sr != 16000) && (requested_sr != 32000) && (requested_sr != 48000)) { + if ((s->aec_type == WebRTCAECTypeNormal) && (requested_sr > 48000)) { + sr = 48000; + } else if ((s->aec_type == WebRTCAECTypeNormal) && (requested_sr > 32000)) { sr = 32000; } else if (requested_sr > 16000) { sr = 16000; diff --git a/aec_splitting_filter.cc b/aec_splitting_filter.cc new file mode 100644 index 0000000000000000000000000000000000000000..2fd039b7f090a847a56baf84e5e6a28645ec17fd --- /dev/null +++ b/aec_splitting_filter.cc @@ -0,0 +1,205 @@ +/* +mediastreamer2 library - modular sound and video processing and streaming +Copyright (C) 2017 Belledonne Communications, Grenoble, France + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation; either version 2 +of the License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +*/ + +#include "audio_util.h" +#include "signal_processing_library.h" +#include "three_band_filter_bank.h" + +#include "aec_splitting_filter.h" + + +struct MSWebRtcAecSplittingFilterStruct { + MSWebRtcAecSplittingFilterStruct(int nbands, int bandsize); + ~MSWebRtcAecSplittingFilterStruct(); + void analysis(int16_t *ref, int16_t *echo); + void synthesis(int16_t *oecho); + + float *mRef; + float *mEcho; + float *mOEcho; + float * mRefBandsArray[3]; + float * mEchoBandsArray[3]; + float * mOEchoBandsArray[3]; + int16_t *mBandsRef; + int16_t *mBandsEcho; + int16_t *mBandsOEcho; + float *mBandsRefFloat; + float *mBandsEchoFloat; + float *mBandsOEchoFloat; + webrtc::ThreeBandFilterBank *mThreeBandFilterBankRef; + webrtc::ThreeBandFilterBank *mThreeBandFilterBankEcho; + webrtc::ThreeBandFilterBank *mThreeBandFilterBankOEcho; + int mNbands; + int mBandsize; + int mFramesize; + int mRefState1[6]; + int mRefState2[6]; + int mEchoState1[6]; + int mEchoState2[6]; + int mOEchoState1[6]; + int mOEchoState2[6]; +}; + + +void intbuf2floatbuf(int16_t *intbuf, float *floatbuf, int framesize) { + int i; + for (i = 0; i < framesize; i++) { + floatbuf[i] = (float)intbuf[i]; + } +} + +void floatbuf2intbuf(float *floatbuf, int16_t *intbuf, int framesize) { + int i; + for (i = 0; i < framesize; i++) { + intbuf[i] = webrtc::FloatS16ToS16(floatbuf[i]); + } +} + + +MSWebRtcAecSplittingFilterStruct::MSWebRtcAecSplittingFilterStruct(int nbands, int bandsize) + : mNbands(nbands), mBandsize(bandsize) { + + mFramesize = mNbands * mBandsize; + mRef = new float[mFramesize]; + mEcho = new float[mFramesize]; + mOEcho = new float[mFramesize]; + memset(mRefState1, 0, sizeof(mRefState1)); + memset(mRefState2, 0, sizeof(mRefState2)); + memset(mEchoState1, 0, sizeof(mEchoState1)); + memset(mEchoState2, 0, sizeof(mEchoState2)); + memset(mOEchoState1, 0, sizeof(mOEchoState1)); + memset(mOEchoState2, 0, sizeof(mOEchoState2)); + + if (mNbands == 3) { + mThreeBandFilterBankRef = new webrtc::ThreeBandFilterBank(mFramesize); + mThreeBandFilterBankEcho = new webrtc::ThreeBandFilterBank(mFramesize); + mThreeBandFilterBankOEcho = new webrtc::ThreeBandFilterBank(mFramesize); + mBandsRefFloat = new float[mFramesize]; + mBandsEchoFloat = new float[mFramesize]; + mBandsOEchoFloat = new float[mFramesize]; + mBandsRef = mBandsEcho = mBandsOEcho = nullptr; + mRefBandsArray[0] = mBandsRefFloat; + mRefBandsArray[1] = mBandsRefFloat + mBandsize; + mRefBandsArray[2] = mBandsRefFloat + 2 * mBandsize; + mEchoBandsArray[0] = mBandsEchoFloat; + mEchoBandsArray[1] = mBandsEchoFloat + mBandsize; + mEchoBandsArray[2] = mBandsEchoFloat + 2 * mBandsize; + mOEchoBandsArray[0] = mBandsOEchoFloat; + mOEchoBandsArray[1] = mBandsOEchoFloat + mBandsize; + mOEchoBandsArray[2] = mBandsOEchoFloat + 2 * mBandsize; + } else if (mNbands == 2) { + mBandsRef = new int16_t[mFramesize]; + mBandsEcho = new int16_t[mFramesize]; + mBandsOEcho = new int16_t[mFramesize]; + mThreeBandFilterBankRef = nullptr; + mThreeBandFilterBankEcho = nullptr; + mThreeBandFilterBankOEcho = nullptr; + mBandsRefFloat = mBandsEchoFloat = mBandsOEchoFloat = nullptr; + mRefBandsArray[0] = mRef; + mEchoBandsArray[0] = mEcho; + mEchoBandsArray[1] = mEcho + mBandsize; + mOEchoBandsArray[0] = mOEcho; + mOEchoBandsArray[1] = mOEcho + mBandsize; + } else { + mRefBandsArray[0] = mRef; + mEchoBandsArray[0] = mEcho; + mOEchoBandsArray[0] = mOEcho; + } +} + +MSWebRtcAecSplittingFilterStruct::~MSWebRtcAecSplittingFilterStruct() { + delete[] mRef; + delete[] mEcho; + delete[] mOEcho; + if (mBandsRef) delete[] mBandsRef; + if (mBandsEcho) delete[] mBandsEcho; + if (mBandsOEcho) delete[] mBandsOEcho; + if (mBandsRefFloat) delete[] mBandsRefFloat; + if (mBandsEchoFloat) delete[] mBandsEchoFloat; + if (mBandsOEchoFloat) delete[] mBandsOEchoFloat; + if (mThreeBandFilterBankRef) delete mThreeBandFilterBankRef; + if (mThreeBandFilterBankEcho) delete mThreeBandFilterBankEcho; + if (mThreeBandFilterBankOEcho) delete mThreeBandFilterBankOEcho; +} + +void MSWebRtcAecSplittingFilterStruct::analysis(int16_t *ref, int16_t *echo) { + if (mNbands == 3) { + intbuf2floatbuf(ref, mRef, mFramesize); + mThreeBandFilterBankRef->Analysis(mRef, mFramesize, mRefBandsArray); + intbuf2floatbuf(echo, mEcho, mFramesize); + mThreeBandFilterBankEcho->Analysis(mEcho, mFramesize, mEchoBandsArray); + } else if (mNbands == 2) { + WebRtcSpl_AnalysisQMF(ref, mFramesize, mBandsRef, mBandsRef + mBandsize, mRefState1, mRefState2); + intbuf2floatbuf(mBandsRef, mRef, mFramesize); + WebRtcSpl_AnalysisQMF(echo, mFramesize, mBandsEcho, mBandsEcho + mBandsize, mEchoState1, mEchoState2); + intbuf2floatbuf(mBandsEcho, mEcho, mFramesize); + } else { + intbuf2floatbuf(ref, mRef, mFramesize); + intbuf2floatbuf(echo, mEcho, mFramesize); + } +} + +void MSWebRtcAecSplittingFilterStruct::synthesis(int16_t *oecho) { + if (mNbands == 3) { + mThreeBandFilterBankOEcho->Synthesis(mOEchoBandsArray, mBandsize, mOEcho); + floatbuf2intbuf(mOEcho, oecho, mFramesize); + } else if (mNbands == 2) { + floatbuf2intbuf(mOEcho, mBandsOEcho, mFramesize); + WebRtcSpl_SynthesisQMF(mBandsOEcho, mBandsOEcho + mBandsize, mBandsize, oecho, mOEchoState1, mOEchoState2); + } else { + floatbuf2intbuf(mOEcho, oecho, mFramesize); + } +} + + +MSWebRtcAecSplittingFilter * mswebrtc_aec_splitting_filter_create(int nbands, int bandsize) { + return new MSWebRtcAecSplittingFilterStruct(nbands, bandsize); +} + +void mswebrtc_aec_splitting_filter_destroy(MSWebRtcAecSplittingFilter *filter) { + delete filter; +} + +void mswebrtc_aec_splitting_filter_analysis(MSWebRtcAecSplittingFilter *filter, int16_t *ref, int16_t *echo) { + filter->analysis(ref, echo); +} + +void mswebrtc_aec_splitting_filter_synthesis(MSWebRtcAecSplittingFilter *filter, int16_t *oecho) { + filter->synthesis(oecho); +} + +float * mswebrtc_aec_splitting_filter_get_ref(MSWebRtcAecSplittingFilter *filter) { + return filter->mRefBandsArray[0]; +} + +const float * const * mswebrtc_aec_splitting_filter_get_echo_bands(MSWebRtcAecSplittingFilter *filter) { + return filter->mEchoBandsArray; +} + +float * const * mswebrtc_aec_splitting_filter_get_output_bands(MSWebRtcAecSplittingFilter *filter) { + return filter->mOEchoBandsArray; +} + +int mswebrtc_aec_splitting_filter_get_number_of_bands(MSWebRtcAecSplittingFilter *filter) { + return filter->mNbands; +} + +int mswebrtc_aec_splitting_filter_get_bandsize(MSWebRtcAecSplittingFilter *filter) { + return filter->mBandsize; +} diff --git a/aec_splitting_filter.h b/aec_splitting_filter.h new file mode 100644 index 0000000000000000000000000000000000000000..a3ca9a52f3b14664ddb51bf387d33a3c00d076cc --- /dev/null +++ b/aec_splitting_filter.h @@ -0,0 +1,56 @@ +/* +mediastreamer2 library - modular sound and video processing and streaming +Copyright (C) 2017 Belledonne Communications, Grenoble, France + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation; either version 2 +of the License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +*/ + +#ifndef AEC_SPLITTING_FILTER_H +#define AEC_SPLITTING_FILTER_H + + +#ifdef __cplusplus +extern "C" { +#endif + + +typedef struct MSWebRtcAecSplittingFilterStruct MSWebRtcAecSplittingFilter; + + +MSWebRtcAecSplittingFilter * mswebrtc_aec_splitting_filter_create(int nbands, int bandsize); + +void mswebrtc_aec_splitting_filter_destroy(MSWebRtcAecSplittingFilter *filter); + +void mswebrtc_aec_splitting_filter_analysis(MSWebRtcAecSplittingFilter *filter, int16_t *ref, int16_t *echo); + +void mswebrtc_aec_splitting_filter_synthesis(MSWebRtcAecSplittingFilter *filter, int16_t *oecho); + +float * mswebrtc_aec_splitting_filter_get_ref(MSWebRtcAecSplittingFilter *filter); + +const float * const * mswebrtc_aec_splitting_filter_get_echo_bands(MSWebRtcAecSplittingFilter *filter); + +float * const * mswebrtc_aec_splitting_filter_get_output_bands(MSWebRtcAecSplittingFilter *filter); + +int mswebrtc_aec_splitting_filter_get_number_of_bands(MSWebRtcAecSplittingFilter *filter); + +int mswebrtc_aec_splitting_filter_get_bandsize(MSWebRtcAecSplittingFilter *filter); + + +#ifdef __cplusplus +} +#endif + + +#endif /* AEC_SPLITTING_FILTER_H */ \ No newline at end of file