Skip to content
GitLab
Commit ab83d321 authored by Ghislain MARY's avatar Ghislain MARY
Browse files

Rework splitting filter code to have AEC at 48KHz.

parent 4fce3d06
master dev_high_sampling_rate feature/AEC-tests feature/AEC3 feature/apple_arm feature/base64-refactor feature/cmake-refactor feature/factorize-cmake-find-package-scripts feature/tester-without-install hotfix/excess-of-samples-to-soundcard release/5.3 release/5.4 5.5.0-alpha 5.4.0 5.4.0-beta 1.1.2 1.1.1
No related merge requests found
Hide whitespace changes
Inline Side-by-side
Showing
with 304 additions and 71 deletions
CMakeLists.txt
+ 21
− 0
View file @ ab83d321
......@@ -59,9 +59,12 @@ endif()
if(LINPHONE_BUILDER_GROUP_EXTERNAL_SOURCE_PATH_BUILDERS)
include("${EP_ms2_CONFIG_DIR}/Mediastreamer2Config.cmake")
include("${EP_ortp_CONFIG_DIR}/ORTPConfig.cmake")
set(BcToolbox_FIND_COMPONENTS tester)
include("${EP_bctoolbox_CONFIG_DIR}/BcToolboxConfig.cmake")
else()
find_package(Mediastreamer2 REQUIRED)
find_package(ORTP REQUIRED)
find_package(BcToolbox REQUIRED)
endif()
find_library(LIBM NAMES m)
......@@ -123,6 +126,8 @@ if(FIXED_POINT_PROCESSOR)
add_definitions(-D__ARMEL__)
endif()
set(AEC_CXX11_FLAGS "-std=c++11")
set(WEBRTC_SIGNAL_PROCESSING_DIR ${WEBRTC_SRC_DIR}/common_audio/signal_processing)
set(WEBRTC_SIGNAL_PROCESSING_SRCS
${WEBRTC_SIGNAL_PROCESSING_DIR}/spl_sqrt.c
......@@ -169,14 +174,26 @@ if(ENABLE_AEC)
set(AEC_SRC_DIR "${WEBRTC_SRC_DIR}/modules/audio_processing/aec")
set(AEC_SRCS
aec.c
aec_splitting_filter.cc
${AEC_SRC_DIR}/aec_core.c
${AEC_SRC_DIR}/aec_rdft.c
${AEC_SRC_DIR}/aec_resampler.c
${AEC_SRC_DIR}/echo_cancellation.c
${WEBRTC_SRC_DIR}/base/checks.cc
${WEBRTC_SRC_DIR}/common_audio/audio_util.cc
${WEBRTC_SRC_DIR}/common_audio/sparse_fir_filter.cc
${WEBRTC_SRC_DIR}/system_wrappers/source/cpu_features.cc
${WEBRTC_SRC_DIR}/modules/audio_processing/three_band_filter_bank.cc
${WEBRTC_SRC_DIR}/modules/audio_processing/utility/delay_estimator.c
${WEBRTC_SRC_DIR}/modules/audio_processing/utility/delay_estimator_wrapper.c
)
set(AEC_CXX11_SRCS
aec_splitting_filter.cc
${WEBRTC_SRC_DIR}/common_audio/audio_util.cc
${WEBRTC_SRC_DIR}/common_audio/sparse_fir_filter.cc
${WEBRTC_SRC_DIR}/modules/audio_processing/three_band_filter_bank.cc
)
bc_apply_compile_flags(AEC_CXX11_SRCS AEC_CXX11_FLAGS)
if(SYSTEM_PROCESSOR MATCHES "^arm.*" OR SYSTEM_PROCESSOR MATCHES "^aarch64.*")
list(APPEND AEC_SRCS
${AEC_SRC_DIR}/aec_core_neon.c
......@@ -190,6 +207,8 @@ if(ENABLE_AEC)
endif()
include_directories(
${AEC_SRC_DIR}/include
${WEBRTC_SRC_DIR}/common_audio/include
${WEBRTC_SRC_DIR}/modules/audio_processing
)
endif()
if(ENABLE_AECM)
......@@ -370,6 +389,7 @@ endif()
if(ENABLE_STATIC)
add_library(mswebrtc-static STATIC ${SOURCE_FILES})
set_target_properties(mswebrtc-static PROPERTIES OUTPUT_NAME mswebrtc)
set_target_properties(mswebrtc-static PROPERTIES LINKER_LANGUAGE CXX)
target_link_libraries(mswebrtc-static ${LIBS})
install(TARGETS mswebrtc-static
ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}/mediastreamer/plugins
......@@ -383,6 +403,7 @@ if(ENABLE_SHARED)
add_library(mswebrtc SHARED ${SOURCE_FILES})
endif()
target_link_libraries(mswebrtc ${LIBS})
set_target_properties(mswebrtc PROPERTIES LINKER_LANGUAGE CXX)
if(IOS)
set(MIN_OS ${LINPHONE_IOS_DEPLOYMENT_TARGET})
set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/build/osx/")
......
aec.c
+ 22
− 71
View file @ ab83d321
......@@ -25,7 +25,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#include "mediastreamer2/msticker.h"
#ifdef BUILD_AEC
#include "echo_cancellation.h"
#include "signal_processing_library.h"
#include "aec_splitting_filter.h"
#endif
#ifdef BUILD_AECM
#include "echo_control_mobile.h"
......@@ -74,12 +74,7 @@ typedef struct WebRTCAECState {
bool_t using_zeroes;
WebRTCAECType aec_type;
#ifdef BUILD_AEC
int ref_state1[6];
int ref_state2[6];
int echo_state1[6];
int echo_state2[6];
int oecho_state1[6];
int oecho_state2[6];
MSWebRtcAecSplittingFilter *splitting_filter;
#endif
} WebRTCAECState;
......@@ -98,14 +93,6 @@ static void webrtc_aecgeneric_init(MSFilter *f, WebRTCAECType aec_type) {
s->echostarted = FALSE;
s->bypass_mode = FALSE;
s->aec_type = aec_type;
#ifdef BUILD_AEC
memset(s->ref_state1, 0, sizeof(s->ref_state1));
memset(s->ref_state2, 0, sizeof(s->ref_state2));
memset(s->echo_state1, 0, sizeof(s->echo_state1));
memset(s->echo_state2, 0, sizeof(s->echo_state2));
memset(s->oecho_state1, 0, sizeof(s->oecho_state1));
memset(s->oecho_state2, 0, sizeof(s->oecho_state2));
#endif
#ifdef EC_DUMP
{
......@@ -225,22 +212,6 @@ static void webrtc_aec_preprocess(MSFilter *f) {
s->nominal_ref_samples = delay_samples;
}
#ifdef BUILD_AEC
void intbuf2floatbuf(int16_t *intbuf, float *floatbuf, int framesize) {
int i;
for (i = 0; i < framesize; i++) {
floatbuf[i] = (float)intbuf[i];
}
}
void floatbuf2intbuf(float *floatbuf, int16_t *intbuf, int framesize) {
int i;
for (i = 0; i < framesize; i++) {
intbuf[i] = (int16_t)floatbuf[i];
}
}
#endif
/* inputs[0]= reference signal from far end (sent to soundcard)
* inputs[1]= near speech & echo signal (read from soundcard)
* outputs[0]= is a copy of inputs[0] to be sent to soundcard
......@@ -251,14 +222,8 @@ static void webrtc_aec_process(MSFilter *f) {
int nbytes = s->framesize * sizeof(int16_t);
mblk_t *refm;
int16_t *ref, *echo;
#ifdef BUILD_AEC
float *fref = NULL, *fecho = NULL, *foecho = NULL;
const float * fecho_bands_array[3];
float * foecho_bands_array[3];
int16_t *bandsref, *bandsecho, *bandsoecho;
int nbands = 1;
int bandsize = s->framesize;
#endif
if (s->bypass_mode) {
while ((refm = ms_queue_get(f->inputs[0])) != NULL) {
......@@ -289,19 +254,12 @@ static void webrtc_aec_process(MSFilter *f) {
echo = (int16_t *) alloca(nbytes);
#ifdef BUILD_AEC
if (s->aec_type == WebRTCAECTypeNormal) {
int nfbytes = s->framesize * sizeof(float);
fref = (float *)alloca(nfbytes);
fecho = (float *)alloca(nfbytes);
foecho = (float *)alloca(nfbytes);
if (s->samplerate > 16000) {
nbands = s->samplerate / 16000;
bandsize = 160;
}
if (nbands > 1) {
int bandsbufsize = nbands * bandsize * sizeof(int16_t);
bandsref = (int16_t *) alloca(bandsbufsize);
bandsecho = (int16_t *) alloca(bandsbufsize);
bandsoecho = (int16_t *) alloca(bandsbufsize);
if (!s->splitting_filter) {
s->splitting_filter = mswebrtc_aec_splitting_filter_create(nbands, bandsize);
}
}
#endif
......@@ -351,31 +309,18 @@ static void webrtc_aec_process(MSFilter *f) {
#endif
#ifdef BUILD_AEC
if (s->aec_type == WebRTCAECTypeNormal) {
if (nbands == 2) {
WebRtcSpl_AnalysisQMF(ref, s->framesize, bandsref, bandsref + bandsize, s->ref_state1, s->ref_state2);
intbuf2floatbuf(bandsref, fref, s->framesize);
WebRtcSpl_AnalysisQMF(echo, s->framesize, bandsecho, bandsecho + bandsize, s->echo_state1, s->echo_state2);
intbuf2floatbuf(bandsecho, fecho, s->framesize);
fecho_bands_array[0] = fecho;
fecho_bands_array[1] = fecho + bandsize;
foecho_bands_array[0] = foecho;
foecho_bands_array[1] = foecho + bandsize;
} else {
intbuf2floatbuf(ref, fref, s->framesize);
intbuf2floatbuf(echo, fecho, s->framesize);
fecho_bands_array[0] = fecho;
foecho_bands_array[0] = foecho;
}
if (WebRtcAec_BufferFarend(s->aecInst, (const float*)fref, (size_t)bandsize) != 0)
mswebrtc_aec_splitting_filter_analysis(s->splitting_filter, ref, echo);
if (WebRtcAec_BufferFarend(s->aecInst,
mswebrtc_aec_splitting_filter_get_ref(s->splitting_filter),
(size_t)mswebrtc_aec_splitting_filter_get_bandsize(s->splitting_filter)) != 0)
ms_error("WebRtcAec_BufferFarend() failed.");
if (WebRtcAec_Process(s->aecInst, fecho_bands_array, nbands, foecho_bands_array, (size_t)bandsize, 0, 0) != 0)
if (WebRtcAec_Process(s->aecInst,
mswebrtc_aec_splitting_filter_get_echo_bands(s->splitting_filter),
mswebrtc_aec_splitting_filter_get_number_of_bands(s->splitting_filter),
mswebrtc_aec_splitting_filter_get_output_bands(s->splitting_filter),
(size_t)mswebrtc_aec_splitting_filter_get_bandsize(s->splitting_filter), 0, 0) != 0)
ms_error("WebRtcAec_Process() failed.");
if (nbands == 2) {
floatbuf2intbuf(foecho, bandsoecho, s->framesize);
WebRtcSpl_SynthesisQMF(bandsoecho, bandsoecho + bandsize, bandsize, (int16_t *)oecho->b_wptr, s->oecho_state1, s->oecho_state2);
} else {
floatbuf2intbuf(foecho, (int16_t *)oecho->b_wptr, s->framesize);
}
mswebrtc_aec_splitting_filter_synthesis(s->splitting_filter, (int16_t *)oecho->b_wptr);
}
#endif
#ifdef BUILD_AECM
......@@ -401,6 +346,10 @@ static void webrtc_aec_postprocess(MSFilter *f) {
ms_bufferizer_flush(&s->delayed_ref);
ms_bufferizer_flush(&s->echo);
ms_flow_controlled_bufferizer_flush(&s->ref);
if (s->splitting_filter) {
mswebrtc_aec_splitting_filter_destroy(s->splitting_filter);
s->splitting_filter = NULL;
}
if (s->aecInst != NULL) {
#ifdef BUILD_AEC
if (s->aec_type == WebRTCAECTypeNormal) {
......@@ -421,8 +370,10 @@ static int webrtc_aec_set_sr(MSFilter *f, void *arg) {
int requested_sr = *(int *) arg;
int sr = requested_sr;
if ((requested_sr != 8000) && (requested_sr != 16000) && (requested_sr != 32000)) {
if ((s->aec_type == WebRTCAECTypeNormal) && (requested_sr > 32000)) {
if ((requested_sr != 8000) && (requested_sr != 16000) && (requested_sr != 32000) && (requested_sr != 48000)) {
if ((s->aec_type == WebRTCAECTypeNormal) && (requested_sr > 48000)) {
sr = 48000;
} else if ((s->aec_type == WebRTCAECTypeNormal) && (requested_sr > 32000)) {
sr = 32000;
} else if (requested_sr > 16000) {
sr = 16000;
......
aec_splitting_filter.cc 0 → 100644
+ 205
− 0
View file @ ab83d321
/*
mediastreamer2 library - modular sound and video processing and streaming
Copyright (C) 2017 Belledonne Communications, Grenoble, France
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; either version 2
of the License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
#include "audio_util.h"
#include "signal_processing_library.h"
#include "three_band_filter_bank.h"
#include "aec_splitting_filter.h"
struct MSWebRtcAecSplittingFilterStruct {
MSWebRtcAecSplittingFilterStruct(int nbands, int bandsize);
~MSWebRtcAecSplittingFilterStruct();
void analysis(int16_t *ref, int16_t *echo);
void synthesis(int16_t *oecho);
float *mRef;
float *mEcho;
float *mOEcho;
float * mRefBandsArray[3];
float * mEchoBandsArray[3];
float * mOEchoBandsArray[3];
int16_t *mBandsRef;
int16_t *mBandsEcho;
int16_t *mBandsOEcho;
float *mBandsRefFloat;
float *mBandsEchoFloat;
float *mBandsOEchoFloat;
webrtc::ThreeBandFilterBank *mThreeBandFilterBankRef;
webrtc::ThreeBandFilterBank *mThreeBandFilterBankEcho;
webrtc::ThreeBandFilterBank *mThreeBandFilterBankOEcho;
int mNbands;
int mBandsize;
int mFramesize;
int mRefState1[6];
int mRefState2[6];
int mEchoState1[6];
int mEchoState2[6];
int mOEchoState1[6];
int mOEchoState2[6];
};
void intbuf2floatbuf(int16_t *intbuf, float *floatbuf, int framesize) {
int i;
for (i = 0; i < framesize; i++) {
floatbuf[i] = (float)intbuf[i];
}
}
void floatbuf2intbuf(float *floatbuf, int16_t *intbuf, int framesize) {
int i;
for (i = 0; i < framesize; i++) {
intbuf[i] = webrtc::FloatS16ToS16(floatbuf[i]);
}
}
MSWebRtcAecSplittingFilterStruct::MSWebRtcAecSplittingFilterStruct(int nbands, int bandsize)
: mNbands(nbands), mBandsize(bandsize) {
mFramesize = mNbands * mBandsize;
mRef = new float[mFramesize];
mEcho = new float[mFramesize];
mOEcho = new float[mFramesize];
memset(mRefState1, 0, sizeof(mRefState1));
memset(mRefState2, 0, sizeof(mRefState2));
memset(mEchoState1, 0, sizeof(mEchoState1));
memset(mEchoState2, 0, sizeof(mEchoState2));
memset(mOEchoState1, 0, sizeof(mOEchoState1));
memset(mOEchoState2, 0, sizeof(mOEchoState2));
if (mNbands == 3) {
mThreeBandFilterBankRef = new webrtc::ThreeBandFilterBank(mFramesize);
mThreeBandFilterBankEcho = new webrtc::ThreeBandFilterBank(mFramesize);
mThreeBandFilterBankOEcho = new webrtc::ThreeBandFilterBank(mFramesize);
mBandsRefFloat = new float[mFramesize];
mBandsEchoFloat = new float[mFramesize];
mBandsOEchoFloat = new float[mFramesize];
mBandsRef = mBandsEcho = mBandsOEcho = nullptr;
mRefBandsArray[0] = mBandsRefFloat;
mRefBandsArray[1] = mBandsRefFloat + mBandsize;
mRefBandsArray[2] = mBandsRefFloat + 2 * mBandsize;
mEchoBandsArray[0] = mBandsEchoFloat;
mEchoBandsArray[1] = mBandsEchoFloat + mBandsize;
mEchoBandsArray[2] = mBandsEchoFloat + 2 * mBandsize;
mOEchoBandsArray[0] = mBandsOEchoFloat;
mOEchoBandsArray[1] = mBandsOEchoFloat + mBandsize;
mOEchoBandsArray[2] = mBandsOEchoFloat + 2 * mBandsize;
} else if (mNbands == 2) {
mBandsRef = new int16_t[mFramesize];
mBandsEcho = new int16_t[mFramesize];
mBandsOEcho = new int16_t[mFramesize];
mThreeBandFilterBankRef = nullptr;
mThreeBandFilterBankEcho = nullptr;
mThreeBandFilterBankOEcho = nullptr;
mBandsRefFloat = mBandsEchoFloat = mBandsOEchoFloat = nullptr;
mRefBandsArray[0] = mRef;
mEchoBandsArray[0] = mEcho;
mEchoBandsArray[1] = mEcho + mBandsize;
mOEchoBandsArray[0] = mOEcho;
mOEchoBandsArray[1] = mOEcho + mBandsize;
} else {
mRefBandsArray[0] = mRef;
mEchoBandsArray[0] = mEcho;
mOEchoBandsArray[0] = mOEcho;
}
}
MSWebRtcAecSplittingFilterStruct::~MSWebRtcAecSplittingFilterStruct() {
delete[] mRef;
delete[] mEcho;
delete[] mOEcho;
if (mBandsRef) delete[] mBandsRef;
if (mBandsEcho) delete[] mBandsEcho;
if (mBandsOEcho) delete[] mBandsOEcho;
if (mBandsRefFloat) delete[] mBandsRefFloat;
if (mBandsEchoFloat) delete[] mBandsEchoFloat;
if (mBandsOEchoFloat) delete[] mBandsOEchoFloat;
if (mThreeBandFilterBankRef) delete mThreeBandFilterBankRef;
if (mThreeBandFilterBankEcho) delete mThreeBandFilterBankEcho;
if (mThreeBandFilterBankOEcho) delete mThreeBandFilterBankOEcho;
}
void MSWebRtcAecSplittingFilterStruct::analysis(int16_t *ref, int16_t *echo) {
if (mNbands == 3) {
intbuf2floatbuf(ref, mRef, mFramesize);
mThreeBandFilterBankRef->Analysis(mRef, mFramesize, mRefBandsArray);
intbuf2floatbuf(echo, mEcho, mFramesize);
mThreeBandFilterBankEcho->Analysis(mEcho, mFramesize, mEchoBandsArray);
} else if (mNbands == 2) {
WebRtcSpl_AnalysisQMF(ref, mFramesize, mBandsRef, mBandsRef + mBandsize, mRefState1, mRefState2);
intbuf2floatbuf(mBandsRef, mRef, mFramesize);
WebRtcSpl_AnalysisQMF(echo, mFramesize, mBandsEcho, mBandsEcho + mBandsize, mEchoState1, mEchoState2);
intbuf2floatbuf(mBandsEcho, mEcho, mFramesize);
} else {
intbuf2floatbuf(ref, mRef, mFramesize);
intbuf2floatbuf(echo, mEcho, mFramesize);
}
}
void MSWebRtcAecSplittingFilterStruct::synthesis(int16_t *oecho) {
if (mNbands == 3) {
mThreeBandFilterBankOEcho->Synthesis(mOEchoBandsArray, mBandsize, mOEcho);
floatbuf2intbuf(mOEcho, oecho, mFramesize);
} else if (mNbands == 2) {
floatbuf2intbuf(mOEcho, mBandsOEcho, mFramesize);
WebRtcSpl_SynthesisQMF(mBandsOEcho, mBandsOEcho + mBandsize, mBandsize, oecho, mOEchoState1, mOEchoState2);
} else {
floatbuf2intbuf(mOEcho, oecho, mFramesize);
}
}
MSWebRtcAecSplittingFilter * mswebrtc_aec_splitting_filter_create(int nbands, int bandsize) {
return new MSWebRtcAecSplittingFilterStruct(nbands, bandsize);
}
void mswebrtc_aec_splitting_filter_destroy(MSWebRtcAecSplittingFilter *filter) {
delete filter;
}
void mswebrtc_aec_splitting_filter_analysis(MSWebRtcAecSplittingFilter *filter, int16_t *ref, int16_t *echo) {
filter->analysis(ref, echo);
}
void mswebrtc_aec_splitting_filter_synthesis(MSWebRtcAecSplittingFilter *filter, int16_t *oecho) {
filter->synthesis(oecho);
}
float * mswebrtc_aec_splitting_filter_get_ref(MSWebRtcAecSplittingFilter *filter) {
return filter->mRefBandsArray[0];
}
const float * const * mswebrtc_aec_splitting_filter_get_echo_bands(MSWebRtcAecSplittingFilter *filter) {
return filter->mEchoBandsArray;
}
float * const * mswebrtc_aec_splitting_filter_get_output_bands(MSWebRtcAecSplittingFilter *filter) {
return filter->mOEchoBandsArray;
}
int mswebrtc_aec_splitting_filter_get_number_of_bands(MSWebRtcAecSplittingFilter *filter) {
return filter->mNbands;
}
int mswebrtc_aec_splitting_filter_get_bandsize(MSWebRtcAecSplittingFilter *filter) {
return filter->mBandsize;
}
aec_splitting_filter.h 0 → 100644
+ 56
− 0
View file @ ab83d321
/*
mediastreamer2 library - modular sound and video processing and streaming
Copyright (C) 2017 Belledonne Communications, Grenoble, France
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; either version 2
of the License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
#ifndef AEC_SPLITTING_FILTER_H
#define AEC_SPLITTING_FILTER_H
#ifdef __cplusplus
extern "C" {
#endif
typedef struct MSWebRtcAecSplittingFilterStruct MSWebRtcAecSplittingFilter;
MSWebRtcAecSplittingFilter * mswebrtc_aec_splitting_filter_create(int nbands, int bandsize);
void mswebrtc_aec_splitting_filter_destroy(MSWebRtcAecSplittingFilter *filter);
void mswebrtc_aec_splitting_filter_analysis(MSWebRtcAecSplittingFilter *filter, int16_t *ref, int16_t *echo);
void mswebrtc_aec_splitting_filter_synthesis(MSWebRtcAecSplittingFilter *filter, int16_t *oecho);
float * mswebrtc_aec_splitting_filter_get_ref(MSWebRtcAecSplittingFilter *filter);
const float * const * mswebrtc_aec_splitting_filter_get_echo_bands(MSWebRtcAecSplittingFilter *filter);
float * const * mswebrtc_aec_splitting_filter_get_output_bands(MSWebRtcAecSplittingFilter *filter);
int mswebrtc_aec_splitting_filter_get_number_of_bands(MSWebRtcAecSplittingFilter *filter);
int mswebrtc_aec_splitting_filter_get_bandsize(MSWebRtcAecSplittingFilter *filter);
#ifdef __cplusplus
}
#endif
#endif /* AEC_SPLITTING_FILTER_H */
\ No newline at end of file
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment