Commit 98c240f7 authored by Simon Morlat's avatar Simon Morlat
Browse files

factorize and optimize flowcontrol code of AEC implementations.

parent 2307fb76
......@@ -90,6 +90,7 @@ LOCAL_SRC_FILES = \
audiofilters/l16.c \
audiofilters/msresample.c \
audiofilters/devices.c \
audiofilters/flowcontrol.c \
android/androidsound_depr.cpp \
android/loader.cpp \
android/androidsound.cpp \
......
......@@ -97,6 +97,8 @@ set(VOIP_SOURCE_FILES
audiofilters/waveheader.h
audiofilters/winsnd3.c
otherfilters/msrtp.c
audiofilters/flowcontrol.c
audiofilters/flowcontrol.h
utils/_kiss_fft_guts.h
utils/dsptools.c
utils/g711common.h
......
......@@ -103,7 +103,8 @@ libmediastreamer_voip_la_SOURCES+= audiofilters/alaw.c \
audiofilters/genericplc.c \
audiofilters/msfileplayer.c \
audiofilters/msfilerec.c \
audiofilters/waveheader.h
audiofilters/waveheader.h \
audiofilters/flowcontrol.c audiofilters/flowcontrol.h
if BUILD_SPEEX
libmediastreamer_voip_la_SOURCES+= audiofilters/msspeex.c audiofilters/speexec.c
......
/*
* android_echo.h -Android echo cancellation utilities.
* devices.c - sound device information table (latency, presence of AEC etc...)
*
* Copyright (C) 2009-2012 Belledonne Communications, Grenoble, France
*
......
/*
* flowcontrol.c - routines to silently discard samples in excess (used by AEC implementation)
*
* Copyright (C) 2009-2012 Belledonne Communications, Grenoble, France
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
*/
#include "mediastreamer2/mscommon.h"
#include "flowcontrol.h"
void audio_flow_controller_init(AudioFlowController *ctl)
{
ctl->target_samples = 0;
ctl->total_samples = 0;
ctl->current_pos = 0;
ctl->current_dropped = 0;
}
void audio_flow_controller_set_target(AudioFlowController *ctl, int samples_to_drop, int total_samples)
{
ctl->target_samples = samples_to_drop;
ctl->total_samples = total_samples;
ctl->current_pos = 0;
ctl->current_dropped = 0;
}
static void discard_well_choosed_samples(mblk_t *m, int nsamples, int todrop)
{
int i;
int16_t *samples = (int16_t *) m->b_rptr;
int min_diff = 32768;
int pos = 0;
#ifdef TWO_SAMPLES_CRITERIA
for (i = 0; i < nsamples - 1; ++i) {
int tmp = abs((int) samples[i] - (int) samples[i + 1]);
#else
for (i = 0; i < nsamples - 2; ++i) {
int tmp = abs((int) samples[i] - (int) samples[i + 1]) + abs((int) samples[i + 1] - (int) samples[i + 2]);
#endif
if (tmp <= min_diff) {
pos = i;
min_diff = tmp;
}
}
/*ms_message("min_diff=%i at pos %i",min_diff, pos);*/
#ifdef TWO_SAMPLES_CRITERIA
memmove(samples + pos, samples + pos + 1, (nsamples - pos - 1) * 2);
#else
memmove(samples + pos + 1, samples + pos + 2, (nsamples - pos - 2) * 2);
#endif
todrop--;
m->b_wptr -= 2;
nsamples--;
if (todrop > 0) {
/*repeat the same process again*/
discard_well_choosed_samples(m, nsamples, todrop);
}
}
mblk_t *audio_flow_controller_process(AudioFlowController *ctl, mblk_t *m)
{
if (ctl->total_samples > 0 && ctl->target_samples > 0) {
int nsamples = (m->b_wptr - m->b_rptr) / 2;
if (ctl->target_samples * 16 > ctl->total_samples) {
ms_warning("Too many samples to drop, dropping entire frames");
freemsg(m);
ctl->current_pos += nsamples;
m=NULL;
} else {
int th_dropped;
int todrop;
ctl->current_pos += nsamples;
th_dropped = (ctl->target_samples * ctl->current_pos) / ctl->total_samples;
todrop = th_dropped - ctl->current_dropped;
if (todrop > 0) {
if (todrop > nsamples) todrop = nsamples;
discard_well_choosed_samples(m, nsamples, todrop);
/*ms_message("th_dropped=%i, current_dropped=%i, %i samples dropped.",th_dropped,ctl->current_dropped,todrop);*/
ctl->current_dropped += todrop;
}
}
if (ctl->current_pos >= ctl->total_samples) ctl->target_samples = 0; /*stop discarding*/
}
return m;
}
/*
* flowcontrol.h - routines to silently discard samples in excess (used by AEC implementation)
*
* Copyright (C) 2009-2012 Belledonne Communications, Grenoble, France
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
*/
#ifndef flowcontrol_h
#define flowcontrol_h
typedef struct _AudioFlowController {
int target_samples;
int total_samples;
int current_pos;
int current_dropped;
} AudioFlowController;
#ifdef __cplusplus
extern "C"{
#endif
void audio_flow_controller_init(AudioFlowController *ctl);
void audio_flow_controller_set_target(AudioFlowController *ctl, int samples_to_drop, int total_samples);
mblk_t *audio_flow_controller_process(AudioFlowController *ctl, mblk_t *m);
#ifdef __cplusplus
}
#endif
#endif
......@@ -292,8 +292,6 @@ static void compute_max_bitrate(OpusEncData *d, int ptimeStep) {
}
d->bitrate = normalized_cbr;
}
......@@ -670,6 +668,7 @@ static void ms_opus_dec_process(MSFilter *f) {
d->lastPacketLength = frames; // store the packet length for eventual PLC if next two packets are missing
om->b_wptr += frames * d->channels * SIGNAL_SAMPLE_SIZE;
ms_queue_put(f->outputs[0], om);
/*ms_message("Opus: outputing a normal frame of %i bytes (%i samples,%i ms)",(int)(om->b_wptr-om->b_rptr),frames,frames*1000/d->samplerate);*/
d->sequence_number = mblk_get_cseq(im); // used to get eventual FEC information if next packet is missing
ms_concealer_inc_sample_time(d->concealer,f->ticker->time, frames*1000/d->samplerate, 1);
}
......@@ -708,12 +707,12 @@ static void ms_opus_dec_process(MSFilter *f) {
freemsg(om);
} else {
om->b_wptr += frames * d->channels * SIGNAL_SAMPLE_SIZE;
ms_message("Opus: outputing a PLC frame of %i bytes (%i samples,%i ms)",(int)(om->b_wptr-om->b_rptr),frames,frames*1000/d->samplerate);
ms_queue_put(f->outputs[0], om);
d->sequence_number++;
ms_concealer_inc_sample_time(d->concealer,f->ticker->time, frames*1000/d->samplerate, 0);
}
}
}
static void ms_opus_dec_postprocess(MSFilter *f) {
......@@ -800,7 +799,7 @@ static MSFilterMethod ms_opus_dec_methods[] = {
#define MS_OPUS_DEC_ENC_FMT "opus"
#define MS_OPUS_DEC_NINPUTS 1
#define MS_OPUS_DEC_NOUTPUTS 1
#define MS_OPUS_DEC_FLAGS 0
#define MS_OPUS_DEC_FLAGS MS_FILTER_IS_PUMP
#ifndef _MSC_VER
......
......@@ -36,87 +36,7 @@ Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#include <malloc.h> /* for alloca */
#endif
typedef struct _AudioFlowController{
int target_samples;
int total_samples;
int current_pos;
int current_dropped;
}AudioFlowController;
void audio_flow_controller_init(AudioFlowController *ctl){
ctl->target_samples=0;
ctl->total_samples=0;
ctl->current_pos=0;
ctl->current_dropped=0;
}
void audio_flow_controller_set_target(AudioFlowController *ctl, int samples_to_drop, int total_samples){
ctl->target_samples=samples_to_drop;
ctl->total_samples=total_samples;
ctl->current_pos=0;
ctl->current_dropped=0;
}
static void discard_well_choosed_samples(mblk_t *m, int nsamples, int todrop){
int i;
int16_t *samples=(int16_t*)m->b_rptr;
int min_diff=32768;
int pos=0;
#ifdef TWO_SAMPLES_CRITERIA
for(i=0;i<nsamples-1;++i){
int tmp=abs((int)samples[i]- (int)samples[i+1]);
#else
for(i=0;i<nsamples-2;++i){
int tmp=abs((int)samples[i]- (int)samples[i+1])+abs((int)samples[i+1]- (int)samples[i+2]);
#endif
if (tmp<=min_diff){
pos=i;
min_diff=tmp;
}
}
/*ms_message("min_diff=%i at pos %i",min_diff, pos);*/
#ifdef TWO_SAMPLES_CRITERIA
memmove(samples+pos,samples+pos+1,(nsamples-pos-1)*2);
#else
memmove(samples+pos+1,samples+pos+2,(nsamples-pos-2)*2);
#endif
todrop--;
m->b_wptr-=2;
nsamples--;
if (todrop>0){
/*repeat the same process again*/
discard_well_choosed_samples(m,nsamples,todrop);
}
}
mblk_t * audio_flow_controller_process(AudioFlowController *ctl, mblk_t *m){
if (ctl->total_samples>0 && ctl->target_samples>0){
int nsamples=(m->b_wptr-m->b_rptr)/2;
if (ctl->target_samples*16>ctl->total_samples){
ms_warning("Too many samples to drop, dropping entire frames");
m->b_wptr=m->b_rptr;
ctl->current_pos+=nsamples;
}else{
int th_dropped;
int todrop;
ctl->current_pos+=nsamples;
th_dropped=(ctl->target_samples*ctl->current_pos)/ctl->total_samples;
todrop=th_dropped-ctl->current_dropped;
if (todrop>0){
if (todrop>nsamples) todrop=nsamples;
discard_well_choosed_samples(m,nsamples,todrop);
/*ms_message("th_dropped=%i, current_dropped=%i, %i samples dropped.",th_dropped,ctl->current_dropped,todrop);*/
ctl->current_dropped+=todrop;
}
}
if (ctl->current_pos>=ctl->total_samples) ctl->target_samples=0;/*stop discarding*/
}
return m;
}
#include "flowcontrol.h"
//#define EC_DUMP 1
......@@ -138,6 +58,7 @@ typedef struct SpeexECState{
MSBufferizer ref;
MSBufferizer echo;
int framesize;
int framesize_at_8000;
int filterlength;
int samplerate;
int delay_ms;
......@@ -167,7 +88,7 @@ static void speex_ec_init(MSFilter *f){
s->delay_ms=0;
s->tail_length_ms=250;
s->ecstate=NULL;
s->framesize=framesize;
s->framesize_at_8000=framesize;
s->den = NULL;
s->state_str=NULL;
s->using_zeroes=FALSE;
......@@ -254,6 +175,17 @@ static void fetch_config(SpeexECState *s){
#endif
static int adjust_framesize(int framesize, int samplerate){
int newsize=(framesize*samplerate)/8000;
int n=1;
int next;
while((next=n<<1)<=newsize){
n=next;
}
return n;
}
static void speex_ec_preprocess(MSFilter *f){
SpeexECState *s=(SpeexECState*)f->data;
int delay_samples=0;
......@@ -261,6 +193,7 @@ static void speex_ec_preprocess(MSFilter *f){
s->echostarted=FALSE;
s->filterlength=(s->tail_length_ms*s->samplerate)/1000;
s->framesize=adjust_framesize(s->framesize_at_8000,s->samplerate);
delay_samples=s->delay_ms*s->samplerate/1000;
ms_message("Initializing speex echo canceler with framesize=%i, filterlength=%i, delay_samples=%i",
s->framesize,s->filterlength,delay_samples);
......@@ -308,9 +241,12 @@ static void speex_ec_process(MSFilter *f){
if (f->inputs[0]!=NULL){
if (s->echostarted){
while((refm=ms_queue_get(f->inputs[0]))!=NULL){
mblk_t *cp=dupmsg(audio_flow_controller_process(&s->afc,refm));
ms_bufferizer_put(&s->delayed_ref,cp);
ms_bufferizer_put(&s->ref,refm);
refm=audio_flow_controller_process(&s->afc,refm);
if (refm){
mblk_t *cp=dupmsg(refm);
ms_bufferizer_put(&s->delayed_ref,cp);
ms_bufferizer_put(&s->ref,refm);
}
}
}else{
ms_warning("Getting reference signal but no echo to synchronize on.");
......@@ -322,7 +258,7 @@ static void speex_ec_process(MSFilter *f){
ref=(uint8_t*)alloca(nbytes);
echo=(uint8_t*)alloca(nbytes);
while (ms_bufferizer_read(&s->echo,echo,nbytes)>=nbytes){
while (ms_bufferizer_read(&s->echo,echo,nbytes)==nbytes){
mblk_t *oecho=allocb(nbytes,0);
int avail;
int avail_samples;
......@@ -330,6 +266,7 @@ static void speex_ec_process(MSFilter *f){
if (!s->echostarted) s->echostarted=TRUE;
if ((avail=ms_bufferizer_get_avail(&s->delayed_ref))<((s->nominal_ref_samples*2)+nbytes)){
/*we don't have enough to read in a reference signal buffer, inject silence instead*/
avail=nbytes;
refm=allocb(nbytes,0);
memset(refm->b_wptr,0,nbytes);
refm->b_wptr+=nbytes;
......@@ -359,6 +296,7 @@ static void speex_ec_process(MSFilter *f){
}
avail-=nbytes;
avail_samples=avail/2;
/*ms_message("avail=%i",avail_samples);*/
if (avail_samples<s->min_ref_samples || s->min_ref_samples==-1){
s->min_ref_samples=avail_samples;
}
......@@ -378,7 +316,7 @@ static void speex_ec_process(MSFilter *f){
oecho->b_wptr+=nbytes;
ms_queue_put(f->outputs[1],oecho);
}
/*verify our ref buffer does not become too big, meaning that we are receiving more samples than we are sending*/
if ((((uint32_t)(f->ticker->time - s->flow_control_time)) >= flow_control_interval_ms) && (s->min_ref_samples != -1)) {
int diff=s->min_ref_samples-s->nominal_ref_samples;
......@@ -416,7 +354,7 @@ static int speex_ec_set_sr(MSFilter *f, void *arg){
static int speex_ec_set_framesize(MSFilter *f, void *arg){
SpeexECState *s=(SpeexECState*)f->data;
s->framesize = *(int*)arg;
s->framesize_at_8000 = *(int*)arg;
return 0;
}
......
......@@ -34,93 +34,6 @@ Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#include <malloc.h> /* for alloca */
#endif
typedef struct _AudioFlowController {
int target_samples;
int total_samples;
int current_pos;
int current_dropped;
} AudioFlowController;
static void audio_flow_controller_init(AudioFlowController *ctl)
{
ctl->target_samples = 0;
ctl->total_samples = 0;
ctl->current_pos = 0;
ctl->current_dropped = 0;
}
static void audio_flow_controller_set_target(AudioFlowController *ctl, int samples_to_drop, int total_samples)
{
ctl->target_samples = samples_to_drop;
ctl->total_samples = total_samples;
ctl->current_pos = 0;
ctl->current_dropped = 0;
}
static void discard_well_choosed_samples(mblk_t *m, int nsamples, int todrop)
{
int i;
int16_t *samples = (int16_t *) m->b_rptr;
int min_diff = 32768;
int pos = 0;
#ifdef TWO_SAMPLES_CRITERIA
for (i = 0; i < nsamples - 1; ++i) {
int tmp = abs((int) samples[i] - (int) samples[i + 1]);
#else
for (i = 0; i < nsamples - 2; ++i) {
int tmp = abs((int) samples[i] - (int) samples[i + 1]) + abs((int) samples[i + 1] - (int) samples[i + 2]);
#endif
if (tmp <= min_diff) {
pos = i;
min_diff = tmp;
}
}
/*ms_message("min_diff=%i at pos %i",min_diff, pos);*/
#ifdef TWO_SAMPLES_CRITERIA
memmove(samples + pos, samples + pos + 1, (nsamples - pos - 1) * 2);
#else
memmove(samples + pos + 1, samples + pos + 2, (nsamples - pos - 2) * 2);
#endif
todrop--;
m->b_wptr -= 2;
nsamples--;
if (todrop > 0) {
/*repeat the same process again*/
discard_well_choosed_samples(m, nsamples, todrop);
}
}
static mblk_t *audio_flow_controller_process(AudioFlowController *ctl, mblk_t *m)
{
if (ctl->total_samples > 0 && ctl->target_samples > 0) {
int nsamples = (m->b_wptr - m->b_rptr) / 2;
if (ctl->target_samples * 16 > ctl->total_samples) {
ms_warning("Too many samples to drop, dropping entire frames");
m->b_wptr = m->b_rptr;
ctl->current_pos += nsamples;
} else {
int th_dropped;
int todrop;
ctl->current_pos += nsamples;
th_dropped = (ctl->target_samples * ctl->current_pos) / ctl->total_samples;
todrop = th_dropped - ctl->current_dropped;
if (todrop > 0) {
if (todrop > nsamples) todrop = nsamples;
discard_well_choosed_samples(m, nsamples, todrop);
/*ms_message("th_dropped=%i, current_dropped=%i, %i samples dropped.",th_dropped,ctl->current_dropped,todrop);*/
ctl->current_dropped += todrop;
}
}
if (ctl->current_pos >= ctl->total_samples) ctl->target_samples = 0; /*stop discarding*/
}
return m;
}
//#define EC_DUMP 1
#ifdef ANDROID
#define EC_DUMP_PREFIX "/sdcard"
......@@ -128,6 +41,8 @@ static mblk_t *audio_flow_controller_process(AudioFlowController *ctl, mblk_t *m
#define EC_DUMP_PREFIX "/dynamic/tests"
#endif
#include "flowcontrol.h"
static const float smooth_factor = 0.05;
static const int framesize = 80;
static const int flow_control_interval_ms = 5000;
......@@ -269,9 +184,12 @@ static void webrtc_aec_process(MSFilter *f)
if (f->inputs[0] != NULL) {
if (s->echostarted) {
while ((refm = ms_queue_get(f->inputs[0])) != NULL) {
mblk_t *cp = dupmsg(audio_flow_controller_process(&s->afc, refm));
ms_bufferizer_put(&s->delayed_ref, cp);
ms_bufferizer_put(&s->ref, refm);
refm=audio_flow_controller_process(&s->afc,refm);
if (refm){
mblk_t *cp=dupmsg(refm);
ms_bufferizer_put(&s->delayed_ref,cp);
ms_bufferizer_put(&s->ref,refm);
}
}
} else {
ms_warning("Getting reference signal but no echo to synchronize on.");
......
......@@ -649,7 +649,7 @@ int ms_concealer_inc_sample_time(MSConcealerContext* obj, uint64_t current_time,
unsigned int ms_concealer_context_is_concealement_required(MSConcealerContext* obj,uint64_t current_time) {
if(obj->sample_time == -1) return 0; /*no valid value*/
if (obj->sample_time < current_time){
int plc_duration;
if (obj->plc_start_time==-1)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment