Commit 5ae1633e authored by Simon Morlat's avatar Simon Morlat

reorganize source code, add new vad-dtx filter for generic confort noise

parent 5cb6762f
......@@ -76,6 +76,7 @@ LOCAL_SRC_FILES = \
utils/g722_encode.c \
utils/audiodiff.c \
utils/stream_regulator.c \
utils/extremum.c \
otherfilters/msrtp.c \
otherfilters/tee.c \
otherfilters/join.c \
......@@ -96,6 +97,7 @@ LOCAL_SRC_FILES = \
audiofilters/msresample.c \
audiofilters/devices.c \
audiofilters/flowcontrol.c \
audiofilters/msvaddtx.c \
audiofilters/aac-eld-android.cpp \
android/hardware_echo_canceller.cpp \
android/androidsound_depr.cpp \
......
......@@ -46,7 +46,9 @@ mediastreamer2_include_HEADERS= ice.h \
zrtp.h \
stun.h \
stun_udp.h \
ms_srtp.h
ms_srtp.h \
msvaddtx.h \
msutils.h
EXTRA_DIST=$(mediastreamer2_include_HEADERS)
......@@ -108,7 +108,7 @@ typedef enum MSFilterId{
MS_ANDROID_VIDEO_READ_ID,
MS_ANDROID_VIDEO_WRITE_ID,
MS_TONE_DETECTOR_ID,
MY_FILTER_ID,
MY_FILTER_ID,
MS_IOS_DISPLAY_ID,
MS_VP8_ENC_ID,
MS_VP8_DEC_ID,
......@@ -143,7 +143,8 @@ typedef enum MSFilterId{
MS_QSA_READ_ID,
MS_QSA_WRITE_ID,
MS_MKV_RECORDER_ID,
MS_MKV_PLAYER_ID
MS_MKV_PLAYER_ID,
MS_VAD_DTX_ID
} MSFilterId;
......
......@@ -382,10 +382,6 @@ void ms_ifft(void *table, ms_word16_t *in, ms_word16_t *out);
void ms_fir_mem16(const ms_word16_t *x, const ms_coef_t *num, ms_word16_t *y, int N, int ord, ms_mem_t *mem);
typedef void (*MSAudioDiffProgressNotify)(void* user_data, int percentage);
/*utility function to check similarity between two audio wav files*/
int ms_audio_diff(const char *file1, const char *file2, double *ret, MSAudioDiffProgressNotify func, void *user_data);
#ifdef __cplusplus
}
#endif
......
......@@ -313,6 +313,7 @@ struct _AudioStream
int videopin;
bool_t plumbed;
}av_player;
MSFilter *vaddtx;
char *recorder_file;
EchoLimiterType el_type; /*use echo limiter: two MSVolume, measured input level controlling local output level*/
EqualizerLocation eq_loc;
......
/*
mediastreamer2 library - modular sound and video processing and streaming
Copyright (C) 2014 Belledonne Communications SARL
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; either version 2
......
/*
mediastreamer2 library - modular sound and video processing and streaming
Copyright (C) 2014 Belledonne Communications SARL
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; either version 2
of the License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
*/
#ifndef msutils_h
#define msutils_h
#include "mediastreamer2/mscommon.h"
#ifdef __cplusplus
extern "C"{
#endif
typedef void (*MSAudioDiffProgressNotify)(void* user_data, int percentage);
/*utility function to check similarity between two audio wav files*/
MS2_PUBLIC int ms_audio_diff(const char *file1, const char *file2, double *ret, MSAudioDiffProgressNotify func, void *user_data);
/*Utility object to determine a maximum or minimum (but not both at the same time), of a signal during a sliding period of time.
**/
typedef struct _MSExtremum{
float current_extremum;
uint64_t extremum_time;
float last_stable;
int period;
}MSExtremum;
MS2_PUBLIC void ms_extremum_reset(MSExtremum *obj);
MS2_PUBLIC void ms_extremum_init(MSExtremum *obj, int period);
MS2_PUBLIC void ms_extremum_record_min(MSExtremum *obj, uint64_t curtime, float value);
MS2_PUBLIC void ms_extremum_record_max(MSExtremum *obj, uint64_t curtime, float value);
MS2_PUBLIC float ms_extremum_get_current(MSExtremum *obj);
#ifdef __cplusplus
}
#endif
#endif
/*
mediastreamer2 library - modular sound and video processing and streaming
Copyright (C) 2014 Belledonne Communications SARL
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; either version 2
of the License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
*/
#include "mediastreamer2/msfilter.h"
typedef struct _MSCngData{
int datasize;
uint8_t data[32];
}MSCngData;
/** Event generated when silence is detected. Payload contains the data encoding the background noise*/
#define MS_VAD_DTX_NO_VOICE MS_FILTER_EVENT(MS_VAD_DTX_ID, 0, MSCngData)
......@@ -128,6 +128,7 @@ libmediastreamer_voip_la_SOURCES+= audiofilters/alaw.c \
utils/kiss_fftr.c \
utils/kiss_fftr.h \
utils/audiodiff.c \
utils/extremum.c \
audiofilters/equalizer.c \
audiofilters/chanadapt.c \
audiofilters/audiomixer.c \
......@@ -142,7 +143,8 @@ libmediastreamer_voip_la_SOURCES+= audiofilters/alaw.c \
audiofilters/msfileplayer.c \
audiofilters/msfilerec.c \
audiofilters/waveheader.h \
audiofilters/flowcontrol.c
audiofilters/flowcontrol.c \
audiofilters/msvaddtx.c
if BUILD_SPEEX
libmediastreamer_voip_la_SOURCES+= audiofilters/msspeex.c audiofilters/speexec.c
......
......@@ -22,7 +22,6 @@ Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#include "mediastreamer2/msfilter.h"
#include "mediastreamer2/mscodecutils.h"
#include "mediastreamer2/msfilter.h"
#include "mediastreamer2/msticker.h"
#include "ortp/rtp.h"
......
/*
msvaddtx.c - Generic VAD/DTX (voice activity detector, discontinuous transmission)
for CN payload type (RFC3389)
mediastreamer2 library - modular sound and video processing and streaming
Copyright (C) 2011-2012 Belledonne Communications, Grenoble, France
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; either version 2
of the License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
*/
#include "mediastreamer2/msfilter.h"
#include "mediastreamer2/msticker.h"
#include "mediastreamer2/msutils.h"
#include "mediastreamer2/msvaddtx.h"
#include <math.h>
static const float max_e = (32768* 0.7); /* 0.7 - is RMS factor */
static const float coef = 0.2; /* floating averaging coeff. for energy */
static const float silence_threshold=0.001;
typedef struct _VadDtxContext{
int silence_mode;/*set to 1 if a silence period is running*/
#ifndef HAVE_G729B
float energy;
MSExtremum max;
#else
#endif
}VadDtxContext;
static void vad_dtx_init(MSFilter *f){
VadDtxContext *ctx=ms_new0(VadDtxContext,1);
f->data=ctx;
ms_extremum_init(&ctx->max,2000);
}
static void vad_dtx_preprocess(MSFilter *f){
VadDtxContext *ctx=(VadDtxContext*)f->data;
ms_extremum_reset(&ctx->max);
}
static void update_energy(VadDtxContext *v, int16_t *signal, int numsamples, uint64_t curtime) {
int i;
float acc = 0;
float en;
for (i=0;i<numsamples;++i){
int s=signal[i];
acc += s * s;
}
en = (sqrt(acc / numsamples)+1) / max_e;
v->energy = (en * coef) + v->energy * (1.0 - coef);
ms_extremum_record_max(&v->max,curtime,v->energy);
}
static void vad_dtx_process(MSFilter *f){
VadDtxContext *ctx=(VadDtxContext*)f->data;
mblk_t *m;
while((m=ms_queue_get(f->inputs[0]))!=NULL){
update_energy(ctx,(int16_t*)m->b_rptr, (m->b_wptr - m->b_rptr) / 2, f->ticker->time);
if (ms_extremum_get_current(&ctx->max)<silence_threshold){
if (!ctx->silence_mode){
MSCngData cngdata={0};
cngdata.datasize=1; /*only noise level*/
cngdata.data[0]=0; /*noise level set to zero for the moment*/
ms_message("vad_dtx_process(): silence period detected.");
ctx->silence_mode=1;
ms_filter_notify(f, MS_VAD_DTX_NO_VOICE, &cngdata);
}
}else{
if (ctx->silence_mode){
ms_message("vad_dtx_process(): silence period finished.");
ctx->silence_mode=1;
}
}
if (!ctx->silence_mode) ms_queue_put(f->outputs[0],m);
}
}
static void vad_dtx_postprocess(MSFilter *f){
//VadDtxContext *ctx=(VadDtxContext*)f->data;
}
static void vad_dtx_uninit(MSFilter *f){
VadDtxContext *ctx=(VadDtxContext*)f->data;
ms_free(ctx);
}
#ifndef _MSC_VER
MSFilterDesc ms_vad_dtx_desc = {
.id = MS_VAD_DTX_ID,
.name = "MSVadDtx",
.text = "A filter detecting silence period and encoding residual noise",
.category = MS_FILTER_OTHER,
.ninputs = 1,
.noutputs = 1,
.init = vad_dtx_init,
.preprocess = vad_dtx_preprocess,
.process = vad_dtx_process,
.postprocess = vad_dtx_postprocess,
.uninit = vad_dtx_uninit,
};
#else
MSFilterDesc ms_vad_dtx_desc = {
MS_VAD_DTX_ID,
"MSVadDtx",
"A filter detecting silence period and encoding residual noise",
MS_FILTER_OTHER,
NULL,
1,
1,
vad_dtx_init,
vad_dtx_preprocess,
vad_dtx_process,
vad_dtx_postprocess,
vad_dtx_uninit
};
#endif
MS_FILTER_DESC_EXPORT(ms_vad_dtx_desc)
......@@ -23,6 +23,7 @@ Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#include "mediastreamer2/msvolume.h"
#include "mediastreamer2/msticker.h"
#include "mediastreamer2/msutils.h"
#include <math.h>
#ifdef HAVE_SPEEXDSP
......@@ -40,75 +41,13 @@ static const float transmit_thres=4;
static const float min_ng_floorgain=0.005;
static const float agc_threshold=0.5;
typedef struct Extremum{
float current_extremum;
uint64_t extremum_time;
float last_stable;
int period;
}Extremum;
static void extremum_reset(Extremum *obj){
obj->current_extremum=0;
obj->extremum_time=(uint64_t)-1;
obj->last_stable=0;
}
static void extremum_init(Extremum *obj, int period){
extremum_reset(obj);
obj->period=period;
}
static void extremum_set_new(Extremum *obj, const char* kind){
obj->last_stable=obj->current_extremum;
/*ms_message("New %s value : %f",kind,obj->last_stable);*/
}
static void extremum_check_init(Extremum *obj, uint64_t curtime, float value, const char *kind){
if (obj->extremum_time!=(uint64_t)-1){
if (curtime-obj->extremum_time>obj->period){
/*last extremum is too old, drop it*/
extremum_set_new(obj,kind);
obj->extremum_time=(uint64_t)-1;
}
}
if (obj->extremum_time==(uint64_t)-1){
obj->current_extremum=value;
obj->extremum_time=curtime;
}
}
static void extremum_record_min(Extremum *obj, uint64_t curtime, float value){
extremum_check_init(obj,curtime,value,"min");
if (value<obj->current_extremum){
obj->current_extremum=value;
obj->extremum_time=curtime;
if (value<obj->last_stable){
extremum_set_new(obj,"min");
}
}
}
static void extremum_record_max(Extremum *obj, uint64_t curtime, float value){
extremum_check_init(obj,curtime,value,"max");
if (value>obj->current_extremum){
obj->current_extremum=value;
obj->extremum_time=curtime;
if (value>obj->last_stable){
extremum_set_new(obj,"max");
}
}
}
static float extremum_get_current(Extremum *obj){
return obj->last_stable;
}
typedef struct Volume{
float energy;
float level_pk;
float instant_energy;
float lt_speaker_en;
float gain; /**< the one really applied, smoothed target_gain version*/
float gain; /**< the one really applied, smoothed target_gain version*/
float static_gain; /**< the one fixed by the user */
int dc_offset;
//float gain_k;
......@@ -133,8 +72,8 @@ typedef struct Volume{
float ng_floorgain;
float ng_gain;
MSBufferizer *buffer;
Extremum min;
Extremum max;
MSExtremum min;
MSExtremum max;
bool_t agc_enabled;
bool_t noise_gate_enabled;
bool_t remove_dc;
......@@ -170,8 +109,8 @@ static void volume_init(MSFilter *f){
#ifdef HAVE_SPEEXDSP
v->speex_pp=NULL;
#endif
extremum_init(&v->max,1000);
extremum_init(&v->min,30000);
ms_extremum_init(&v->max,1000);
ms_extremum_init(&v->min,30000);
f->data=v;
}
......@@ -200,14 +139,14 @@ static int volume_get(MSFilter *f, void *arg){
static int volume_get_min(MSFilter *f, void *arg){
float *farg=(float*)arg;
Volume *v=(Volume*)f->data;
*farg=linear_to_db(extremum_get_current(&v->min));
*farg=linear_to_db(ms_extremum_get_current(&v->min));
return 0;
}
static int volume_get_max(MSFilter *f, void *arg){
float *farg=(float*)arg;
Volume *v=(Volume*)f->data;
*farg=linear_to_db(extremum_get_current(&v->max));
*farg=linear_to_db(ms_extremum_get_current(&v->max));
return 0;
}
......@@ -463,8 +402,8 @@ static void update_energy(Volume *v, int16_t *signal, int numsamples, uint64_t c
v->energy = (en * coef) + v->energy * (1.0 - coef);
v->level_pk = (float)pk / max_e;
v->instant_energy = en;// currently non-averaged energy seems better (short artefacts)
extremum_record_max(&v->max,curtime,v->energy);
extremum_record_min(&v->min,curtime,v->energy);
ms_extremum_record_max(&v->max,curtime,v->energy);
ms_extremum_record_min(&v->min,curtime,v->energy);
}
static void apply_gain(Volume *v, mblk_t *m, float tgain) {
......@@ -532,8 +471,8 @@ static void volume_preprocess(MSFilter *f){
}
#endif
}
extremum_reset(&v->min);
extremum_reset(&v->max);
ms_extremum_reset(&v->min);
ms_extremum_reset(&v->max);
}
static void volume_process(MSFilter *f){
......
......@@ -655,7 +655,7 @@ struct _MSEventQueue *ms_factory_get_event_queue(MSFactory *obj){
return obj->evq;
}
/*this function is for compatibility, when event queues where created by the application*/
/*this function is for compatibility, when event queues were created by the application*/
void ms_factory_set_event_queue(MSFactory *obj, MSEventQueue *evq){
obj->evq=evq;
}
......
......@@ -19,7 +19,7 @@ Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
*/
#include "mediastreamer2/mscommon.h"
#include "mediastreamer2/dsptools.h"
#include "mediastreamer2/msutils.h"
#include "waveheader.h"
#include <math.h>
......
/*
mediastreamer2 library - modular sound and video processing and streaming
Copyright (C) 2014 Belledonne Communications SARL
Author: Simon MORLAT (simon.morlat@linphone.org)
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; either version 2
of the License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
*/
#include "mediastreamer2/msutils.h"
void ms_extremum_reset(MSExtremum *obj){
obj->current_extremum=0;
obj->extremum_time=(uint64_t)-1;
obj->last_stable=0;
}
void ms_extremum_init(MSExtremum *obj, int period){
ms_extremum_reset(obj);
obj->period=period;
}
static void extremum_set_new(MSExtremum *obj, const char* kind){
obj->last_stable=obj->current_extremum;
/*ms_message("New %s value : %f",kind,obj->last_stable);*/
}
static void extremum_check_init(MSExtremum *obj, uint64_t curtime, float value, const char *kind){
if (obj->extremum_time!=(uint64_t)-1){
if (curtime-obj->extremum_time>obj->period){
/*last extremum is too old, drop it*/
extremum_set_new(obj,kind);
obj->extremum_time=(uint64_t)-1;
}
}
if (obj->extremum_time==(uint64_t)-1){
obj->current_extremum=value;
obj->extremum_time=curtime;
}
}
void ms_extremum_record_min(MSExtremum *obj, uint64_t curtime, float value){
extremum_check_init(obj,curtime,value,"min");
if (value<obj->current_extremum){
obj->current_extremum=value;
obj->extremum_time=curtime;
if (value<obj->last_stable){
extremum_set_new(obj,"min");
}
}
}
void ms_extremum_record_max(MSExtremum *obj, uint64_t curtime, float value){
extremum_check_init(obj,curtime,value,"max");
if (value>obj->current_extremum){
obj->current_extremum=value;
obj->extremum_time=curtime;
if (value>obj->last_stable){
extremum_set_new(obj,"max");
}
}
}
float ms_extremum_get_current(MSExtremum *obj){
return obj->last_stable;
}
......@@ -71,6 +71,7 @@ static void audio_stream_free(AudioStream *stream) {
if (stream->av_recorder.recorder) ms_filter_destroy(stream->av_recorder.recorder);
if (stream->av_recorder.resampler) ms_filter_destroy(stream->av_recorder.resampler);
if (stream->av_recorder.video_input) ms_filter_destroy(stream->av_recorder.video_input);
if (stream->vaddtx) ms_filter_destroy(stream->vaddtx);
if (stream->outbound_mixer) ms_filter_destroy(stream->outbound_mixer);
if (stream->recorder_file) ms_free(stream->recorder_file);
......@@ -586,6 +587,22 @@ static void setup_recorder(AudioStream *stream, int sample_rate, int nchannels){
setup_av_recorder(stream,sample_rate,nchannels);
}
static void on_silence_detected(void *data, MSFilter *f, unsigned int event_id, void *event_arg){
ms_message("on_silence_detected(): CN packet to be sent !");
}
static void setup_generic_confort_noise(AudioStream *stream){
RtpProfile *prof=rtp_session_get_profile(stream->ms.sessions.rtp_session);
PayloadType *pt=rtp_profile_get_payload(prof, rtp_session_get_send_payload_type(stream->ms.sessions.rtp_session));
PayloadType *cn=rtp_profile_find_payload(prof, "CN", 8000, 1);
if (cn && pt && pt->channels==1 && pt->clock_rate==8000){
/* RFC3389 CN can be used*/
stream->vaddtx=ms_filter_new(MS_VAD_DTX_ID);
ms_filter_add_notify_callback(stream->vaddtx, on_silence_detected, stream, TRUE);
}
}
int audio_stream_start_full(AudioStream *stream, RtpProfile *profile, const char *rem_rtp_ip,int rem_rtp_port,
const char *rem_rtcp_ip, int rem_rtcp_port, int payload,int jitt_comp, const char *infile, const char *outfile,
MSSndCard *playcard, MSSndCard *captcard, bool_t use_ec){
......@@ -855,6 +872,8 @@ int audio_stream_start_full(AudioStream *stream, RtpProfile *profile, const char
ms_filter_call_method(stream->plc, MS_FILTER_SET_NCHANNELS, &nchannels);
ms_filter_call_method(stream->plc, MS_FILTER_SET_SAMPLE_RATE, &sample_rate);
}
/*as first rough approximation, a codec without PLC capabilities has no VAD/DTX builtin, thus setup generic confort noise if possible*/
setup_generic_confort_noise(stream);
} else {
stream->plc = NULL;
}
......@@ -889,6 +908,8 @@ int audio_stream_start_full(AudioStream *stream, RtpProfile *profile, const char
ms_connection_helper_link(&h,stream->dtmfgen_rtp,0,0);
if (stream->outbound_mixer)
ms_connection_helper_link(&h,stream->outbound_mixer,0,0);
if (stream->vaddtx)
ms_connection_helper_link(&h,stream->vaddtx,0,0);
ms_connection_helper_link(&h,stream->ms.encoder,0,0);
ms_connection_helper_link(&h,stream->ms.rtpsend,0,-1);
......@@ -1275,6 +1296,8 @@ void audio_stream_stop(AudioStream * stream){
ms_connection_helper_unlink(&h,stream->dtmfgen_rtp,0,0);
if (stream->outbound_mixer)
ms_connection_helper_unlink(&h,stream->outbound_mixer,0,0);
if (stream->vaddtx)
ms_connection_helper_unlink(&h,stream->vaddtx,0,0);
ms_connection_helper_unlink(&h,stream->ms.encoder,0,0);
ms_connection_helper_unlink(&h,stream->ms.rtpsend,0,-1);
......
......@@ -19,7 +19,7 @@ Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
*/
#include "mediastreamer2/mscommon.h"
#include "mediastreamer2/dsptools.h"
#include "mediastreamer2/msutils.h"
static void completion_cb(void *user_data, int percentage){
fprintf(stdout,"%i %% completed\r",percentage);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment