reorganize source code, add new vad-dtx filter for generic confort noise

build/android/Android.mk

@@ -76,6 +76,7 @@ LOCAL_SRC_FILES = \
 	utils/g722_encode.c \
 	utils/audiodiff.c \
 	utils/stream_regulator.c \
+	utils/extremum.c \
 	otherfilters/msrtp.c \
 	otherfilters/tee.c \
 	otherfilters/join.c \
@@ -96,6 +97,7 @@ LOCAL_SRC_FILES = \
 	audiofilters/msresample.c \
 	audiofilters/devices.c \
 	audiofilters/flowcontrol.c \
+	audiofilters/msvaddtx.c \
 	audiofilters/aac-eld-android.cpp \
 	android/hardware_echo_canceller.cpp \
 	android/androidsound_depr.cpp \

include/mediastreamer2/Makefile.am

@@ -46,7 +46,9 @@ mediastreamer2_include_HEADERS= ice.h \
 				zrtp.h \
 				stun.h \
 				stun_udp.h \
-				ms_srtp.h
+				ms_srtp.h \
+				msvaddtx.h \
+				msutils.h
 
 EXTRA_DIST=$(mediastreamer2_include_HEADERS)
 

include/mediastreamer2/allfilters.h

@@ -108,7 +108,7 @@ typedef enum MSFilterId{
 	MS_ANDROID_VIDEO_READ_ID,
 	MS_ANDROID_VIDEO_WRITE_ID,
 	MS_TONE_DETECTOR_ID,
-    MY_FILTER_ID,
+	MY_FILTER_ID,
 	MS_IOS_DISPLAY_ID,
 	MS_VP8_ENC_ID,
 	MS_VP8_DEC_ID,
@@ -143,7 +143,8 @@ typedef enum MSFilterId{
 	MS_QSA_READ_ID,
 	MS_QSA_WRITE_ID,
 	MS_MKV_RECORDER_ID,
-	MS_MKV_PLAYER_ID
+	MS_MKV_PLAYER_ID,
+	MS_VAD_DTX_ID
 } MSFilterId;
 
 

include/mediastreamer2/dsptools.h

@@ -382,10 +382,6 @@ void ms_ifft(void *table, ms_word16_t *in, ms_word16_t *out);
 void ms_fir_mem16(const ms_word16_t *x, const ms_coef_t *num, ms_word16_t *y, int N, int ord, ms_mem_t *mem);
 
 
-typedef void (*MSAudioDiffProgressNotify)(void* user_data, int percentage);
-/*utility function to check similarity between two audio wav files*/
-int ms_audio_diff(const char *file1, const char *file2, double *ret, MSAudioDiffProgressNotify func, void *user_data);
-
 #ifdef __cplusplus
 }
 #endif

include/mediastreamer2/mediastream.h

@@ -313,6 +313,7 @@ struct _AudioStream
 		int videopin;
 		bool_t plumbed;
 	}av_player;
+	MSFilter *vaddtx;
 	char *recorder_file;
 	EchoLimiterType el_type; /*use echo limiter: two MSVolume, measured input level controlling local output level*/
 	EqualizerLocation eq_loc;

include/mediastreamer2/msfactory.h

 /*
+mediastreamer2 library - modular sound and video processing and streaming
+Copyright (C) 2014  Belledonne Communications SARL
+
 This program is free software; you can redistribute it and/or
 modify it under the terms of the GNU General Public License
 as published by the Free Software Foundation; either version 2

include/mediastreamer2/msutils.h

+/*
+mediastreamer2 library - modular sound and video processing and streaming
+Copyright (C) 2014  Belledonne Communications SARL
+
+This program is free software; you can redistribute it and/or
+modify it under the terms of the GNU General Public License
+as published by the Free Software Foundation; either version 2
+of the License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+*/
+
+#ifndef msutils_h
+#define msutils_h
+
+#include "mediastreamer2/mscommon.h"
+
+
+#ifdef __cplusplus
+extern "C"{
+#endif
+
+typedef void (*MSAudioDiffProgressNotify)(void* user_data, int percentage);
+/*utility function to check similarity between two audio wav files*/
+MS2_PUBLIC int ms_audio_diff(const char *file1, const char *file2, double *ret, MSAudioDiffProgressNotify func, void *user_data);
+
+
+/*Utility object to determine a maximum or minimum (but not both at the same time), of a signal during a sliding period of time.
+**/
+typedef struct _MSExtremum{
+	float current_extremum;
+	uint64_t extremum_time;
+	float last_stable;
+	int period;
+}MSExtremum;
+
+MS2_PUBLIC void ms_extremum_reset(MSExtremum *obj);
+
+MS2_PUBLIC void ms_extremum_init(MSExtremum *obj, int period);
+
+MS2_PUBLIC void ms_extremum_record_min(MSExtremum *obj, uint64_t curtime, float value);
+
+MS2_PUBLIC void ms_extremum_record_max(MSExtremum *obj, uint64_t curtime, float value);
+
+MS2_PUBLIC float ms_extremum_get_current(MSExtremum *obj);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
+

include/mediastreamer2/msvaddtx.h

+/*
+mediastreamer2 library - modular sound and video processing and streaming
+Copyright (C) 2014  Belledonne Communications SARL
+
+This program is free software; you can redistribute it and/or
+modify it under the terms of the GNU General Public License
+as published by the Free Software Foundation; either version 2
+of the License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+*/
+
+#include "mediastreamer2/msfilter.h"
+
+typedef struct _MSCngData{
+	int datasize;
+	uint8_t data[32];
+}MSCngData;
+
+/** Event generated when silence is detected. Payload contains the data encoding the background noise*/
+#define MS_VAD_DTX_NO_VOICE	MS_FILTER_EVENT(MS_VAD_DTX_ID, 0, MSCngData)
+

src/Makefile.am

@@ -128,6 +128,7 @@ libmediastreamer_voip_la_SOURCES+=	audiofilters/alaw.c \
 					utils/kiss_fftr.c \
 					utils/kiss_fftr.h \
 					utils/audiodiff.c \
+					utils/extremum.c \
 					audiofilters/equalizer.c \
 					audiofilters/chanadapt.c \
 					audiofilters/audiomixer.c \
@@ -142,7 +143,8 @@ libmediastreamer_voip_la_SOURCES+=	audiofilters/alaw.c \
 					audiofilters/msfileplayer.c \
 					audiofilters/msfilerec.c \
 					audiofilters/waveheader.h \
-					audiofilters/flowcontrol.c 
+					audiofilters/flowcontrol.c \
+					audiofilters/msvaddtx.c
 
 if BUILD_SPEEX
 libmediastreamer_voip_la_SOURCES+=	audiofilters/msspeex.c audiofilters/speexec.c

src/audiofilters/msopus.c

@@ -22,7 +22,6 @@ Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
 
 #include "mediastreamer2/msfilter.h"
 #include "mediastreamer2/mscodecutils.h"
-#include "mediastreamer2/msfilter.h"
 #include "mediastreamer2/msticker.h"
 #include "ortp/rtp.h"
 

src/audiofilters/msvaddtx.c

+/*
+msvaddtx.c - Generic VAD/DTX (voice activity detector, discontinuous transmission) 
+	for CN payload type (RFC3389)
+
+mediastreamer2 library - modular sound and video processing and streaming
+Copyright (C) 2011-2012 Belledonne Communications, Grenoble, France
+
+This program is free software; you can redistribute it and/or
+modify it under the terms of the GNU General Public License
+as published by the Free Software Foundation; either version 2
+of the License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+*/
+
+
+#include "mediastreamer2/msfilter.h"
+#include "mediastreamer2/msticker.h"
+#include "mediastreamer2/msutils.h"
+#include "mediastreamer2/msvaddtx.h"
+
+#include <math.h>
+
+static const float max_e = (32768* 0.7);   /* 0.7 - is RMS factor */
+static const float coef = 0.2; /* floating averaging coeff. for energy */
+static const float silence_threshold=0.001;
+
+typedef struct _VadDtxContext{
+	int silence_mode;/*set to 1 if a silence period is running*/
+#ifndef HAVE_G729B
+	float energy;
+	MSExtremum max;
+#else
+#endif
+}VadDtxContext;
+
+
+static void vad_dtx_init(MSFilter *f){
+	VadDtxContext *ctx=ms_new0(VadDtxContext,1);
+	f->data=ctx;
+	ms_extremum_init(&ctx->max,2000);
+}
+
+static void vad_dtx_preprocess(MSFilter *f){
+	VadDtxContext *ctx=(VadDtxContext*)f->data;
+	ms_extremum_reset(&ctx->max);
+	
+}
+
+static void update_energy(VadDtxContext *v, int16_t *signal, int numsamples, uint64_t curtime) {
+	int i;
+	float acc = 0;
+	float en;
+		
+	for (i=0;i<numsamples;++i){
+		int s=signal[i];
+		acc += s * s;
+	}
+	en = (sqrt(acc / numsamples)+1) / max_e;
+	v->energy = (en * coef) + v->energy * (1.0 - coef);
+	ms_extremum_record_max(&v->max,curtime,v->energy);
+}
+
+static void vad_dtx_process(MSFilter *f){
+	VadDtxContext *ctx=(VadDtxContext*)f->data;
+	mblk_t *m;
+	
+	while((m=ms_queue_get(f->inputs[0]))!=NULL){
+		update_energy(ctx,(int16_t*)m->b_rptr, (m->b_wptr - m->b_rptr) / 2, f->ticker->time);
+		
+		if (ms_extremum_get_current(&ctx->max)<silence_threshold){
+			if (!ctx->silence_mode){
+				MSCngData cngdata={0};
+				cngdata.datasize=1; /*only noise level*/
+				cngdata.data[0]=0; /*noise level set to zero for the moment*/
+				ms_message("vad_dtx_process(): silence period detected.");
+				ctx->silence_mode=1;
+				ms_filter_notify(f, MS_VAD_DTX_NO_VOICE, &cngdata);
+			}
+		}else{
+			if (ctx->silence_mode){
+				ms_message("vad_dtx_process(): silence period finished.");
+				ctx->silence_mode=1;
+			}
+		}
+		if (!ctx->silence_mode) ms_queue_put(f->outputs[0],m);
+	}
+	
+}
+
+static void vad_dtx_postprocess(MSFilter *f){
+	//VadDtxContext *ctx=(VadDtxContext*)f->data;
+	
+}
+
+static void vad_dtx_uninit(MSFilter *f){
+	VadDtxContext *ctx=(VadDtxContext*)f->data;
+	ms_free(ctx);
+}
+
+#ifndef _MSC_VER
+
+MSFilterDesc ms_vad_dtx_desc = {
+	.id = MS_VAD_DTX_ID,
+	.name = "MSVadDtx",
+	.text = "A filter detecting silence period and encoding residual noise",
+	.category = MS_FILTER_OTHER,
+	.ninputs = 1,
+	.noutputs = 1,
+	.init = vad_dtx_init,
+	.preprocess = vad_dtx_preprocess,
+	.process = vad_dtx_process,
+	.postprocess = vad_dtx_postprocess,
+	.uninit = vad_dtx_uninit,
+};
+
+#else
+
+MSFilterDesc ms_vad_dtx_desc = {
+	MS_VAD_DTX_ID,
+	"MSVadDtx",
+	"A filter detecting silence period and encoding residual noise",
+	MS_FILTER_OTHER,
+	NULL,
+	1,
+	1,
+	vad_dtx_init,
+	vad_dtx_preprocess,
+	vad_dtx_process,
+	vad_dtx_postprocess,
+	vad_dtx_uninit
+};
+
+#endif
+
+MS_FILTER_DESC_EXPORT(ms_vad_dtx_desc)
+

src/audiofilters/msvolume.c

@@ -23,6 +23,7 @@ Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
 
 #include "mediastreamer2/msvolume.h"
 #include "mediastreamer2/msticker.h"
+#include "mediastreamer2/msutils.h"
 #include <math.h>
 
 #ifdef HAVE_SPEEXDSP
@@ -40,75 +41,13 @@ static const float transmit_thres=4;
 static const float min_ng_floorgain=0.005;
 static const float agc_threshold=0.5;
 
-typedef struct Extremum{
-	float current_extremum;
-	uint64_t extremum_time;
-	float last_stable;
-	int period;
-}Extremum;
-
-static void extremum_reset(Extremum *obj){
-	obj->current_extremum=0;
-	obj->extremum_time=(uint64_t)-1;
-	obj->last_stable=0;
-}
-
-static void extremum_init(Extremum *obj, int period){
-	extremum_reset(obj);
-	obj->period=period;
-}
-
-static void extremum_set_new(Extremum *obj, const char* kind){
-	obj->last_stable=obj->current_extremum;
-	/*ms_message("New %s value : %f",kind,obj->last_stable);*/
-}
-
-static void extremum_check_init(Extremum *obj, uint64_t curtime, float value, const char *kind){
-	if (obj->extremum_time!=(uint64_t)-1){
-		if (curtime-obj->extremum_time>obj->period){
-			/*last extremum is too old, drop it*/
-			extremum_set_new(obj,kind);
-			obj->extremum_time=(uint64_t)-1;
-		}
-	}
-	if (obj->extremum_time==(uint64_t)-1){
-		obj->current_extremum=value;
-		obj->extremum_time=curtime;
-	}
-}
-
-static void extremum_record_min(Extremum *obj, uint64_t curtime, float value){
-	extremum_check_init(obj,curtime,value,"min");
-	if (value<obj->current_extremum){
-		obj->current_extremum=value;
-		obj->extremum_time=curtime;
-		if (value<obj->last_stable){
-			extremum_set_new(obj,"min");
-		}
-	}
-}
-
-static void extremum_record_max(Extremum *obj, uint64_t curtime, float value){
-	extremum_check_init(obj,curtime,value,"max");
-	if (value>obj->current_extremum){
-		obj->current_extremum=value;
-		obj->extremum_time=curtime;
-		if (value>obj->last_stable){
-			extremum_set_new(obj,"max");
-		}
-	}
-}
-
-static float extremum_get_current(Extremum *obj){
-	return obj->last_stable;
-}
 
 typedef struct Volume{
 	float energy;
 	float level_pk;
 	float instant_energy;
 	float lt_speaker_en;
-	float gain; 				/**< the one really applied, smoothed target_gain version*/
+	float gain; 		/**< the one really applied, smoothed target_gain version*/
 	float static_gain;	/**< the one fixed by the user */
 	int dc_offset;
 	//float gain_k;
@@ -133,8 +72,8 @@ typedef struct Volume{
 	float ng_floorgain;
 	float ng_gain;
 	MSBufferizer *buffer;
-	Extremum min;
-	Extremum max;
+	MSExtremum min;
+	MSExtremum max;
 	bool_t agc_enabled;
 	bool_t noise_gate_enabled;
 	bool_t remove_dc;
@@ -170,8 +109,8 @@ static void volume_init(MSFilter *f){
 #ifdef HAVE_SPEEXDSP
 	v->speex_pp=NULL;
 #endif
-	extremum_init(&v->max,1000);
-	extremum_init(&v->min,30000);
+	ms_extremum_init(&v->max,1000);
+	ms_extremum_init(&v->min,30000);
 	f->data=v;
 }
 
@@ -200,14 +139,14 @@ static int volume_get(MSFilter *f, void *arg){
 static int volume_get_min(MSFilter *f, void *arg){
 	float *farg=(float*)arg;
 	Volume *v=(Volume*)f->data;
-	*farg=linear_to_db(extremum_get_current(&v->min));
+	*farg=linear_to_db(ms_extremum_get_current(&v->min));
 	return 0;
 }
 
 static int volume_get_max(MSFilter *f, void *arg){
 	float *farg=(float*)arg;
 	Volume *v=(Volume*)f->data;
-	*farg=linear_to_db(extremum_get_current(&v->max));
+	*farg=linear_to_db(ms_extremum_get_current(&v->max));
 	return 0;
 }
 
@@ -463,8 +402,8 @@ static void update_energy(Volume *v, int16_t *signal, int numsamples, uint64_t c
 	v->energy = (en * coef) + v->energy * (1.0 - coef);
 	v->level_pk = (float)pk / max_e;
 	v->instant_energy = en;// currently non-averaged energy seems better (short artefacts)
-	extremum_record_max(&v->max,curtime,v->energy);
-	extremum_record_min(&v->min,curtime,v->energy);
+	ms_extremum_record_max(&v->max,curtime,v->energy);
+	ms_extremum_record_min(&v->min,curtime,v->energy);
 }
 
 static void apply_gain(Volume *v, mblk_t *m, float tgain) {
@@ -532,8 +471,8 @@ static void volume_preprocess(MSFilter *f){
 		}
 #endif
 	}
-	extremum_reset(&v->min);
-	extremum_reset(&v->max);
+	ms_extremum_reset(&v->min);
+	ms_extremum_reset(&v->max);
 }
 
 static void volume_process(MSFilter *f){

src/base/msfactory.c

@@ -655,7 +655,7 @@ struct _MSEventQueue *ms_factory_get_event_queue(MSFactory *obj){
 	return obj->evq;
 }
 
-/*this function is for compatibility, when event queues where created by the application*/
+/*this function is for compatibility, when event queues were created by the application*/
 void ms_factory_set_event_queue(MSFactory *obj, MSEventQueue *evq){
 	obj->evq=evq;
 }

src/utils/audiodiff.c

@@ -19,7 +19,7 @@ Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
 */
 
 #include "mediastreamer2/mscommon.h"
-#include "mediastreamer2/dsptools.h"
+#include "mediastreamer2/msutils.h"
 #include "waveheader.h"
 
 #include <math.h>

src/utils/extremum.c

+/*
+mediastreamer2 library - modular sound and video processing and streaming
+Copyright (C) 2014 Belledonne Communications SARL
+Author: Simon MORLAT (simon.morlat@linphone.org)
+
+This program is free software; you can redistribute it and/or
+modify it under the terms of the GNU General Public License
+as published by the Free Software Foundation; either version 2
+of the License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+*/
+
+#include "mediastreamer2/msutils.h"
+
+void ms_extremum_reset(MSExtremum *obj){
+	obj->current_extremum=0;
+	obj->extremum_time=(uint64_t)-1;
+	obj->last_stable=0;
+}
+
+void ms_extremum_init(MSExtremum *obj, int period){
+	ms_extremum_reset(obj);
+	obj->period=period;
+}
+
+static void extremum_set_new(MSExtremum *obj, const char* kind){
+	obj->last_stable=obj->current_extremum;
+	/*ms_message("New %s value : %f",kind,obj->last_stable);*/
+}
+
+static void extremum_check_init(MSExtremum *obj, uint64_t curtime, float value, const char *kind){
+	if (obj->extremum_time!=(uint64_t)-1){
+		if (curtime-obj->extremum_time>obj->period){
+			/*last extremum is too old, drop it*/
+			extremum_set_new(obj,kind);
+			obj->extremum_time=(uint64_t)-1;
+		}
+	}
+	if (obj->extremum_time==(uint64_t)-1){
+		obj->current_extremum=value;
+		obj->extremum_time=curtime;
+	}
+}
+
+void ms_extremum_record_min(MSExtremum *obj, uint64_t curtime, float value){
+	extremum_check_init(obj,curtime,value,"min");
+	if (value<obj->current_extremum){
+		obj->current_extremum=value;
+		obj->extremum_time=curtime;
+		if (value<obj->last_stable){
+			extremum_set_new(obj,"min");
+		}
+	}
+}
+
+void ms_extremum_record_max(MSExtremum *obj, uint64_t curtime, float value){
+	extremum_check_init(obj,curtime,value,"max");
+	if (value>obj->current_extremum){
+		obj->current_extremum=value;
+		obj->extremum_time=curtime;
+		if (value>obj->last_stable){
+			extremum_set_new(obj,"max");
+		}
+	}
+}
+
+float ms_extremum_get_current(MSExtremum *obj){
+	return obj->last_stable;
+}
+

src/voip/audiostream.c

@@ -71,6 +71,7 @@ static void audio_stream_free(AudioStream *stream) {
 	if (stream->av_recorder.recorder) ms_filter_destroy(stream->av_recorder.recorder);
 	if (stream->av_recorder.resampler) ms_filter_destroy(stream->av_recorder.resampler);
 	if (stream->av_recorder.video_input) ms_filter_destroy(stream->av_recorder.video_input);
+	if (stream->vaddtx) ms_filter_destroy(stream->vaddtx);
 	if (stream->outbound_mixer) ms_filter_destroy(stream->outbound_mixer);
 	if (stream->recorder_file) ms_free(stream->recorder_file);
 
@@ -586,6 +587,22 @@ static void setup_recorder(AudioStream *stream, int sample_rate, int nchannels){
 	setup_av_recorder(stream,sample_rate,nchannels);
 }
 
+static void on_silence_detected(void *data, MSFilter *f, unsigned int event_id, void *event_arg){
+	ms_message("on_silence_detected(): CN packet to be sent !");
+}
+
+static void setup_generic_confort_noise(AudioStream *stream){
+	RtpProfile *prof=rtp_session_get_profile(stream->ms.sessions.rtp_session);
+	PayloadType *pt=rtp_profile_get_payload(prof, rtp_session_get_send_payload_type(stream->ms.sessions.rtp_session));
+	PayloadType *cn=rtp_profile_find_payload(prof, "CN", 8000, 1);
+	
+	if (cn && pt && pt->channels==1 && pt->clock_rate==8000){
+		/* RFC3389 CN can be used*/
+		stream->vaddtx=ms_filter_new(MS_VAD_DTX_ID);
+		ms_filter_add_notify_callback(stream->vaddtx, on_silence_detected, stream, TRUE);
+	}
+}
+
 int audio_stream_start_full(AudioStream *stream, RtpProfile *profile, const char *rem_rtp_ip,int rem_rtp_port,
 	const char *rem_rtcp_ip, int rem_rtcp_port, int payload,int jitt_comp, const char *infile, const char *outfile,
 	MSSndCard *playcard, MSSndCard *captcard, bool_t use_ec){
@@ -855,6 +872,8 @@ int audio_stream_start_full(AudioStream *stream, RtpProfile *profile, const char
 			ms_filter_call_method(stream->plc, MS_FILTER_SET_NCHANNELS, &nchannels);
 			ms_filter_call_method(stream->plc, MS_FILTER_SET_SAMPLE_RATE, &sample_rate);
 		}
+		/*as first rough approximation, a codec without PLC capabilities has no VAD/DTX builtin, thus setup generic confort noise if possible*/
+		setup_generic_confort_noise(stream);
 	} else {