add adaptive rate control and quality indicator (experimental)

NEWS

+mediastreamer2-2.8.0: XXXXX
+	* mac os X video support
+	* new experimental adaptive audio bitrate control api
+	* new call quality indicator api
+
 mediastreamer-2.7.3: March 28, 2011
 	* major rework of the Mac OS X Audio Unit sound filter (macsnd.c)
 	* compilation fix for FreeBSD

include/mediastreamer2/Makefile.am

@@ -31,7 +31,9 @@ mediastreamer2_include_HEADERS= ice.h \
 				msextdisplay.h \
 				msjpegwriter.h \
 				mstonedetector.h \
-				msjava.h
+				msjava.h \
+				bitratecontrol.h \
+				qualityindicator.h 
 
 EXTRA_DIST=$(mediastreamer2_include_HEADERS)
 

include/mediastreamer2/bitratecontrol.h

+/*
+mediastreamer2 library - modular sound and video processing and streaming
+
+ * Copyright (C) 2011  Belledonne Communications, Grenoble, France
+
+This program is free software; you can redistribute it and/or
+modify it under the terms of the GNU General Public License
+as published by the Free Software Foundation; either version 2
+of the License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+ */
+
+#ifndef ms2_ratecontrol
+#define ms2_ratecontrol
+
+#include "mediastreamer2/msfilter.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+
+/**
+ * Audio Bitrate controller object
+**/
+
+typedef struct _MSAudioBitrateController MSAudioBitrateController;
+
+#define MS_AUDIO_RATE_CONTROL_OPTIMIZE_QUALITY (1)
+#define MS_AUDIO_RATE_CONTROL_OPTIMIZE_LATENCY (1<<1)
+
+MSAudioBitrateController *ms_audio_bitrate_controller_new(RtpSession *session, MSFilter *encoder, unsigned int flags);
+
+void ms_audio_bitrate_controller_process_rtcp(MSAudioBitrateController *obj, mblk_t *rtcp);
+
+void ms_audio_bitrate_controller_destroy(MSAudioBitrateController *obj); 
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
+
+

include/mediastreamer2/mediastream.h

@@ -26,6 +26,8 @@ Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
 #include <mediastreamer2/mssndcard.h>
 #include <mediastreamer2/mswebcam.h>
 #include <mediastreamer2/msvideo.h>
+#include <mediastreamer2/bitratecontrol.h>
+#include <mediastreamer2/qualityindicator.h>
 #include <ortp/ortp.h>
 #include <ortp/event.h>
 
@@ -57,11 +59,14 @@ struct _AudioStream
 	time_t last_packet_time;
 	EchoLimiterType el_type; /*use echo limiter: two MSVolume, measured input level controlling local output level*/
 	OrtpEvQueue *evq;
+	MSAudioBitrateController *rc;
+	MSQualityIndicator *qi;
 	bool_t play_dtmfs;
 	bool_t use_gc;
 	bool_t use_agc;
 	bool_t eq_active;
 	bool_t use_ng;/*noise gate*/
+	bool_t use_rc;
 };
 
 #ifdef __cplusplus
@@ -114,6 +119,9 @@ MS2_PUBLIC void audio_stream_set_relay_session_id(AudioStream *stream, const cha
 /*returns true if we are still receiving some data from remote end in the last timeout seconds*/
 MS2_PUBLIC bool_t audio_stream_alive(AudioStream * stream, int timeout);
 
+/*execute background tasks related to audio processing*/
+MS2_PUBLIC void audio_stream_iterate(AudioStream *stream);
+
 /*enable echo-limiter dispositve: one MSVolume in input branch controls a MSVolume in the output branch*/
 MS2_PUBLIC void audio_stream_enable_echo_limiter(AudioStream *stream, EchoLimiterType type);
 
@@ -126,6 +134,10 @@ MS2_PUBLIC void audio_stream_enable_automatic_gain_control(AudioStream *stream,
 /*to be done before start */
 MS2_PUBLIC void audio_stream_set_echo_canceller_params(AudioStream *st, int tail_len_ms, int delay_ms, int framesize);
 
+/*enable rate control */
+MS2_PUBLIC void audio_stream_enable_bitrate_control(AudioStream *st, bool_t enabled);
+
+
 MS2_PUBLIC void audio_stream_set_mic_gain(AudioStream *stream, float gain);
 
 /* enable/disable rtp stream */ 
@@ -155,6 +167,9 @@ MS2_PUBLIC void audio_stream_set_default_card(int cardindex);
 /* retrieve RTP statistics*/
 MS2_PUBLIC void audio_stream_get_local_rtp_stats(AudioStream *stream, rtp_stats_t *stats);
 
+/* returns an indicator of the stream quality between 0 and 5 */
+MS2_PUBLIC float audio_stream_get_quality_rating(AudioStream *stream);
+
 
 /*****************
   Video Support

include/mediastreamer2/qualityindicator.h

+/*
+mediastreamer2 library - modular sound and video processing and streaming
+
+ * Copyright (C) 2011  Belledonne Communications, Grenoble, France
+
+This program is free software; you can redistribute it and/or
+modify it under the terms of the GNU General Public License
+as published by the Free Software Foundation; either version 2
+of the License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+*/
+
+#ifndef ms2_qualityindicator_h
+#define ms2_qualityindicator_h
+
+#include "mediastreamer2/mscommon.h"
+
+typedef struct _MSQualityIndicator MSQualityIndicator;
+
+#ifdef __cplusplus
+extern "C"{
+#endif
+
+MSQualityIndicator *ms_quality_indicator_new(RtpSession *session);
+
+float ms_quality_indicator_get_rating(MSQualityIndicator *qi);
+
+void ms_quality_indicator_update_from_feedback(MSQualityIndicator *qi, mblk_t *rtcp);
+
+void ms_quality_indicator_update_local(MSQualityIndicator *qi);
+
+void ms_quality_indicator_destroy(MSQualityIndicator *qi);
+
+#ifdef __cplusplus
+}
+#endif
+
+
+#endif
+

src/Makefile.am

@@ -48,7 +48,9 @@ libmediastreamer_la_SOURCES=	mscommon.c     \
 				chanadapt.c \
 				audiomixer.c \
 				itc.c \
-				tonedetector.c 
+				tonedetector.c \
+				bitratecontrol.c \
+				qualityindicator.c
 
 #dummy c++ file to force libtool to use c++ linking (because of msdscap-mingw.cc)
 nodist_EXTRA_libmediastreamer_la_SOURCES = dummy.cxx

src/audiostream.c

@@ -67,6 +67,8 @@ void audio_stream_free(AudioStream *stream)
 	if (stream->read_resampler!=NULL) ms_filter_destroy(stream->read_resampler);
 	if (stream->write_resampler!=NULL) ms_filter_destroy(stream->write_resampler);
 	if (stream->dtmfgen_rtp!=NULL) ms_filter_destroy(stream->dtmfgen_rtp);
+	if (stream->rc) ms_audio_bitrate_controller_destroy(stream->rc);
+	if (stream->qi) ms_quality_indicator_destroy(stream->qi);
 	ms_free(stream);
 }
 
@@ -146,44 +148,52 @@ ms_time (time_t *t)
 
 static void audio_stream_process_rtcp(AudioStream *stream, mblk_t *m){
 	do{
+		const report_block_t *rb=NULL;
 		if (rtcp_is_SR(m)){
-			const report_block_t *rb;
 			rb=rtcp_SR_get_report_block(m,0);
-			if (rb){
-				unsigned int ij;
-				float rt=rtp_session_get_round_trip_propagation(stream->session);
-				float flost;
-				ij=report_block_get_interarrival_jitter(rb);
-				flost=(float)(100.0*report_block_get_fraction_lost(rb)/256.0);
-				ms_message("audio_stream_process_rtcp: interarrival jitter=%u , "
-				           "lost packets percentage since last report=%f, round trip time=%f seconds",ij,flost,rt);
-			}
+		}else if (rtcp_is_RR(m)){
+			rb=rtcp_RR_get_report_block(m,0);
+		}
+		if (rb){
+			unsigned int ij;
+			float rt=rtp_session_get_round_trip_propagation(stream->session);
+			float flost;
+			ij=report_block_get_interarrival_jitter(rb);
+			flost=(float)(100.0*report_block_get_fraction_lost(rb)/256.0);
+			ms_message("audio_stream_process_rtcp: interarrival jitter=%u , "
+			           "lost packets percentage since last report=%f, round trip time=%f seconds",ij,flost,rt);
+			if (stream->rc) ms_audio_bitrate_controller_process_rtcp(stream->rc,m);
+			if (stream->qi) ms_quality_indicator_update_from_feedback(stream->qi,m);
 		}
 	}while(rtcp_next_packet(m));
 }
 
-bool_t audio_stream_alive(AudioStream * stream, int timeout){
-	RtpSession *session=stream->session;
-	const rtp_stats_t *stats=rtp_session_get_stats(session);
-	if (stats->recv!=0){
-		if (stream->evq){
-			OrtpEvent *ev=ortp_ev_queue_get(stream->evq);
-			if (ev!=NULL){
-				if (ortp_event_get_type(ev)==ORTP_EVENT_RTCP_PACKET_RECEIVED){
-					audio_stream_process_rtcp(stream,ortp_event_get_data(ev)->packet);
-					stream->last_packet_time=ms_time(NULL);
-				}
-				ortp_event_destroy(ev);
+void audio_stream_iterate(AudioStream *stream){
+	
+	if (stream->evq){
+		OrtpEvent *ev=ortp_ev_queue_get(stream->evq);
+		if (ev!=NULL){
+			if (ortp_event_get_type(ev)==ORTP_EVENT_RTCP_PACKET_RECEIVED){
+				audio_stream_process_rtcp(stream,ortp_event_get_data(ev)->packet);
+				stream->last_packet_time=ms_time(NULL);
 			}
+			ortp_event_destroy(ev);
 		}
+	}
+}
+
+bool_t audio_stream_alive(AudioStream * stream, int timeout){
+	const rtp_stats_t *stats=rtp_session_get_stats(stream->session);
+	if (stats->recv!=0){
 		if (stats->recv!=stream->last_packet_count){
 			stream->last_packet_count=stats->recv;
 			stream->last_packet_time=ms_time(NULL);
-		}else{
-			if (ms_time(NULL)-stream->last_packet_time>timeout){
-				/* more than timeout seconds of inactivity*/
-				return FALSE;
-			}
+		}
+	}
+	if (stats->recv!=0){
+		if (ms_time(NULL)-stream->last_packet_time>timeout){
+			/* more than timeout seconds of inactivity*/
+			return FALSE;
 		}
 	}
 	return TRUE;
@@ -353,6 +363,12 @@ int audio_stream_start_full(AudioStream *stream, RtpProfile *profile, const char
 	if (stream->write_resampler){
 		audio_stream_configure_resampler(stream->write_resampler,stream->rtprecv,stream->soundwrite);
 	}
+
+	if (stream->use_rc){
+		stream->rc=ms_audio_bitrate_controller_new(stream->session,stream->encoder,0);
+	}
+	stream->qi=ms_quality_indicator_new(stream->session);
+	
 	/* and then connect all */
 	/* tip: draw yourself the picture if you don't understand */
 
@@ -394,6 +410,9 @@ int audio_stream_start_full(AudioStream *stream, RtpProfile *profile, const char
 	return 0;
 }
 
+void audio_stream_enable_bitrate_control(AudioStream *st, bool_t enabled){
+	st->use_rc=enabled;
+}
 
 int audio_stream_start_with_files(AudioStream *stream, RtpProfile *prof,const char *remip, int remport,
 	int rem_rtcp_port, int pt,int jitt_comp, const char *infile, const char * outfile)
@@ -716,10 +735,18 @@ void audio_stream_get_local_rtp_stats(AudioStream *stream, rtp_stats_t *lstats){
 
 void audio_stream_mute_rtp(AudioStream *stream, bool_t val) 
 {
-  if (stream->rtpsend){
-    if (val)
-      ms_filter_call_method(stream->rtpsend,MS_RTP_SEND_MUTE_MIC,&val);
-    else
-      ms_filter_call_method(stream->rtpsend,MS_RTP_SEND_UNMUTE_MIC,&val);
-  }
+	if (stream->rtpsend){
+		if (val)
+			ms_filter_call_method(stream->rtpsend,MS_RTP_SEND_MUTE_MIC,&val);
+		else
+			ms_filter_call_method(stream->rtpsend,MS_RTP_SEND_UNMUTE_MIC,&val);
+	}
+}
+
+float audio_stream_get_quality_rating(AudioStream *stream){
+	if (stream->qi){
+		ms_quality_indicator_update_local(stream->qi);
+		return ms_quality_indicator_get_rating(stream->qi);
+	}
+	return 0;
 }

src/bitratecontrol.c

+/*
+mediastreamer2 library - modular sound and video processing and streaming
+
+ * Copyright (C) 2011  Belledonne Communications, Grenoble, France
+
+This program is free software; you can redistribute it and/or
+modify it under the terms of the GNU General Public License
+as published by the Free Software Foundation; either version 2
+of the License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+ */
+
+#include "mediastreamer2/bitratecontrol.h"
+
+#define STATS_HISTORY 3
+
+static const float unacceptable_loss_rate=20;
+static const int big_jitter=40; /*ms */
+static const float significant_delay=0.2; /*seconds*/
+static const int max_ptime=100;
+
+enum state_t{
+	Init,
+	Probing,
+	Stable,
+	ProbingUp
+};
+
+const char *state_name(enum state_t st){
+	switch(st){
+		case Init: return "Init";
+		case Probing: return "Probing";
+		case Stable: return "Stable";
+		case ProbingUp: return "ProbingUp";
+	}
+	return "bad state";
+}
+
+typedef struct rtpstats{
+	uint64_t high_seq_recv; /*highest sequence number received*/
+	float lost_percentage; /*percentage of lost packet since last report*/
+	float int_jitter; /*interrarrival jitter */
+	float rt_prop; /*round trip propagation*/
+}rtpstats_t;
+
+enum action_type{
+	DoNothing,
+	DecreaseBitrate,
+	DecreasePacketRate,
+	IncreaseQuality
+};
+
+static const char *action_type_name(enum action_type t){
+	switch(t){
+		case DoNothing:
+			return "DoNothing";
+		case IncreaseQuality:
+			return "IncreaseQuality";
+		case DecreaseBitrate:
+			return "DecreaseBitrate";
+		case DecreasePacketRate:
+			return "DecreasePacketRate";
+	}
+	return "bad action type";
+}
+
+typedef struct action{
+	enum action_type type;
+	int value;
+}action_t;
+
+struct _MSAudioBitrateController{
+	RtpSession *session;
+	MSFilter *encoder;
+	int clockrate;
+	rtpstats_t stats[STATS_HISTORY];
+	int curindex;
+	enum state_t state;
+	int min_ptime;
+	int nom_bitrate;
+	int cur_ptime;
+	int cur_bitrate;
+	int stable_count;
+};
+
+MSAudioBitrateController *ms_audio_bitrate_controller_new(RtpSession *session, MSFilter *encoder, unsigned int flags){
+	MSAudioBitrateController *rc=ms_new0(MSAudioBitrateController,1);
+	rc->session=session;
+	rc->encoder=encoder;
+	rc->cur_ptime=rc->min_ptime=20;
+	rc->cur_bitrate=rc->nom_bitrate=0;
+	if (ms_filter_call_method(encoder,MS_FILTER_GET_BITRATE,&rc->nom_bitrate)!=0){
+		ms_message("Encoder has nominal bitrate %i",rc->nom_bitrate);
+	}	
+	rc->cur_bitrate=rc->nom_bitrate;
+	return rc;
+}
+
+static bool_t rt_prop_doubled(rtpstats_t *cur,rtpstats_t *prev){
+	//ms_message("AudioBitrateController: cur=%f, prev=%f",cur->rt_prop,prev->rt_prop);
+	if (cur->rt_prop>=significant_delay && prev->rt_prop>0){	
+		if (cur->rt_prop>=(prev->rt_prop*2.0)){
+			/*propagation doubled since last report */
+			return TRUE;
+		}
+	}
+	return FALSE;
+}
+
+static bool_t rt_prop_increased(MSAudioBitrateController *obj){
+	rtpstats_t *cur=&obj->stats[obj->curindex % STATS_HISTORY];
+	rtpstats_t *prev=&obj->stats[(STATS_HISTORY+obj->curindex-1) % STATS_HISTORY];
+
+	if (rt_prop_doubled(cur,prev)){
+		return TRUE;
+	}
+	return FALSE;
+}
+
+static void analyse_quality(MSAudioBitrateController *obj, action_t *action){
+	rtpstats_t *cur=&obj->stats[obj->curindex % STATS_HISTORY];
+	/*big losses and big jitter */
+	if (cur->lost_percentage>=unacceptable_loss_rate && cur->int_jitter>=big_jitter){
+		action->type=DecreaseBitrate;
+		action->value=MIN(cur->lost_percentage,50);
+		ms_message("AudioBitrateController: analyse - loss rate unacceptable and big jitter");
+	}else if (rt_prop_increased(obj)){
+		action->type=DecreaseBitrate;
+		action->value=20;
+		ms_message("AudioBitrateController: analyse - rt_prop doubled.");
+	}else if (cur->lost_percentage>=unacceptable_loss_rate){
+		/*big loss rate but no jitter, and no big rtp_prop: pure lossy network*/
+		action->type=DecreasePacketRate;
+		ms_message("AudioBitrateController: analyse - loss rate unacceptable.");
+	}else{
+		action->type=DoNothing;
+		ms_message("AudioBitrateController: analyse - everything is fine.");
+	}
+}
+
+static bool_t has_improved(MSAudioBitrateController *obj){
+	rtpstats_t *cur=&obj->stats[obj->curindex % STATS_HISTORY];
+	rtpstats_t *prev=&obj->stats[(STATS_HISTORY+obj->curindex-1) % STATS_HISTORY];
+	rtpstats_t *prev2=&obj->stats[(STATS_HISTORY+obj->curindex-2) % STATS_HISTORY];
+
+	if (prev->lost_percentage>=unacceptable_loss_rate){
+		if (cur->lost_percentage<prev->lost_percentage){
+			ms_message("AudioBitrateController: lost percentage has improved");
+			return TRUE;
+		}else goto end;
+	}
+	if (rt_prop_doubled(prev,prev2) && cur->rt_prop<prev->rt_prop){
+		ms_message("AudioBitrateController: rt prop decrased");
+		return TRUE;
+	}
+
+end:
+	ms_message("AudioBitrateController: no improvements.");
+	
+	return FALSE;
+}
+
+static void apply_ptime(MSAudioBitrateController *obj){
+	char tmp[64];
+	snprintf(tmp,sizeof(tmp),"ptime=%i",obj->cur_ptime);
+	if (ms_filter_call_method(obj->encoder,MS_FILTER_ADD_FMTP,tmp)!=0){
+		ms_message("AudioBitrateController: failed ptime command.");
+	}else ms_message("AudioBitrateController: ptime changed to %i",obj->cur_ptime);
+}
+
+static void inc_ptime(MSAudioBitrateController *obj){
+	if (obj->cur_ptime>=max_ptime){
+		ms_message("AudioBitrateController: maximum ptime reached");
+		return;
+	}
+	obj->cur_ptime+=obj->min_ptime;
+	apply_ptime(obj);
+}
+
+static int execute_action(MSAudioBitrateController *obj, action_t *action){
+	ms_message("AudioBitrateController: executing action of type %s, value=%i",action_type_name(action->type),action->value);
+	if (action->type==DecreaseBitrate){
+		if (obj->nom_bitrate==0){
+			/*not a vbr codec*/
+			inc_ptime(obj);
+		}else{
+			int cur_br=0;
+			int new_br;
+			if (ms_filter_call_method(obj->encoder,MS_FILTER_GET_BITRATE,&cur_br)!=0){
+				ms_message("AudioBitrateController: GET_BITRATE failed");
+				inc_ptime(obj);
+				return 0;
+			}
+			new_br=cur_br-((cur_br*action->value)/100);
+		
+			ms_message("AudioBitrateController: Attempting to reduce audio bitrate to %i",new_br);
+			if (ms_filter_call_method(obj->encoder,MS_FILTER_SET_BITRATE,&new_br)!=0){
+				ms_message("AudioBitrateController: SET_BITRATE failed");
+				inc_ptime(obj);
+				return 0;
+			}
+			new_br=0;
+			ms_filter_call_method(obj->encoder,MS_FILTER_GET_BITRATE,&new_br);
+			ms_message("AudioBitrateController: bitrate actually set to %i");
+		}
+	}else if (action->type==DecreasePacketRate){
+		inc_ptime(obj);
+	}else if (action->type==IncreaseQuality){
+		if (obj->cur_ptime>obj->min_ptime){
+			obj->cur_ptime-=obj->min_ptime;
+			apply_ptime(obj);
+		}else return -1;
+	}
+	return 0;
+}
+
+static void state_machine(MSAudioBitrateController *obj){
+	action_t action;
+	switch(obj->state){
+		case Stable:
+			obj->stable_count++;
+		case Init:
+			analyse_quality(obj,&action);
+			if (action.type!=DoNothing){
+				execute_action(obj,&action);
+				obj->state=Probing;
+			}else if (obj->stable_count>=5){
+				action.type=IncreaseQuality;
+				execute_action(obj,&action);
+				obj->state=ProbingUp;
+			}
+		break;
+		case Probing:
+			obj->stable_count=0;
+			if (has_improved(obj)){
+				obj->state=Stable;
+			}else{
+				analyse_quality(obj,&action);
+				if (action.type!=DoNothing){
+					execute_action(obj,&action);
+				}
+			}
+		break;
+		case ProbingUp:
+			obj->stable_count=0;
+			analyse_quality(obj,&action);
+			if (action.type!=DoNothing){
+				execute_action(obj,&action);
+				obj->state=Probing;
+			}else{
+				/*continue*/
+				action.type=IncreaseQuality;
+				if (execute_action(obj,&action)==-1){
+					/* we reached the maximum*/
+					obj->state=Init;
+				}
+			}
+		break;
+		default:
+		break;
+	}
+	ms_message("AudioBitrateController: current state is %s",state_name(obj->state));
+}
+
+static void read_report(MSAudioBitrateController *obj, const report_block_t *rb){
+	rtpstats_t *cur;
+
+	obj->curindex++;
+	cur=&obj->stats[obj->curindex % STATS_HISTORY];
+	
+	if (obj->clockrate==0){
+		PayloadType *pt=rtp_profile_get_payload(rtp_session_get_send_profile(obj->session),rtp_session_get_send_payload_type(obj->session));
+		if (pt!=NULL) obj->clockrate=pt->clock_rate;
+		else return;
+	}
+	
+	cur->high_seq_recv=report_block_get_high_ext_seq(rb);
+	cur->lost_percentage=100.0*(float)report_block_get_fraction_lost(rb)/256.0;