implement RTCP based rate control for mpeg and vp8

include/mediastreamer2/bitratecontrol.h

@@ -38,7 +38,7 @@ enum _MSRateControlActionType{
 	MSRateControlActionDoNothing,
 	MSRateControlActionDecreaseBitrate,
 	MSRateControlActionDecreasePacketRate,
-	MSRateControlActionIncreaseQuality
+	MSRateControlActionIncreaseQuality,
 };
 
 typedef enum _MSRateControlActionType MSRateControlActionType;
@@ -72,7 +72,7 @@ MSBitrateDriver * ms_bitrate_driver_ref(MSBitrateDriver *obj);
 void ms_bitrate_driver_unref(MSBitrateDriver *obj);
 
 MSBitrateDriver *ms_audio_bitrate_driver_new(MSFilter *encoder);
-
+MSBitrateDriver *ms_av_bitrate_driver_new(MSFilter *a_encoder, MSFilter *venc);
 
 typedef struct _MSQosAnalyser MSQosAnalyser;
 typedef struct _MSQosAnalyserDesc MSQosAnalyserDesc;
@@ -100,10 +100,15 @@ bool_t ms_qos_analyser_has_improved(MSQosAnalyser *obj);
 bool_t ms_qos_analyser_process_rtcp(MSQosAnalyser *obj, mblk_t *rtcp);
 
 /**
- * The simple qos analyzer is an implementation of MSQosAnalizer that performs analysis for single stream.
+ * The simple qos analyzer is an implementation of MSQosAnalyser that performs analysis for single stream.
 **/
 MSQosAnalyser * ms_simple_qos_analyser_new(RtpSession *session);
 
+/**
+ * The audio/video qos analyser is an implementation of MSQosAnalyser that performs analysis of two audio and video streams.
+**/
+MSQosAnalyser * ms_av_qos_analyser_new(RtpSession *asession, RtpSession *vsession);
+
 /**
  * The MSBitrateController the overall behavior and state machine of the adaptive rate control system.
  * It requires a MSQosAnalyser to obtain analyse of the quality of service, and a MSBitrateDriver
@@ -151,6 +156,19 @@ void ms_bitrate_controller_destroy(MSBitrateController *obj);
 **/
 MSBitrateController *ms_audio_bitrate_controller_new(RtpSession *session, MSFilter *encoder, unsigned int flags);
 
+/**
+ * Convenience fonction to create a bitrate controller managing a video and an audio stream.
+ * @param vsession the video RtpSession
+ * @param venc the video encoder
+ * @param asession the audio RtpSession
+ * @param aenc the audio encoder
+ * This function actually calls internally:
+ * <br>
+ * \code
+ * ms_bitrate_controller_new(ms_av_qos_analyser_new(asession,vsession),ms_av_bitrate_driver_new(aenc,venc));
+ * \endcode
+**/
+MSBitrateController *ms_av_bitrate_controller_new(RtpSession *asession, MSFilter *aenc, RtpSession *vsession, MSFilter *venc);
 
 #ifdef __cplusplus
 }

include/mediastreamer2/mediastream.h

@@ -232,10 +232,12 @@ struct _VideoStream
 	VideoStreamDir dir;
 	MSWebCam *cam;
 	bool_t use_preview_window;
-	bool_t adapt_bitrate;
+	bool_t use_rc;
+	bool_t pad[2];
 	int device_orientation; /* warning: meaning of this variable depends on the platform (Android, iOS, ...) */
 	OrtpZrtpContext *ortpZrtpContext;
 	srtp_t srtp_session;
+	MSBitrateController *rc;
 };
 
 typedef struct _VideoStream VideoStream;

src/bitratecontrol.c

@@ -22,6 +22,7 @@ Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
 
 #include "mediastreamer2/bitratecontrol.h"
 
+static const int probing_up_interval=10;
 
 enum state_t{
 	Init,
@@ -70,8 +71,9 @@ static void state_machine(MSBitrateController *obj){
 			if (action.type!=MSRateControlActionDoNothing){
 				execute_action(obj,&action);
 				obj->state=Probing;
-			}else if (obj->stable_count>=5){
+			}else if (obj->stable_count>=probing_up_interval){
 				action.type=MSRateControlActionIncreaseQuality;
+				action.value=10;
 				execute_action(obj,&action);
 				obj->state=ProbingUp;
 				obj->probing_up_count=0;
@@ -99,6 +101,7 @@ static void state_machine(MSBitrateController *obj){
 				/*continue with slow ramp up*/
 				if (obj->probing_up_count==2){
 					action.type=MSRateControlActionIncreaseQuality;
+					action.value=10;
 					if (execute_action(obj,&action)==-1){
 						/* we reached the maximum*/
 						obj->state=Init;
@@ -133,3 +136,10 @@ MSBitrateController *ms_audio_bitrate_controller_new(RtpSession *session, MSFilt
 	                                 ms_audio_bitrate_driver_new(encoder));
 }
 
+MSBitrateController *ms_av_bitrate_controller_new(RtpSession *asession, MSFilter *aenc, RtpSession *vsession, MSFilter *venc){
+	return ms_bitrate_controller_new(
+	                                 ms_simple_qos_analyser_new(vsession),
+	                                 ms_av_bitrate_driver_new(aenc,venc));
+}
+
+

src/bitratedriver.c

@@ -145,3 +145,83 @@ MSBitrateDriver *ms_audio_bitrate_driver_new(MSFilter *encoder){
 	obj->cur_bitrate=obj->nom_bitrate=0;
 	return (MSBitrateDriver*)obj;
 }
+
+static const int min_video_bitrate=64000;
+static const float increase_ramp=1.1;
+
+typedef struct _MSAVBitrateDriver{
+	MSBitrateDriver parent;
+	MSBitrateDriver *audio_driver;
+	MSFilter *venc;
+	int nom_bitrate;
+	int cur_bitrate;
+}MSAVBitrateDriver;
+
+static int dec_video_bitrate(MSAVBitrateDriver *obj, const MSRateControlAction *action){
+	int new_br;
+	if (obj->nom_bitrate==0){
+		ms_filter_call_method(obj->venc,MS_FILTER_GET_BITRATE,&obj->nom_bitrate);
+		if (obj->nom_bitrate==0){
+			ms_warning("MSAVBitrateDriver: Not doing adaptive rate control on video encoder, it does not seem to support that.");
+			return -1;
+		}
+	}
+	ms_filter_call_method(obj->venc,MS_FILTER_GET_BITRATE,&obj->cur_bitrate);
+	if (obj->cur_bitrate<=min_video_bitrate){
+		ms_warning("MSAVBitrateDriver: Reaching the minimum video bitrate.");
+		return -1;
+	}
+	new_br=((float)obj->cur_bitrate)*(100.0-(float)action->value)/100.0;
+	ms_message("MSAVBitrateDriver: targeting %i bps for video encoder.",new_br);
+	ms_filter_call_method(obj->venc,MS_FILTER_SET_BITRATE,&new_br);
+	obj->cur_bitrate=new_br;
+	return 0;
+}
+
+static int av_driver_execute_action(MSBitrateDriver *objbase, const MSRateControlAction *action){
+	MSAVBitrateDriver *obj=(MSAVBitrateDriver*)objbase;
+	int ret=0;
+	switch(action->type){
+		case MSRateControlActionDecreaseBitrate:
+			ret=dec_video_bitrate(obj,action);
+		break;
+		case MSRateControlActionDecreasePacketRate:
+			if (obj->audio_driver){
+				ret=ms_bitrate_driver_execute_action(obj->audio_driver,action);
+			}
+		break;
+		case MSRateControlActionIncreaseQuality:
+		{
+			obj->cur_bitrate=(float)obj->cur_bitrate*(1.0+((float)action->value/100.0));
+			ms_message("MSAVBitrateDriver: increasing bitrate to %i bps for video encoder.",obj->cur_bitrate);
+			ms_filter_call_method(obj->venc,MS_FILTER_SET_BITRATE,&obj->cur_bitrate);
+		}
+		break;
+		case MSRateControlActionDoNothing:
+		break;
+	}
+	return ret;
+}
+
+static void av_bitrate_driver_uninit(MSBitrateDriver *objbase){
+	MSAVBitrateDriver *obj=(MSAVBitrateDriver*)objbase;
+	if (obj->audio_driver)
+		ms_bitrate_driver_unref(obj->audio_driver);
+	
+}
+
+static MSBitrateDriverDesc av_bitrate_driver={
+	av_driver_execute_action,
+	av_bitrate_driver_uninit
+};
+
+MSBitrateDriver *ms_av_bitrate_driver_new(MSFilter *aenc, MSFilter *venc){
+	MSAVBitrateDriver *obj=ms_new0(MSAVBitrateDriver,1);
+	obj->parent.desc=&av_bitrate_driver;
+	obj->audio_driver=(aenc!=NULL) ? ms_bitrate_driver_ref(ms_audio_bitrate_driver_new(aenc)) : NULL;
+	obj->venc=venc;
+	
+	return (MSBitrateDriver*)obj;
+}
+
+

src/qosanalyzer.c

@@ -65,8 +65,8 @@ void ms_qos_analyser_unref(MSQosAnalyser *obj){
 
 #define STATS_HISTORY 3
 
-static const float unacceptable_loss_rate=20;
-static const int big_jitter=20; /*ms */
+static const float unacceptable_loss_rate=10;
+static const int big_jitter=10; /*ms */
 static const float significant_delay=0.2; /*seconds*/
 
 

src/videoenc.c

@@ -837,6 +837,14 @@ static int enc_set_br(MSFilter *f, void *arg){
 	EncState *s=(EncState*)f->data;
 	bool_t snow=s->codec==CODEC_ID_SNOW;
 	s->maxbr=*(int*)arg;
+	if (s->av_context.codec!=NULL){
+		/*when we are processing, apply new settings immediately*/
+		ms_filter_lock(f);
+		enc_postprocess(f);
+		enc_preprocess(f);
+		ms_filter_unlock(f);
+		return 0;
+	}
 	if (s->maxbr>=1024000 && s->codec!=CODEC_ID_H263P){
 		s->vsize.width = MS_VIDEO_SIZE_SVGA_W;
 		s->vsize.height = MS_VIDEO_SIZE_SVGA_H;
@@ -875,14 +883,6 @@ static int enc_set_br(MSFilter *f, void *arg){
 		s->fps=5;
 		s->qmin=5;
 	}
-
-	if (s->av_context.codec!=NULL){
-		/*apply new settings dynamically*/
-		ms_filter_lock(f);
-		enc_postprocess(f);
-		enc_preprocess(f);
-		ms_filter_unlock(f);
-	}
 	return 0;
 }
 

src/videostream.c

@@ -73,6 +73,9 @@ void video_stream_free (VideoStream * stream)
 		ortp_ev_queue_destroy(stream->evq);
 	if (stream->display_name!=NULL)
 		ms_free(stream->display_name);
+	if (stream->rc!=NULL){
+		ms_bitrate_controller_destroy(stream->rc);
+	}
 
 	ms_free (stream);
 }
@@ -130,30 +133,6 @@ void video_stream_change_decoder(VideoStream *stream, int payload){
 	}
 }
 
-static void video_stream_adapt_bitrate(VideoStream *stream, int jitter, float lost){
-	if (stream->encoder!=NULL){
-		if (lost>10){
-			int bitrate=0;
-			int new_bitrate;
-			ms_warning("Remote reports bad receiving experience, trying to reduce bitrate of video encoder.");
-
-			ms_filter_call_method(stream->encoder,MS_FILTER_GET_BITRATE,&bitrate);
-			if (bitrate==0){
-				ms_error("Video encoder does not implement MS_FILTER_GET_BITRATE.");
-				return;
-			}
-			if (bitrate>=20000){
-				new_bitrate=bitrate-10000;
-				ms_warning("Encoder bitrate reduced from %i to %i b/s.",bitrate,new_bitrate);
-				ms_filter_call_method(stream->encoder,MS_FILTER_SET_BITRATE,&new_bitrate);
-			}else{
-				ms_warning("Video encoder bitrate already at minimum.");
-			}
-
-		}
-	}
-}
-
 static void video_steam_process_rtcp(VideoStream *stream, mblk_t *m){
 	do{
 		if (rtcp_is_SR(m)){
@@ -167,7 +146,8 @@ static void video_steam_process_rtcp(VideoStream *stream, mblk_t *m){
 				ij=report_block_get_interarrival_jitter(rb);
 				flost=(float)(100.0*report_block_get_fraction_lost(rb)/256.0);
 				ms_message("video_steam_process_rtcp: interarrival jitter=%u , lost packets percentage since last report=%f, round trip time=%f seconds",ij,flost,rt);
-				if (stream->adapt_bitrate) video_stream_adapt_bitrate(stream,ij,flost);
+				if (stream->rc)
+					ms_bitrate_controller_process_rtcp(stream->rc,m);
 			}
 		}
 	}while(rtcp_next_packet(m));
@@ -244,7 +224,7 @@ void video_stream_enable_self_view(VideoStream *stream, bool_t val){
 }
 
 void video_stream_enable_adaptive_bitrate_control(VideoStream *s, bool_t yesno){
-	s->adapt_bitrate=yesno;
+	s->use_rc=yesno;
 }
 
 void video_stream_set_render_callback (VideoStream *s, VideoStreamRenderCallback cb, void *user_pointer){
@@ -334,7 +314,13 @@ static void configure_video_source(VideoStream *stream){
 	}
 	stream->sizeconv=ms_filter_new(MS_SIZE_CONV_ID);
 	ms_filter_call_method(stream->sizeconv,MS_FILTER_SET_VIDEO_SIZE,&vsize);
-
+	if (stream->rc){
+		ms_bitrate_controller_destroy(stream->rc);
+		stream->rc=NULL;
+	}
+	if (stream->use_rc){
+		stream->rc=ms_av_bitrate_controller_new(NULL,NULL,stream->session,stream->encoder);
+	}
 }
 
 int video_stream_start (VideoStream *stream, RtpProfile *profile, const char *remip, int remport,

src/vp8.c

@@ -70,12 +70,14 @@ static bool_t video_starter_need_i_frame(VideoStarter *vs, uint64_t curtime){
 typedef struct EncState {
 	vpx_codec_ctx_t codec;
 	vpx_codec_enc_cfg_t cfg;
+	int bitrate;
 	int width, height;
 	long long frame_count;
 	unsigned int mtu;
 	float fps;
 	VideoStarter starter;
 	bool_t req_vfu;
+	bool_t ready;
 #ifdef FRAGMENT_ON_PARTITIONS
 	uint8_t token_partition_count;
 #endif
@@ -85,7 +87,7 @@ static void vp8_fragment_and_send(MSFilter *f,EncState *s,mblk_t *frame, uint32_
 
 static void enc_init(MSFilter *f) {
 	vpx_codec_err_t res;
-	EncState *s=(EncState *)ms_new(EncState,1);
+	EncState *s=(EncState *)ms_new0(EncState,1);
 
 	ms_message("Using %s\n",vpx_codec_iface_name(interface));
 
@@ -97,19 +99,20 @@ static void enc_init(MSFilter *f) {
 
 	s->width = MS_VIDEO_SIZE_CIF_W;
 	s->height = MS_VIDEO_SIZE_CIF_H;
+	s->bitrate=256000;
 	s->frame_count = 0;
 	s->cfg.g_w = s->width;
 	s->cfg.g_h = s->height;
 	/* encoder automatically places keyframes */
 	s->cfg.kf_mode = VPX_KF_AUTO;
 	s->cfg.kf_max_dist = 300;
-	s->cfg.rc_target_bitrate = 250;
+	s->cfg.rc_target_bitrate = ((float)s->bitrate)*0.92/1024.0; //0.9=take into account IP/UDP/RTP overhead, in average.
 	s->cfg.g_pass = VPX_RC_ONE_PASS; /* -p 1 */
 	s->fps=15;
 	s->cfg.g_timebase.num = 1;
 	s->cfg.g_timebase.den = s->fps;
 	s->cfg.rc_end_usage = VPX_CBR; /* --end-usage=cbr */
-	s->cfg.g_threads = 4; /* -t 4 */
+	s->cfg.g_threads = 1;
 	s->cfg.rc_undershoot_pct = 95; /* --undershoot-pct=95 */
 	s->cfg.g_error_resilient = 1;
 	s->cfg.g_lag_in_frames = 0;
@@ -120,7 +123,7 @@ static void enc_init(MSFilter *f) {
 
 static void enc_uninit(MSFilter *f) {
 	EncState *s=(EncState*)f->data;
-	vpx_codec_destroy(&s->codec);
+	
 	ms_free(s);
 }
 
@@ -130,7 +133,7 @@ static void enc_preprocess(MSFilter *f) {
 
 	s->cfg.g_w = s->width;
 	s->cfg.g_h = s->height;
-
+	s->cfg.g_timebase.den=s->fps;
 	/* Initialize codec */
 	#ifdef FRAGMENT_ON_PARTITIONS
 	/* VPX_CODEC_USE_OUTPUT_PARTITION: output 1 frame per partition */
@@ -142,7 +145,7 @@ static void enc_preprocess(MSFilter *f) {
 		ms_error("vpx_codec_enc_init failed: %s (%s)n", vpx_codec_err_to_string(res), vpx_codec_error_detail(&s->codec));
 	}
 
-	vpx_codec_control(&s->codec, VP8E_SET_CPUUSED, 4); /* --cpu-used=4 */
+	/*vpx_codec_control(&s->codec, VP8E_SET_CPUUSED, 4); */
 	vpx_codec_control(&s->codec, VP8E_SET_STATIC_THRESHOLD, 0);
 	vpx_codec_control(&s->codec, VP8E_SET_ENABLEAUTOALTREF, 1);
 	#ifdef FRAGMENT_ON_PARTITIONS
@@ -152,6 +155,7 @@ static void enc_preprocess(MSFilter *f) {
 	/* vpx_codec_control(&s->codec, VP8E_SET_CPUUSED, 0);*/ /* -16 (quality) .. 16 (speed) */
 
 	video_starter_init(&s->starter);
+	s->ready=TRUE;
 }
 
 static void enc_process(MSFilter *f) {
@@ -163,6 +167,7 @@ static void enc_process(MSFilter *f) {
 	vpx_codec_err_t err;
 	YuvBuf yuv;
 
+	ms_filter_lock(f);
 	while((im=ms_queue_get(f->inputs[0]))!=NULL){
 		vpx_image_t img;
 
@@ -211,10 +216,13 @@ static void enc_process(MSFilter *f) {
 		}
 		freemsg(im);
 	}
+	ms_filter_unlock(f);
 }
 
 static void enc_postprocess(MSFilter *f) {
-
+	EncState *s=(EncState*)f->data;
+	if (s->ready) vpx_codec_destroy(&s->codec);
+	s->ready=FALSE;
 }
 
 static int enc_set_vsize(MSFilter *f, void*data){
@@ -253,10 +261,24 @@ static int enc_get_fps(MSFilter *f, void *data){
 	return 0;
 }
 
+static int enc_get_br(MSFilter *f, void*data){
+	EncState *s=(EncState*)f->data;
+	*(int*)data=s->bitrate;
+	return 0;
+}
+
 static int enc_set_br(MSFilter *f, void*data){
 	int br=*(int*)data;
 	EncState *s=(EncState*)f->data;
-	s->cfg.rc_target_bitrate = br / 1024;
+	s->bitrate=br;
+	s->cfg.rc_target_bitrate = ((float)s->bitrate)*0.92/1024.0; //0.9=take into account IP/UDP/RTP overhead, in average.
+	if (s->ready){
+		ms_filter_lock(f);
+		enc_postprocess(f);
+		enc_preprocess(f);
+		ms_filter_unlock(f);
+		return 0;
+	}
 	if (br>=1024000){
 		s->width = MS_VIDEO_SIZE_VGA_W;
 		s->height = MS_VIDEO_SIZE_VGA_H;
@@ -287,7 +309,7 @@ static int enc_set_br(MSFilter *f, void*data){
 		s->fps=5;
 	}
 
-	ms_warning("bitrate requested...: %d (%d x %d)\n", br, s->width, s->height);
+	ms_message("bitrate requested...: %d (%d x %d)\n", br, s->width, s->height);
 	return 0;
 }
 
@@ -310,6 +332,7 @@ static MSFilterMethod enc_methods[]={
 	{	MS_FILTER_GET_FPS,	  enc_get_fps	},
 	{	MS_FILTER_ADD_ATTR,       enc_add_attr	},
 	{	MS_FILTER_SET_BITRATE,    enc_set_br	},
+	{	MS_FILTER_GET_BITRATE,    enc_get_br	},
 	{	MS_FILTER_SET_MTU,        enc_set_mtu	},
 	{	MS_FILTER_REQ_VFU,        enc_req_vfu  },
 	{	0			, NULL }

tests/mediastream.c

@@ -71,7 +71,7 @@ static int cond=1;
 
 typedef struct _MediastreamDatas {
 	int localport,remoteport,payload;
-	char ip[50];
+	char ip[64];
 	char *fmtp;
 	int jitter;
 	int bitrate;
@@ -93,13 +93,15 @@ typedef struct _MediastreamDatas {
 	bool_t use_ng;
 	bool_t two_windows;
 	bool_t el;
+	bool_t use_rc;
+	bool_t enable_srtp;
+	bool_t pad[3];
 	float el_speed;
 	float el_thres;
 	float el_force;
 	int el_sustain;
 	float el_transmit_thres;
 	float ng_floorgain;
-	bool_t use_rc;
 	char * zrtp_id;
 	char * zrtp_secrets;
 	PayloadType *custom_pt;
@@ -107,9 +109,9 @@ typedef struct _MediastreamDatas {
 	int preview_window_id;
 	/* starting values echo canceller */
 	int ec_len_ms, ec_delay_ms, ec_framesize;
-	bool_t enable_srtp;
 	char* srtp_local_master_key;
 	char* srtp_remote_master_key;
+	int netsim_bw;
 	
 	AudioStream *audio;	
 	PayloadType *pt;
@@ -172,6 +174,7 @@ const char *usage="mediastream --local <port> --remote <ip:port> \n"
 								"[ --verbose (most verbose messages) ]\n"
 								"[ --video-windows-id <video surface:preview surface>]\n"
 								"[ --srtp <local master_key> <remote master_key> (enable srtp, master key is generated if absent from comand line)\n"
+								"[ --netsim-bandwidth <bandwidth limit in bits/s> (simulates a network download bandwidth limit)\n"
 		;
 
 
@@ -236,7 +239,7 @@ static int _main(int argc, char * argv[])
 
 
 MediastreamDatas* init_default_args() {
-	MediastreamDatas* args = (MediastreamDatas*)malloc(sizeof(MediastreamDatas));
+	MediastreamDatas* args = (MediastreamDatas*)ms_malloc0(sizeof(MediastreamDatas));
 	args->localport=0;
 	args->remoteport=0;
 	args->payload=0;
@@ -418,6 +421,9 @@ bool_t parse_args(int argc, char** argv, MediastreamDatas* out) {
 				out->srtp_local_master_key = argv[i++];
 				out->srtp_remote_master_key = argv[i++];
 			}
+		} else if (strcmp(argv[i],"--netsim-bandwidth")==0){
+			i++;
+			out->netsim_bw=atoi(argv[i]);
 		} else if (strcmp(argv[i],"--help")==0){
 			printf("%s",usage);
 			return FALSE;
@@ -582,7 +588,7 @@ void setup_media_streams(MediastreamDatas* args) {
 		const char*  nowebcam = [[myBundle pathForResource:@"nowebcamCIF"ofType:@"jpg"] cStringUsingEncoding:[NSString defaultCStringEncoding]];
 		ms_static_image_set_default_image(nowebcam);
 #endif
-
+		video_stream_enable_adaptive_bitrate_control(args->video,args->use_rc);
 		if (args->camera)
 			cam=ms_web_cam_manager_get_cam(ms_web_cam_manager_get(),args->camera);
 		if (cam==NULL)
@@ -607,6 +613,12 @@ void setup_media_streams(MediastreamDatas* args) {
 		printf("Error: video support not compiled.\n");
 #endif
 	}
+	if (args->netsim_bw>0){
+		OrtpNetworkSimulatorParams params={0};
+		params.enabled=TRUE;
+		params.max_bandwidth=args->netsim_bw;
+		rtp_session_enable_network_simulation(args->session,&params);
+	}
 }