Commit 954911c0 authored by Simon Morlat's avatar Simon Morlat

implement RTCP based rate control for mpeg and vp8

parent e643594a
......@@ -38,7 +38,7 @@ enum _MSRateControlActionType{
MSRateControlActionDoNothing,
MSRateControlActionDecreaseBitrate,
MSRateControlActionDecreasePacketRate,
MSRateControlActionIncreaseQuality
MSRateControlActionIncreaseQuality,
};
typedef enum _MSRateControlActionType MSRateControlActionType;
......@@ -72,7 +72,7 @@ MSBitrateDriver * ms_bitrate_driver_ref(MSBitrateDriver *obj);
void ms_bitrate_driver_unref(MSBitrateDriver *obj);
MSBitrateDriver *ms_audio_bitrate_driver_new(MSFilter *encoder);
MSBitrateDriver *ms_av_bitrate_driver_new(MSFilter *a_encoder, MSFilter *venc);
typedef struct _MSQosAnalyser MSQosAnalyser;
typedef struct _MSQosAnalyserDesc MSQosAnalyserDesc;
......@@ -100,10 +100,15 @@ bool_t ms_qos_analyser_has_improved(MSQosAnalyser *obj);
bool_t ms_qos_analyser_process_rtcp(MSQosAnalyser *obj, mblk_t *rtcp);
/**
* The simple qos analyzer is an implementation of MSQosAnalizer that performs analysis for single stream.
* The simple qos analyzer is an implementation of MSQosAnalyser that performs analysis for single stream.
**/
MSQosAnalyser * ms_simple_qos_analyser_new(RtpSession *session);
/**
* The audio/video qos analyser is an implementation of MSQosAnalyser that performs analysis of two audio and video streams.
**/
MSQosAnalyser * ms_av_qos_analyser_new(RtpSession *asession, RtpSession *vsession);
/**
* The MSBitrateController the overall behavior and state machine of the adaptive rate control system.
* It requires a MSQosAnalyser to obtain analyse of the quality of service, and a MSBitrateDriver
......@@ -151,6 +156,19 @@ void ms_bitrate_controller_destroy(MSBitrateController *obj);
**/
MSBitrateController *ms_audio_bitrate_controller_new(RtpSession *session, MSFilter *encoder, unsigned int flags);
/**
* Convenience fonction to create a bitrate controller managing a video and an audio stream.
* @param vsession the video RtpSession
* @param venc the video encoder
* @param asession the audio RtpSession
* @param aenc the audio encoder
* This function actually calls internally:
* <br>
* \code
* ms_bitrate_controller_new(ms_av_qos_analyser_new(asession,vsession),ms_av_bitrate_driver_new(aenc,venc));
* \endcode
**/
MSBitrateController *ms_av_bitrate_controller_new(RtpSession *asession, MSFilter *aenc, RtpSession *vsession, MSFilter *venc);
#ifdef __cplusplus
}
......
......@@ -232,10 +232,12 @@ struct _VideoStream
VideoStreamDir dir;
MSWebCam *cam;
bool_t use_preview_window;
bool_t adapt_bitrate;
bool_t use_rc;
bool_t pad[2];
int device_orientation; /* warning: meaning of this variable depends on the platform (Android, iOS, ...) */
OrtpZrtpContext *ortpZrtpContext;
srtp_t srtp_session;
MSBitrateController *rc;
};
typedef struct _VideoStream VideoStream;
......
......@@ -22,6 +22,7 @@ Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#include "mediastreamer2/bitratecontrol.h"
static const int probing_up_interval=10;
enum state_t{
Init,
......@@ -70,8 +71,9 @@ static void state_machine(MSBitrateController *obj){
if (action.type!=MSRateControlActionDoNothing){
execute_action(obj,&action);
obj->state=Probing;
}else if (obj->stable_count>=5){
}else if (obj->stable_count>=probing_up_interval){
action.type=MSRateControlActionIncreaseQuality;
action.value=10;
execute_action(obj,&action);
obj->state=ProbingUp;
obj->probing_up_count=0;
......@@ -99,6 +101,7 @@ static void state_machine(MSBitrateController *obj){
/*continue with slow ramp up*/
if (obj->probing_up_count==2){
action.type=MSRateControlActionIncreaseQuality;
action.value=10;
if (execute_action(obj,&action)==-1){
/* we reached the maximum*/
obj->state=Init;
......@@ -133,3 +136,10 @@ MSBitrateController *ms_audio_bitrate_controller_new(RtpSession *session, MSFilt
ms_audio_bitrate_driver_new(encoder));
}
MSBitrateController *ms_av_bitrate_controller_new(RtpSession *asession, MSFilter *aenc, RtpSession *vsession, MSFilter *venc){
return ms_bitrate_controller_new(
ms_simple_qos_analyser_new(vsession),
ms_av_bitrate_driver_new(aenc,venc));
}
......@@ -145,3 +145,83 @@ MSBitrateDriver *ms_audio_bitrate_driver_new(MSFilter *encoder){
obj->cur_bitrate=obj->nom_bitrate=0;
return (MSBitrateDriver*)obj;
}
static const int min_video_bitrate=64000;
static const float increase_ramp=1.1;
typedef struct _MSAVBitrateDriver{
MSBitrateDriver parent;
MSBitrateDriver *audio_driver;
MSFilter *venc;
int nom_bitrate;
int cur_bitrate;
}MSAVBitrateDriver;
static int dec_video_bitrate(MSAVBitrateDriver *obj, const MSRateControlAction *action){
int new_br;
if (obj->nom_bitrate==0){
ms_filter_call_method(obj->venc,MS_FILTER_GET_BITRATE,&obj->nom_bitrate);
if (obj->nom_bitrate==0){
ms_warning("MSAVBitrateDriver: Not doing adaptive rate control on video encoder, it does not seem to support that.");
return -1;
}
}
ms_filter_call_method(obj->venc,MS_FILTER_GET_BITRATE,&obj->cur_bitrate);
if (obj->cur_bitrate<=min_video_bitrate){
ms_warning("MSAVBitrateDriver: Reaching the minimum video bitrate.");
return -1;
}
new_br=((float)obj->cur_bitrate)*(100.0-(float)action->value)/100.0;
ms_message("MSAVBitrateDriver: targeting %i bps for video encoder.",new_br);
ms_filter_call_method(obj->venc,MS_FILTER_SET_BITRATE,&new_br);
obj->cur_bitrate=new_br;
return 0;
}
static int av_driver_execute_action(MSBitrateDriver *objbase, const MSRateControlAction *action){
MSAVBitrateDriver *obj=(MSAVBitrateDriver*)objbase;
int ret=0;
switch(action->type){
case MSRateControlActionDecreaseBitrate:
ret=dec_video_bitrate(obj,action);
break;
case MSRateControlActionDecreasePacketRate:
if (obj->audio_driver){
ret=ms_bitrate_driver_execute_action(obj->audio_driver,action);
}
break;
case MSRateControlActionIncreaseQuality:
{
obj->cur_bitrate=(float)obj->cur_bitrate*(1.0+((float)action->value/100.0));
ms_message("MSAVBitrateDriver: increasing bitrate to %i bps for video encoder.",obj->cur_bitrate);
ms_filter_call_method(obj->venc,MS_FILTER_SET_BITRATE,&obj->cur_bitrate);
}
break;
case MSRateControlActionDoNothing:
break;
}
return ret;
}
static void av_bitrate_driver_uninit(MSBitrateDriver *objbase){
MSAVBitrateDriver *obj=(MSAVBitrateDriver*)objbase;
if (obj->audio_driver)
ms_bitrate_driver_unref(obj->audio_driver);
}
static MSBitrateDriverDesc av_bitrate_driver={
av_driver_execute_action,
av_bitrate_driver_uninit
};
MSBitrateDriver *ms_av_bitrate_driver_new(MSFilter *aenc, MSFilter *venc){
MSAVBitrateDriver *obj=ms_new0(MSAVBitrateDriver,1);
obj->parent.desc=&av_bitrate_driver;
obj->audio_driver=(aenc!=NULL) ? ms_bitrate_driver_ref(ms_audio_bitrate_driver_new(aenc)) : NULL;
obj->venc=venc;
return (MSBitrateDriver*)obj;
}
......@@ -65,8 +65,8 @@ void ms_qos_analyser_unref(MSQosAnalyser *obj){
#define STATS_HISTORY 3
static const float unacceptable_loss_rate=20;
static const int big_jitter=20; /*ms */
static const float unacceptable_loss_rate=10;
static const int big_jitter=10; /*ms */
static const float significant_delay=0.2; /*seconds*/
......
......@@ -837,6 +837,14 @@ static int enc_set_br(MSFilter *f, void *arg){
EncState *s=(EncState*)f->data;
bool_t snow=s->codec==CODEC_ID_SNOW;
s->maxbr=*(int*)arg;
if (s->av_context.codec!=NULL){
/*when we are processing, apply new settings immediately*/
ms_filter_lock(f);
enc_postprocess(f);
enc_preprocess(f);
ms_filter_unlock(f);
return 0;
}
if (s->maxbr>=1024000 && s->codec!=CODEC_ID_H263P){
s->vsize.width = MS_VIDEO_SIZE_SVGA_W;
s->vsize.height = MS_VIDEO_SIZE_SVGA_H;
......@@ -875,14 +883,6 @@ static int enc_set_br(MSFilter *f, void *arg){
s->fps=5;
s->qmin=5;
}
if (s->av_context.codec!=NULL){
/*apply new settings dynamically*/
ms_filter_lock(f);
enc_postprocess(f);
enc_preprocess(f);
ms_filter_unlock(f);
}
return 0;
}
......
......@@ -73,6 +73,9 @@ void video_stream_free (VideoStream * stream)
ortp_ev_queue_destroy(stream->evq);
if (stream->display_name!=NULL)
ms_free(stream->display_name);
if (stream->rc!=NULL){
ms_bitrate_controller_destroy(stream->rc);
}
ms_free (stream);
}
......@@ -130,30 +133,6 @@ void video_stream_change_decoder(VideoStream *stream, int payload){
}
}
static void video_stream_adapt_bitrate(VideoStream *stream, int jitter, float lost){
if (stream->encoder!=NULL){
if (lost>10){
int bitrate=0;
int new_bitrate;
ms_warning("Remote reports bad receiving experience, trying to reduce bitrate of video encoder.");
ms_filter_call_method(stream->encoder,MS_FILTER_GET_BITRATE,&bitrate);
if (bitrate==0){
ms_error("Video encoder does not implement MS_FILTER_GET_BITRATE.");
return;
}
if (bitrate>=20000){
new_bitrate=bitrate-10000;
ms_warning("Encoder bitrate reduced from %i to %i b/s.",bitrate,new_bitrate);
ms_filter_call_method(stream->encoder,MS_FILTER_SET_BITRATE,&new_bitrate);
}else{
ms_warning("Video encoder bitrate already at minimum.");
}
}
}
}
static void video_steam_process_rtcp(VideoStream *stream, mblk_t *m){
do{
if (rtcp_is_SR(m)){
......@@ -167,7 +146,8 @@ static void video_steam_process_rtcp(VideoStream *stream, mblk_t *m){
ij=report_block_get_interarrival_jitter(rb);
flost=(float)(100.0*report_block_get_fraction_lost(rb)/256.0);
ms_message("video_steam_process_rtcp: interarrival jitter=%u , lost packets percentage since last report=%f, round trip time=%f seconds",ij,flost,rt);
if (stream->adapt_bitrate) video_stream_adapt_bitrate(stream,ij,flost);
if (stream->rc)
ms_bitrate_controller_process_rtcp(stream->rc,m);
}
}
}while(rtcp_next_packet(m));
......@@ -244,7 +224,7 @@ void video_stream_enable_self_view(VideoStream *stream, bool_t val){
}
void video_stream_enable_adaptive_bitrate_control(VideoStream *s, bool_t yesno){
s->adapt_bitrate=yesno;
s->use_rc=yesno;
}
void video_stream_set_render_callback (VideoStream *s, VideoStreamRenderCallback cb, void *user_pointer){
......@@ -334,7 +314,13 @@ static void configure_video_source(VideoStream *stream){
}
stream->sizeconv=ms_filter_new(MS_SIZE_CONV_ID);
ms_filter_call_method(stream->sizeconv,MS_FILTER_SET_VIDEO_SIZE,&vsize);
if (stream->rc){
ms_bitrate_controller_destroy(stream->rc);
stream->rc=NULL;
}
if (stream->use_rc){
stream->rc=ms_av_bitrate_controller_new(NULL,NULL,stream->session,stream->encoder);
}
}
int video_stream_start (VideoStream *stream, RtpProfile *profile, const char *remip, int remport,
......
......@@ -70,12 +70,14 @@ static bool_t video_starter_need_i_frame(VideoStarter *vs, uint64_t curtime){
typedef struct EncState {
vpx_codec_ctx_t codec;
vpx_codec_enc_cfg_t cfg;
int bitrate;
int width, height;
long long frame_count;
unsigned int mtu;
float fps;
VideoStarter starter;
bool_t req_vfu;
bool_t ready;
#ifdef FRAGMENT_ON_PARTITIONS
uint8_t token_partition_count;
#endif
......@@ -85,7 +87,7 @@ static void vp8_fragment_and_send(MSFilter *f,EncState *s,mblk_t *frame, uint32_
static void enc_init(MSFilter *f) {
vpx_codec_err_t res;
EncState *s=(EncState *)ms_new(EncState,1);
EncState *s=(EncState *)ms_new0(EncState,1);
ms_message("Using %s\n",vpx_codec_iface_name(interface));
......@@ -97,19 +99,20 @@ static void enc_init(MSFilter *f) {
s->width = MS_VIDEO_SIZE_CIF_W;
s->height = MS_VIDEO_SIZE_CIF_H;
s->bitrate=256000;
s->frame_count = 0;
s->cfg.g_w = s->width;
s->cfg.g_h = s->height;
/* encoder automatically places keyframes */
s->cfg.kf_mode = VPX_KF_AUTO;
s->cfg.kf_max_dist = 300;
s->cfg.rc_target_bitrate = 250;
s->cfg.rc_target_bitrate = ((float)s->bitrate)*0.92/1024.0; //0.9=take into account IP/UDP/RTP overhead, in average.
s->cfg.g_pass = VPX_RC_ONE_PASS; /* -p 1 */
s->fps=15;
s->cfg.g_timebase.num = 1;
s->cfg.g_timebase.den = s->fps;
s->cfg.rc_end_usage = VPX_CBR; /* --end-usage=cbr */
s->cfg.g_threads = 4; /* -t 4 */
s->cfg.g_threads = 1;
s->cfg.rc_undershoot_pct = 95; /* --undershoot-pct=95 */
s->cfg.g_error_resilient = 1;
s->cfg.g_lag_in_frames = 0;
......@@ -120,7 +123,7 @@ static void enc_init(MSFilter *f) {
static void enc_uninit(MSFilter *f) {
EncState *s=(EncState*)f->data;
vpx_codec_destroy(&s->codec);
ms_free(s);
}
......@@ -130,7 +133,7 @@ static void enc_preprocess(MSFilter *f) {
s->cfg.g_w = s->width;
s->cfg.g_h = s->height;
s->cfg.g_timebase.den=s->fps;
/* Initialize codec */
#ifdef FRAGMENT_ON_PARTITIONS
/* VPX_CODEC_USE_OUTPUT_PARTITION: output 1 frame per partition */
......@@ -142,7 +145,7 @@ static void enc_preprocess(MSFilter *f) {
ms_error("vpx_codec_enc_init failed: %s (%s)n", vpx_codec_err_to_string(res), vpx_codec_error_detail(&s->codec));
}
vpx_codec_control(&s->codec, VP8E_SET_CPUUSED, 4); /* --cpu-used=4 */
/*vpx_codec_control(&s->codec, VP8E_SET_CPUUSED, 4); */
vpx_codec_control(&s->codec, VP8E_SET_STATIC_THRESHOLD, 0);
vpx_codec_control(&s->codec, VP8E_SET_ENABLEAUTOALTREF, 1);
#ifdef FRAGMENT_ON_PARTITIONS
......@@ -152,6 +155,7 @@ static void enc_preprocess(MSFilter *f) {
/* vpx_codec_control(&s->codec, VP8E_SET_CPUUSED, 0);*/ /* -16 (quality) .. 16 (speed) */
video_starter_init(&s->starter);
s->ready=TRUE;
}
static void enc_process(MSFilter *f) {
......@@ -163,6 +167,7 @@ static void enc_process(MSFilter *f) {
vpx_codec_err_t err;
YuvBuf yuv;
ms_filter_lock(f);
while((im=ms_queue_get(f->inputs[0]))!=NULL){
vpx_image_t img;
......@@ -211,10 +216,13 @@ static void enc_process(MSFilter *f) {
}
freemsg(im);
}
ms_filter_unlock(f);
}
static void enc_postprocess(MSFilter *f) {
EncState *s=(EncState*)f->data;
if (s->ready) vpx_codec_destroy(&s->codec);
s->ready=FALSE;
}
static int enc_set_vsize(MSFilter *f, void*data){
......@@ -253,10 +261,24 @@ static int enc_get_fps(MSFilter *f, void *data){
return 0;
}
static int enc_get_br(MSFilter *f, void*data){
EncState *s=(EncState*)f->data;
*(int*)data=s->bitrate;
return 0;
}
static int enc_set_br(MSFilter *f, void*data){
int br=*(int*)data;
EncState *s=(EncState*)f->data;
s->cfg.rc_target_bitrate = br / 1024;
s->bitrate=br;
s->cfg.rc_target_bitrate = ((float)s->bitrate)*0.92/1024.0; //0.9=take into account IP/UDP/RTP overhead, in average.
if (s->ready){
ms_filter_lock(f);
enc_postprocess(f);
enc_preprocess(f);
ms_filter_unlock(f);
return 0;
}
if (br>=1024000){
s->width = MS_VIDEO_SIZE_VGA_W;
s->height = MS_VIDEO_SIZE_VGA_H;
......@@ -287,7 +309,7 @@ static int enc_set_br(MSFilter *f, void*data){
s->fps=5;
}
ms_warning("bitrate requested...: %d (%d x %d)\n", br, s->width, s->height);
ms_message("bitrate requested...: %d (%d x %d)\n", br, s->width, s->height);
return 0;
}
......@@ -310,6 +332,7 @@ static MSFilterMethod enc_methods[]={
{ MS_FILTER_GET_FPS, enc_get_fps },
{ MS_FILTER_ADD_ATTR, enc_add_attr },
{ MS_FILTER_SET_BITRATE, enc_set_br },
{ MS_FILTER_GET_BITRATE, enc_get_br },
{ MS_FILTER_SET_MTU, enc_set_mtu },
{ MS_FILTER_REQ_VFU, enc_req_vfu },
{ 0 , NULL }
......
......@@ -71,7 +71,7 @@ static int cond=1;
typedef struct _MediastreamDatas {
int localport,remoteport,payload;
char ip[50];
char ip[64];
char *fmtp;
int jitter;
int bitrate;
......@@ -93,13 +93,15 @@ typedef struct _MediastreamDatas {
bool_t use_ng;
bool_t two_windows;
bool_t el;
bool_t use_rc;
bool_t enable_srtp;
bool_t pad[3];
float el_speed;
float el_thres;
float el_force;
int el_sustain;
float el_transmit_thres;
float ng_floorgain;
bool_t use_rc;
char * zrtp_id;
char * zrtp_secrets;
PayloadType *custom_pt;
......@@ -107,9 +109,9 @@ typedef struct _MediastreamDatas {
int preview_window_id;
/* starting values echo canceller */
int ec_len_ms, ec_delay_ms, ec_framesize;
bool_t enable_srtp;
char* srtp_local_master_key;
char* srtp_remote_master_key;
int netsim_bw;
AudioStream *audio;
PayloadType *pt;
......@@ -172,6 +174,7 @@ const char *usage="mediastream --local <port> --remote <ip:port> \n"
"[ --verbose (most verbose messages) ]\n"
"[ --video-windows-id <video surface:preview surface>]\n"
"[ --srtp <local master_key> <remote master_key> (enable srtp, master key is generated if absent from comand line)\n"
"[ --netsim-bandwidth <bandwidth limit in bits/s> (simulates a network download bandwidth limit)\n"
;
......@@ -236,7 +239,7 @@ static int _main(int argc, char * argv[])
MediastreamDatas* init_default_args() {
MediastreamDatas* args = (MediastreamDatas*)malloc(sizeof(MediastreamDatas));
MediastreamDatas* args = (MediastreamDatas*)ms_malloc0(sizeof(MediastreamDatas));
args->localport=0;
args->remoteport=0;
args->payload=0;
......@@ -418,6 +421,9 @@ bool_t parse_args(int argc, char** argv, MediastreamDatas* out) {
out->srtp_local_master_key = argv[i++];
out->srtp_remote_master_key = argv[i++];
}
} else if (strcmp(argv[i],"--netsim-bandwidth")==0){
i++;
out->netsim_bw=atoi(argv[i]);
} else if (strcmp(argv[i],"--help")==0){
printf("%s",usage);
return FALSE;
......@@ -582,7 +588,7 @@ void setup_media_streams(MediastreamDatas* args) {
const char* nowebcam = [[myBundle pathForResource:@"nowebcamCIF"ofType:@"jpg"] cStringUsingEncoding:[NSString defaultCStringEncoding]];
ms_static_image_set_default_image(nowebcam);
#endif
video_stream_enable_adaptive_bitrate_control(args->video,args->use_rc);
if (args->camera)
cam=ms_web_cam_manager_get_cam(ms_web_cam_manager_get(),args->camera);
if (cam==NULL)
......@@ -607,6 +613,12 @@ void setup_media_streams(MediastreamDatas* args) {
printf("Error: video support not compiled.\n");
#endif
}
if (args->netsim_bw>0){
OrtpNetworkSimulatorParams params={0};
params.enabled=TRUE;
params.max_bandwidth=args->netsim_bw;
rtp_session_enable_network_simulation(args->session,&params);
}
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment