Skip to content

GitLab

Commit f77b325c authored by jehan's avatar jehan
Browse files

add FEC to silk

parent b15f580b
Hide whitespace changes
Inline Side-by-side
Showing with 133 additions and 35 deletions
include/mediastreamer2/msfilter.h
View file @ f77b325c
......@@ -553,6 +553,8 @@ typedef enum _MSFilterInterfaceId MSFilterInterfaceId;
#define MS_FILTER_SET_VAD_PROB_CONTINUE MS_FILTER_BASE_METHOD(24,int)
#define MS_FILTER_SET_MAX_GAIN MS_FILTER_BASE_METHOD(25,int)
#define MS_VIDEO_CAPTURE_SET_AUTOFOCUS MS_FILTER_BASE_METHOD(26,int)
/* pass value of type MSRtpPayloadPickerContext copied by the filter*/
#define MS_FILTER_SET_RTP_PAYLOAD_PICKER MS_FILTER_BASE_METHOD(27,void*)
#define MS_CONF_SPEEX_PREPROCESS_MIC MS_FILTER_EVENT(MS_CONF_ID, 1, void*)
#define MS_CONF_CHANNEL_VOLUME MS_FILTER_EVENT(MS_CONF_ID, 3, void*)
......
include/mediastreamer2/msqueue.h
View file @ f77b325c
......@@ -77,15 +77,13 @@ MS2_PUBLIC void ms_queue_destroy(MSQueue *q);
#define mblk_set_timestamp_info(m,ts) (m)->reserved1=(ts);
#define mblk_get_timestamp_info(m) ((m)->reserved1)
#define mblk_set_marker_info(m,bit) (m)->reserved2=((m)->reserved2|bit)
#define mblk_get_marker_info(m) ((m)->reserved2&0x1)
#define mblk_set_rate(m,bits) (m)->reserved2=((m)->reserved2|(bits)<<1)
#define mblk_get_rate(m) (((m)->reserved2>>1)&0x3)
#define mblk_set_payload_type(m,bits) (m)->reserved2=((m)->reserved2|(bits<<3))
#define mblk_get_payload_type(m) (((m)->reserved2>>3)&0x7F)
#define mblk_set_precious_flag(m,bit) (m)->reserved2=(m)->reserved2|((bit & 0x1)<<10) /*use to prevent mirroring*/
#define mblk_get_precious_flag(m) (((m)->reserved2)>>10 & 0x1)
#define mblk_set_marker_info(m,bit) (m)->reserved2=((m)->reserved2|bit& 0x1)
#define mblk_get_marker_info(m) ((m)->reserved2&0x1) /*bit 1*/
#define mblk_set_precious_flag(m,bit) (m)->reserved2=(m)->reserved2|((bit & 0x1)<<1) /*use to prevent mirroring*/
#define mblk_get_precious_flag(m) (((m)->reserved2)>>1 & 0x1) /*bit 2*/
#define mblk_set_cseq(m,value) (m)->reserved2=(m)->reserved2| ((value&0xFFFF)<<16);
#define mblk_get_cseq(m) ((m)->reserved2>>16)
struct _MSBufferizer{
queue_t q;
int size;
......
src/audiostream.c
View file @ f77b325c
......@@ -41,7 +41,7 @@ Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#endif
#define MAX_RTP_SIZE 1500
#include "msprivate.h"
/* this code is not part of the library itself, it is part of the mediastream program */
void audio_stream_free(AudioStream *stream)
......@@ -257,7 +257,10 @@ static void payload_type_changed(RtpSession *session, unsigned long data){
int pt=rtp_session_get_recv_payload_type(stream->session);
audio_stream_change_decoder(stream,pt);
}
/*invoked from FEC capable filters*/
static mblk_t* audio_stream_payload_picker(MSRtpPayloadPickerContext* context,unsigned int sequence_number) {
return rtp_session_pick_with_cseq(((AudioStream*)(context->filter_graph_manager))->session, sequence_number);
}
int audio_stream_start_full(AudioStream *stream, RtpProfile *profile, const char *remip,int remport,
int rem_rtcp_port, int payload,int jitt_comp, const char *infile, const char *outfile,
MSSndCard *playcard, MSSndCard *captcard, bool_t use_ec)
......@@ -267,6 +270,7 @@ int audio_stream_start_full(AudioStream *stream, RtpProfile *profile, const char
int tmp;
MSConnectionHelper h;
int sample_rate;
MSRtpPayloadPickerContext picker_context;
rtp_session_set_profile(rtps,profile);
if (remport>0) rtp_session_set_remote_addr_full(rtps,remip,remport,rem_rtcp_port);
......@@ -325,8 +329,13 @@ int audio_stream_start_full(AudioStream *stream, RtpProfile *profile, const char
ms_error("mediastream.c: No decoder available for payload %i.",payload);
return -1;
}
stream->volsend=ms_filter_new(MS_VOLUME_ID);
if (ms_filter_has_method(stream->decoder, MS_FILTER_SET_RTP_PAYLOAD_PICKER)) {
ms_message(" decoder has FEC capabilities");
picker_context.filter_graph_manager=stream;
picker_context.picker=&audio_stream_payload_picker;
ms_filter_call_method(stream->decoder,MS_FILTER_SET_RTP_PAYLOAD_PICKER, &picker_context);
}
stream->volsend=ms_filter_new(MS_VOLUME_ID);
stream->volrecv=ms_filter_new(MS_VOLUME_ID);
audio_stream_enable_echo_limiter(stream,stream->el_type);
audio_stream_enable_noise_gate(stream,stream->use_ng);
......
src/mscommon.c
View file @ f77b325c
......@@ -757,10 +757,11 @@ void ms_get_cur_time(MSTimeSpec *ret){
unsigned long ms_concealer_context_get_total_number_of_plc(MSConcealerContext* obj) {
return obj->total_number_for_plc;
}
void ms_concealer_context_init(MSConcealerContext* obj){
void ms_concealer_context_init(MSConcealerContext* obj,unsigned int max_plc_count){
obj->sample_time=0;
obj->plc_count=0;
obj->total_number_for_plc=0;
obj->max_plc_count=max_plc_count;
}
unsigned long ms_concealer_context_get_sampling_time(MSConcealerContext* obj) {
return obj->sample_time;
......@@ -769,17 +770,17 @@ void ms_concealer_context_set_sampling_time(MSConcealerContext* obj,unsigned lon
obj->sample_time=value;
}
#define MAX_PLC 10
unsigned int ms_concealer_context_is_concealement_required(MSConcealerContext* obj,uint64_t current_time) {
if(obj->sample_time == 0) return 0; /*no valid value*/
if (obj->sample_time < current_time && obj->plc_count<MAX_PLC) {
if (obj->sample_time < current_time && obj->plc_count<obj->max_plc_count) {
obj->plc_count++;
obj->total_number_for_plc++;
} else {
obj->plc_count=0;
if (obj->plc_count>=MAX_PLC) {
if (obj->plc_count>=obj->max_plc_count) {
/*reset sample time*/
obj->sample_time=0;
}
......
src/msjoin.c
View file @ f77b325c
......@@ -34,7 +34,6 @@ static void join_process(MSFilter *f){
if (f->inputs[1]!=NULL)
{
while((im=ms_queue_get(f->inputs[1]))!=NULL){
mblk_set_payload_type(im, 123);
ms_queue_put(f->outputs[0],im);
}
}
......
src/msprivate.h
View file @ f77b325c
......@@ -24,13 +24,23 @@ typedef struct MSConcealerContext {
uint64_t sample_time;
int plc_count;
unsigned long total_number_for_plc;
unsigned int max_plc_count;
}MSConcealerContext;
void ms_concealer_context_init(MSConcealerContext* obj);
void ms_concealer_context_init(MSConcealerContext* obj,unsigned int max_plc_count);
void ms_concealer_context_set_sampling_time(MSConcealerContext* obj,unsigned long value);
unsigned long ms_concealer_context_get_sampling_time(MSConcealerContext* obj);
unsigned long ms_concealer_context_get_total_number_of_plc(MSConcealerContext* obj);
/* return number of concelad packet since the begening of the concealement period or 0 if not needed*/
unsigned int ms_concealer_context_is_concealement_required(MSConcealerContext* obj,uint64_t current_time);
/*FEC API*/
typedef struct _MSRtpPayloadPickerContext MSRtpPayloadPickerContext;
typedef mblk_t* (*RtpPayloadPicker)(MSRtpPayloadPickerContext* context,unsigned int sequence_number);
struct _MSRtpPayloadPickerContext {
void* filter_graph_manager; /*I.E stream*/
RtpPayloadPicker picker;
};
#endif
\ No newline at end of file
src/msrtp.c
View file @ f77b325c
......@@ -346,10 +346,7 @@ static void sender_process(MSFilter * f)
}
if (im){
if (d->skip == FALSE && d->mute_mic==FALSE){
int pt = mblk_get_payload_type(im);
header = rtp_session_create_packet(s, 12, NULL, 0);
if (pt>0)
rtp_set_payload_type(header, pt);
rtp_set_markbit(header, mblk_get_marker_info(im));
header->b_cont = im;
rtp_session_sendm_with_ts(s, header, timestamp);
......@@ -500,7 +497,7 @@ static void receiver_process(MSFilter * f)
while ((m = rtp_session_recvm_with_ts(d->session, timestamp)) != NULL) {
mblk_set_timestamp_info(m, rtp_get_timestamp(m));
mblk_set_marker_info(m, rtp_get_markbit(m));
mblk_set_payload_type(m, rtp_get_payload_type(m));
mblk_set_cseq(m, rtp_get_seqnumber(m));
rtp_get_payload(m,&m->b_rptr);
ms_queue_put(f->outputs[0], m);
}
......
src/silk_dec.c
View file @ f77b325c
......@@ -25,6 +25,8 @@ struct silk_dec_struct {
SKP_SILK_SDK_DecControlStruct control;
void *psDec;
MSConcealerContext concealer;
MSRtpPayloadPickerContext rtp_picker_context;
unsigned short int sequence_number;
};
......@@ -49,7 +51,7 @@ static void filter_preprocess(MSFilter *f){
if(ret) {
ms_error( "SKP_Silk_InitDecoder returned %d", ret );
}
ms_concealer_context_init(&obj->concealer);
ms_concealer_context_init(&obj->concealer,UINT32_MAX);
}
/**
put im to NULL for PLC
......@@ -62,7 +64,7 @@ static void decode(MSFilter *f, mblk_t *im) {
SKP_int16 ret;
/* Decode 20 ms */
om=allocb(obj->control.API_sampleRate*4/100,0); /*samplingrate*0.02*2*/
ret = SKP_Silk_SDK_Decode( obj->psDec, &obj->control, im?0:1, im?im->b_rptr:0, im?im->b_wptr - im->b_rptr:0, (SKP_int16*)om->b_wptr, &len );
ret = SKP_Silk_SDK_Decode( obj->psDec, &obj->control, im?0:1, im?im->b_rptr:0, im?(im->b_wptr - im->b_rptr):0, (SKP_int16*)om->b_wptr, &len );
if( ret ) {
ms_error( "SKP_Silk_SDK_Decode returned %d", ret );
ms_free(om);
......@@ -75,13 +77,18 @@ static void decode(MSFilter *f, mblk_t *im) {
/*need to initialize the time*/
ms_concealer_context_set_sampling_time(&obj->concealer,f->ticker->time);
}
obj->sequence_number = im?mblk_get_cseq(im):++obj->sequence_number;
ms_concealer_context_set_sampling_time(&obj->concealer,(im?ms_concealer_context_get_sampling_time(&obj->concealer):f->ticker->time)+20);
}
static void filter_process(MSFilter *f){
struct silk_dec_struct* obj= (struct silk_dec_struct*) f->data;
mblk_t *im;
mblk_t* im;
mblk_t* fec_im;
int i;
SKP_int16 n_bytes_fec=0;
while((im=ms_queue_get(f->inputs[0]))) {
do {
......@@ -91,7 +98,29 @@ static void filter_process(MSFilter *f){
}
if (ms_concealer_context_is_concealement_required(&obj->concealer, f->ticker->time)) {
decode(f,NULL);
//first try fec
if (obj->rtp_picker_context.picker) {
fec_im = allocb(obj->control.API_sampleRate*4/100,0);/*probbaly too big*/
for (i=0;i<2;i++) {
im = obj->rtp_picker_context.picker(&obj->rtp_picker_context,obj->sequence_number+i+1);
if (im) {
SKP_Silk_SDK_search_for_LBRR( im->b_rptr, im->b_wptr - im->b_rptr, i + 1, (SKP_uint8*)fec_im->b_wptr, &n_bytes_fec );
if (n_bytes_fec>0) {
ms_message("Silk dec, got fec from jitter buffer");
fec_im->b_wptr+=n_bytes_fec;
mblk_set_cseq(fec_im,obj->sequence_number+1);
break;
}
}
}
if (n_bytes_fec ==0) {
/*too bad no fec packet found*/
freeb(fec_im);
fec_im=NULL;
}
}
decode(f,fec_im); /*ig fec_im == NULL, plc*/
}
}
......@@ -122,7 +151,7 @@ static int filter_set_sample_rate(MSFilter *f, void *arg) {
obj->control.API_sampleRate=*(SKP_int32*)arg;
break;
default:
ms_warning("unsupported output sampling rate [%i] for silk, using 44 000",*(SKP_int32*)arg);
ms_warning("Unsupported output sampling rate [%i] for silk, using 44 000",*(SKP_int32*)arg);
obj->control.API_sampleRate=44000;
}
return 0;
......@@ -133,10 +162,15 @@ static int filter_get_sample_rate(MSFilter *f, void *arg) {
*(int*)arg = obj->control.API_sampleRate;
return 0;
}
static int filter_set_rtp_picker(MSFilter *f, void *arg) {
struct silk_dec_struct* obj= (struct silk_dec_struct*) f->data;
obj->rtp_picker_context=*(MSRtpPayloadPickerContext*)arg;
return 0;
}
static MSFilterMethod filter_methods[]={
{ MS_FILTER_SET_SAMPLE_RATE , filter_set_sample_rate },
{ MS_FILTER_GET_SAMPLE_RATE , filter_get_sample_rate },
{ MS_FILTER_SET_RTP_PAYLOAD_PICKER,filter_set_rtp_picker},
{ 0, NULL}
};
......
src/silk_enc.c
View file @ f77b325c
......@@ -30,6 +30,8 @@ struct silk_enc_struct {
uint32_t ts;
MSBufferizer *bufferizer;
unsigned char ptime;
unsigned char max_ptime;
unsigned int max_network_bitrate;
};
static void filter_init(MSFilter *f){
......@@ -51,7 +53,11 @@ static void filter_init(MSFilter *f){
ms_error( "SKP_Silk_SDK_InitEncoder returned %i", ret );
}
obj->ptime=20;
obj->max_ptime=100;
obj->bufferizer=ms_bufferizer_new();
obj->control.useInBandFEC=1;
obj->control.complexity=1;
obj->control.packetLossPercentage=5;
}
static void filter_preprocess(MSFilter *f){
......@@ -145,29 +151,71 @@ static int filter_add_fmtp(MSFilter *f, void *arg){
const char *fmtp=(const char *)arg;
buf[0] ='\0';
if (fmtp_get_value(fmtp,"ptime",buf,sizeof(buf))){
if (fmtp_get_value(fmtp,"maxptime:",buf,sizeof(buf))){
obj->max_ptime=atoi(buf);
if (obj->max_ptime <20 || obj->max_ptime >100 ) {
ms_warning("MSSilkEnc unknown value [%i] for maxptime, use default value (100) instead",obj->max_ptime);
obj->max_ptime=100;
}
ms_message("MSSilkEnc: got useinbandfec=%i",obj->max_ptime);
} else if (fmtp_get_value(fmtp,"ptime",buf,sizeof(buf))){
obj->ptime=atoi(buf);
if (obj->ptime >100) {
obj->ptime=100;
if (obj->ptime > obj->max_ptime) {
obj->ptime=obj->max_ptime;
} else if (obj->ptime%20) {
//if the ptime is not a mulptiple of 20, go to the next multiple
obj->ptime = obj->ptime - obj->ptime%20 + 20;
}
ms_message("MSSilkEnc: got ptime=%i",obj->ptime);
}
} else if (fmtp_get_value(fmtp,"useinbandfec",buf,sizeof(buf))){
obj->control.useInBandFEC=atoi(buf);
if (obj->control.useInBandFEC != 0 && obj->control.useInBandFEC != 1) {
ms_warning("MSSilkEnc unknown value [%i] for useinbandfec, use default value (0) instead",obj->control.useInBandFEC);
obj->control.useInBandFEC=1;
}
ms_message("MSSilkEnc: got useinbandfec=%i",obj->control.useInBandFEC);
}
return 0;
}
static int filter_set_bitrate(MSFilter *f, void *arg){
struct silk_enc_struct* obj= (struct silk_enc_struct*) f->data;
obj->control.bitRate=*(int*)arg;
int inital_cbr=0;
int normalized_cbr=0;
int pps=1000/obj->ptime;
obj->max_network_bitrate=*(int*)arg;
normalized_cbr=inital_cbr=(int)( ((((float)obj->max_network_bitrate)/(pps*8))-20-12-8)*pps*8);
switch(obj->control.maxInternalSampleRate) {
case 8000:
normalized_cbr=MIN(normalized_cbr,20000);
normalized_cbr=MAX(normalized_cbr,5000);
break;
case 12000:
normalized_cbr=MIN(normalized_cbr,25000);
normalized_cbr=MAX(normalized_cbr,7000);
break;
case 16000:
normalized_cbr=MIN(normalized_cbr,32000);
normalized_cbr=MAX(normalized_cbr,8000);
break;
case 24000:
normalized_cbr=MIN(normalized_cbr,40000);
normalized_cbr=MAX(normalized_cbr,20000);
break;
}
if (normalized_cbr!=inital_cbr) {
ms_warning("Silk enc unsupported codec bitrate [%i], normalizing",inital_cbr);
}
obj->control.bitRate=normalized_cbr;
ms_message("Setting silk codec birate to [%i] from network bitrate [%i] with ptime [%i]",obj->control.bitRate,obj->max_network_bitrate,obj->ptime);
return 0;
}
static int filter_get_bitrate(MSFilter *f, void *arg){
struct silk_enc_struct* obj= (struct silk_enc_struct*) f->data;
*(int*)arg=obj->control.bitRate;
*(int*)arg=obj->max_network_bitrate;
return 0;
}
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment