Commit 5c83d00e authored by Simon Morlat's avatar Simon Morlat

implementation of conferencing filter, and conference API

parent a82dcdbf
......@@ -33,7 +33,8 @@ mediastreamer2_include_HEADERS= ice.h \
mstonedetector.h \
msjava.h \
bitratecontrol.h \
qualityindicator.h
qualityindicator.h \
msconference.h
EXTRA_DIST=$(mediastreamer2_include_HEADERS)
......@@ -23,9 +23,14 @@ Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
typedef struct MSAudioMixerCtl{
int pin;
float gain;
union {
float gain; /**<gain correction */
int active; /**< to mute or unmute the channel */
};
} MSAudioMixerCtl;
#define MS_AUDIO_MIXER_SET_INPUT_GAIN MS_FILTER_METHOD(MS_AUDIO_MIXER_ID,0,MSAudioMixerCtl)
#define MS_AUDIO_MIXER_SET_INPUT_GAIN MS_FILTER_METHOD(MS_AUDIO_MIXER_ID,0,MSAudioMixerCtl)
#define MS_AUDIO_MIXER_SET_ACTIVE MS_FILTER_METHOD(MS_AUDIO_MIXER_ID,1,MSAudioMixerCtl)
#define MS_AUDIO_MIXER_ENABLE_CONFERENCE_MODE MS_FILTER_METHOD(MS_AUDIO_MIXER_ID,2,int)
#endif
/*
mediastreamer2 library - modular sound and video processing and streaming
Copyright (C) 2011 Belledonne Communications SARL
Author: Simon MORLAT (simon.morlat@linphone.org)
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; either version 2
of the License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
*/
/*
* Convenient API to create and manage audio conferences.
*/
#ifndef conference_h
#define conference_h
#include "mediastreamer2/mediastream.h"
struct _MSAudioConference{
MSTicker *ticker;
MSFilter *mixer;
int nmembers;
};
typedef struct _MSAudioConference MSAudioConference;
struct _MSAudioEndpoint{
AudioStream *st;
MSFilter *in_resampler,*out_resampler;
MSCPoint out_cut_point;
MSCPoint in_cut_point;
MSCPoint mixer_in;
MSCPoint mixer_out;
MSAudioConference *conference;
int pin;
bool_t is_remote;
};
typedef struct _MSAudioEndpoint MSAudioEndpoint;
#ifdef __cplusplus
extern "C" {
#endif
MSAudioConference * ms_audio_conference_new(void);
void ms_audio_conference_add_member(MSAudioConference *obj, MSAudioEndpoint *ep);
void ms_audio_conference_remove_member(MSAudioConference *obj, MSAudioEndpoint *ep);
void ms_audio_conference_mute_member(MSAudioConference *obj, MSAudioEndpoint *ep, bool_t muted);
void ms_audio_conference_destroy(MSAudioConference *obj);
MSAudioEndpoint *ms_audio_endpoint_new_from_audio_stream(AudioStream *st, bool_t is_remote);
void ms_audio_endpoint_destroy(MSAudioEndpoint *ep);
#ifdef __cplusplus
}
#endif
#endif
......@@ -22,7 +22,6 @@ Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#include <ortp/str_utils.h>
#include <mediastreamer2/mscommon.h>
/* for the moment these are stupid queues limited to one element*/
typedef struct _MSCPoint{
struct _MSFilter *filter;
......
......@@ -58,7 +58,8 @@ libmediastreamer_la_SOURCES= mscommon.c $(GITVERSION_FILE) \
g722_decode.c g722.h \
g722_encode.c \
msg722.c \
l16.c
l16.c \
audioconference.c
#dummy c++ file to force libtool to use c++ linking (because of msdscap-mingw.cc)
......
/*
mediastreamer2 library - modular sound and video processing and streaming
Copyright (C) 2011 Belledonne Communications SARL
Author: Simon MORLAT (simon.morlat@linphone.org)
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; either version 2
of the License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
*/
#include "mediastreamer2/msconference.h"
#include "mediastreamer2/msaudiomixer.h"
MSAudioConference * ms_audio_conference_new(void){
MSAudioConference *obj=ms_new0(MSAudioConference,1);
obj->ticker=ms_ticker_new();
obj->mixer=ms_filter_new(MS_AUDIO_MIXER_ID);
return obj;
}
static MSCPoint just_before(MSFilter *f){
MSQueue *q;
MSCPoint pnull={0};
if ((q=f->inputs[0])!=NULL){
return q->prev;
}
ms_fatal("No filter before %s",f->desc->name);
return pnull;
}
static MSCPoint just_after(MSFilter *f){
MSQueue *q;
MSCPoint pnull={0};
if ((q=f->outputs[0])!=NULL){
return q->next;
}
ms_fatal("No filter after %s",f->desc->name);
return pnull;
}
static void cut_audio_stream_graph(MSAudioEndpoint *ep){
AudioStream *st=ep->st;
/*stop the audio graph*/
ms_ticker_detach(st->ticker,st->soundread);
if (!st->ec) ms_ticker_detach(st->ticker,st->soundwrite);
ep->in_cut_point=just_after(st->decoder);
ms_filter_unlink(st->decoder,0,ep->in_cut_point.filter, ep->in_cut_point.pin);
ep->out_cut_point=just_before(st->encoder);
ms_filter_unlink(ep->out_cut_point.filter,ep->out_cut_point.pin,st->encoder,0);
}
static void redo_audio_stream_graph(MSAudioEndpoint *ep){
AudioStream *st=ep->st;
ms_filter_link(st->decoder,0,ep->in_cut_point.filter,ep->in_cut_point.pin);
ms_filter_link(ep->out_cut_point.filter,ep->out_cut_point.pin,st->encoder,0);
ms_ticker_attach(st->ticker,st->soundread);
if (!st->ec)
ms_ticker_attach(st->ticker,st->soundwrite);
}
static int find_free_pin(MSFilter *mixer){
int i;
for(i=0;i<mixer->desc->ninputs;++i){
if (mixer->inputs[i]==NULL){
return i;
}
}
ms_fatal("No more free pin in mixer filter");
return -1;
}
static void plumb_to_conf(MSAudioEndpoint *ep){
MSAudioConference *conf=ep->conference;
AudioStream *st=ep->st;
ep->pin=find_free_pin(conf->mixer);
if (ep->is_remote){
ep->mixer_in.filter=st->decoder;
ep->mixer_in.pin=0;
ep->mixer_out.filter=st->encoder;
ep->mixer_out.pin=0;
}else{
ep->mixer_in=ep->out_cut_point;
ep->mixer_out=ep->in_cut_point;
}
ms_filter_link(ep->mixer_in.filter,ep->mixer_in.pin,ep->in_resampler,0);
ms_filter_link(ep->in_resampler,0,conf->mixer,ep->pin);
ms_filter_link(conf->mixer,ep->pin,ep->out_resampler,0);
ms_filter_link(ep->out_resampler,0,ep->mixer_out.filter,ep->mixer_out.pin);
}
void ms_audio_conference_add_member(MSAudioConference *obj, MSAudioEndpoint *ep){
/*disconnect between the local and remote part*/
cut_audio_stream_graph(ep);
/* and now connect to the mixer */
ep->conference=obj;
if (obj->nmembers>0) ms_ticker_detach(obj->ticker,obj->mixer);
plumb_to_conf(ep);
ms_ticker_attach(obj->ticker,obj->mixer);
obj->nmembers++;
}
static void unplumb_from_conf(MSAudioEndpoint *ep){
MSAudioConference *conf=ep->conference;
ms_filter_unlink(ep->mixer_in.filter,ep->mixer_in.pin,ep->in_resampler,0);
ms_filter_unlink(ep->in_resampler,0,conf->mixer,ep->pin);
ms_filter_unlink(conf->mixer,ep->pin,ep->out_resampler,0);
ms_filter_unlink(ep->out_resampler,0,ep->mixer_out.filter,ep->mixer_out.pin);
}
void ms_audio_conference_remove_member(MSAudioConference *obj, MSAudioEndpoint *ep){
ms_ticker_detach(obj->ticker,obj->mixer);
unplumb_from_conf(ep);
ep->conference=NULL;
obj->nmembers--;
if (obj->nmembers>0) ms_ticker_attach(obj->ticker,obj->mixer);
/*re-plumb the normal audio stream graph*/
redo_audio_stream_graph(ep);
}
void ms_audio_conference_mute_member(MSAudioConference *obj, MSAudioEndpoint *ep, bool_t muted){
}
void ms_audio_conference_destroy(MSAudioConference *obj){
ms_ticker_destroy(obj->ticker);
ms_filter_destroy(obj->mixer);
ms_free(obj);
}
MSAudioEndpoint *ms_audio_endpoint_new_from_audio_stream(AudioStream *st, bool_t is_remote){
MSAudioEndpoint *ep=ms_new0(MSAudioEndpoint,1);
ep->st=st;
ep->is_remote=is_remote;
return ep;
}
void ms_audio_endpoint_destroy(MSAudioEndpoint *ep){
if (ep->in_resampler) ms_filter_destroy(ep->in_resampler);
if (ep->out_resampler) ms_filter_destroy(ep->out_resampler);
ms_free(ep);
}
......@@ -30,16 +30,99 @@ Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#define MAX_LATENCY 0.08
#define ALWAYS_STREAMOUT 1
static void accumulate(int32_t *sum, int16_t* contrib, int nwords){
int i;
for(i=0;i<nwords;++i){
sum[i]+=contrib[i];
}
}
static inline int16_t saturate(int32_t s){
if (s>32767) return 32767;
if (s<-32767) return -32767;
return (int16_t)s;
}
static void apply_gain(int16_t *samples, int nsamples, float gain){
int i;
for(i=0;i<nsamples;++i){
samples[i]=saturate((int)(gain*(float)samples[i]));
}
}
typedef struct Channel{
MSBufferizer bufferizer;
int16_t *input; /*the channel contribution, for removal at output*/
float gain;
int active;
} Channel;
static void channel_init(Channel *chan){
ms_bufferizer_init(&chan->bufferizer);
chan->input=NULL;
chan->gain=1.0;
chan->active=1;
}
static void channel_prepare(Channel *chan, int bytes_per_tick){
chan->input=ms_malloc0(bytes_per_tick);
}
static int channel_process_in(Channel *chan, MSQueue *q, int32_t *sum, int nsamples){
ms_bufferizer_put_from_queue(&chan->bufferizer,q);
if (ms_bufferizer_read(&chan->bufferizer,(uint8_t*)chan->input,nsamples*2)!=0){
if (chan->active){
if (chan->gain!=1.0){
apply_gain(chan->input,nsamples,chan->gain);
accumulate(sum,chan->input,nsamples);
}
}
return nsamples;
}else memset(chan->input,0,nsamples*2);
return 0;
}
static mblk_t *channel_process_out(Channel *chan, int32_t *sum, int nsamples){
int i;
mblk_t *om=allocb(nsamples*2,0);
int16_t *out=(int16_t*)om->b_wptr;
if (chan->active){
/*remove own contribution from sum*/
for(i=0;i<nsamples;++i){
out[i]=saturate(sum[i]-(int32_t)chan->input[i]);
}
}else{
for(i=0;i<nsamples;++i){
out[i]=saturate(sum[i]);
}
}
om->b_wptr+=nsamples*2;
return om;
}
static void channel_unprepare(Channel *chan){
ms_free(chan->input);
chan->input=NULL;
}
static void channel_uninit(Channel *chan){
ms_bufferizer_uninit(&chan->bufferizer);
}
typedef struct MixerState{
int nchannels;
int rate;
int purgeoffset;
int bytespertick;
MSBufferizer channels[MIXER_MAX_CHANNELS];
float gains[MIXER_MAX_CHANNELS];
Channel channels[MIXER_MAX_CHANNELS];
int32_t *sum;
int conf_mode;
} MixerState;
static void mixer_init(MSFilter *f){
MixerState *s=ms_new0(MixerState,1);
int i;
......@@ -47,8 +130,7 @@ static void mixer_init(MSFilter *f){
s->nchannels=1;
s->rate=44100;
for(i=0;i<MIXER_MAX_CHANNELS;++i){
ms_bufferizer_init(&s->channels[i]);
s->gains[i]=1;
channel_init(&s->channels[i]);
}
f->data=s;
}
......@@ -57,43 +139,31 @@ static void mixer_uninit(MSFilter *f){
int i;
MixerState *s=(MixerState *)f->data;
for(i=0;i<MIXER_MAX_CHANNELS;++i){
ms_bufferizer_uninit(&s->channels[i]);
channel_uninit(&s->channels[i]);
}
ms_free(s);
}
static void mixer_preprocess(MSFilter *f){
MixerState *s=(MixerState *)f->data;
int i;
s->purgeoffset=(int)(MAX_LATENCY*(float)(2*s->nchannels*s->rate));
s->bytespertick=(2*s->nchannels*s->rate*f->ticker->interval)/1000;
s->sum=(int32_t*)ms_malloc0((s->bytespertick/2)*sizeof(int32_t));
for(i=0;i<MIXER_MAX_CHANNELS;++i)
channel_prepare(&s->channels[i],s->bytespertick);
/*ms_message("bytespertick=%i, purgeoffset=%i",s->bytespertick,s->purgeoffset);*/
}
static void mixer_postprocess(MSFilter *f){
MixerState *s=(MixerState *)f->data;
int i;
ms_free(s->sum);
s->sum=NULL;
}
static void accumulate(int32_t *sum, int16_t* contrib, int nwords){
int i;
for(i=0;i<nwords;++i){
sum[i]+=contrib[i];
}
}
static void accumulate_mpy(int32_t *sum, int16_t* contrib, int nwords, float gain){
int i;
for(i=0;i<nwords;++i){
sum[i]+=(int32_t)(gain*(float)contrib[i]);
}
}
static inline int16_t saturate(int32_t s){
if (s>32767) return 32767;
if (s<-32767) return -32767;
return (int16_t)s;
for(i=0;i<MIXER_MAX_CHANNELS;++i)
channel_unprepare(&s->channels[i]);
}
static mblk_t *make_output(int32_t *sum, int nwords){
......@@ -109,35 +179,49 @@ static void mixer_process(MSFilter *f){
MixerState *s=(MixerState *)f->data;
int i;
int nwords=s->bytespertick/2;
uint8_t *tmpbuf=(uint8_t *)alloca(s->bytespertick);
bool_t got_something=FALSE;
memset(s->sum,0,nwords*sizeof(int32_t));
/* read from all inputs and sum everybody */
for(i=0;i<MIXER_MAX_CHANNELS;++i){
MSQueue *q=f->inputs[i];
if (q){
ms_bufferizer_put_from_queue(&s->channels[i],q);
if (ms_bufferizer_get_avail(&s->channels[i])>=s->bytespertick){
ms_bufferizer_read(&s->channels[i],tmpbuf,s->bytespertick);
if (s->gains[i]==1)
accumulate(s->sum,(int16_t*)tmpbuf,nwords);
else
accumulate_mpy(s->sum,(int16_t*)tmpbuf,nwords,s->gains[i]);
if (channel_process_in(&s->channels[i],q,s->sum,nwords))
got_something=TRUE;
}
if (ms_bufferizer_get_avail(&s->channels[i])>s->purgeoffset){
/*FIXME: incorporate the following into the channel and use a better flow control algorithm*/
if (ms_bufferizer_get_avail(&s->channels[i].bufferizer)>s->purgeoffset){
ms_warning("Too much data in channel %i",i);
ms_bufferizer_flush (&s->channels[i]);
ms_bufferizer_flush(&s->channels[i].bufferizer);
}
}
}
#ifdef ALWAYS_STREAMOUT
got_something=TRUE;
#endif
/* compute outputs. In conference mode each one has a different output, because its channel own contribution has to be removed*/
if (got_something){
ms_queue_put(f->outputs[0],make_output(s->sum,nwords));
if (s->conf_mode==0){
mblk_t *om=NULL;
for(i=0;i<MIXER_MAX_CHANNELS;++i){
MSQueue *q=f->outputs[i];
if (q){
if (om==NULL){
om=make_output(s->sum,nwords);
}else{
om=dupb(om);
}
ms_queue_put(q,om);
}
}
}else{
for(i=0;i<MIXER_MAX_CHANNELS;++i){
MSQueue *q=f->outputs[i];
if (q){
ms_queue_put(q,channel_process_out(&s->channels[i],s->sum,nwords));
}
}
}
}
}
......@@ -172,7 +256,24 @@ static int mixer_set_input_gain(MSFilter *f, void *data){
ms_warning("mixer_set_input_gain: invalid pin number %i",ctl->pin);
return -1;
}
s->gains[ctl->pin]=ctl->gain;
s->channels[ctl->pin].gain=ctl->gain;
return 0;
}
static int mixer_set_active(MSFilter *f, void *data){
MixerState *s=(MixerState *)f->data;
MSAudioMixerCtl *ctl=(MSAudioMixerCtl*)data;
if (ctl->pin<0 || ctl->pin>=MIXER_MAX_CHANNELS){
ms_warning("mixer_set_active_gain: invalid pin number %i",ctl->pin);
return -1;
}
s->channels[ctl->pin].active=ctl->active;
return 0;
}
static int mixer_set_conference_mode(MSFilter *f, void *data){
MixerState *s=(MixerState *)f->data;
s->conf_mode=*(int*)data;
return 0;
}
......@@ -182,6 +283,8 @@ static MSFilterMethod methods[]={
{ MS_FILTER_SET_SAMPLE_RATE, mixer_set_rate },
{ MS_FILTER_GET_SAMPLE_RATE, mixer_get_rate },
{ MS_AUDIO_MIXER_SET_INPUT_GAIN , mixer_set_input_gain },
{ MS_AUDIO_MIXER_SET_ACTIVE , mixer_set_active },
{ MS_AUDIO_MIXER_ENABLE_CONFERENCE_MODE, mixer_set_conference_mode },
{0,NULL}
};
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment