Commit 870dddd9 authored by Simon Morlat's avatar Simon Morlat
Browse files

Audio flowcontrol algorithm improved: try to eliminate silent audio buffers...

Audio flowcontrol algorithm improved: try to eliminate silent audio buffers first, then use zero-crossing technique. This should cause sample drops to be more silent.
parent 66748ac4
......@@ -20,9 +20,18 @@
#ifndef flowcontrol_h
#define flowcontrol_h
typedef enum _MSAudioFlowControlStrategy{
MSAudioFlowControlBasic, /**< Immediately drop requested number of samples */
MSAudioFlowControlSoft /**< Elimate silent frames first, use zero-crossing sample deletion */
}MSAudioFlowControlStrategy;
typedef struct _MSAudioFlowControlConfig{
MSAudioFlowControlStrategy strategy;
float silent_threshold; /**< threshold under which a frame is considered as silent (linear), used by "soft" strategy */
}MSAudioFlowControlConfig;
typedef struct _MSAudioFlowController {
MSAudioFlowControlConfig config;
uint32_t target_samples;
uint32_t total_samples;
uint32_t current_pos;
......@@ -36,6 +45,8 @@ extern "C"{
MS2_PUBLIC void ms_audio_flow_controller_init(MSAudioFlowController *ctl);
MS2_PUBLIC void ms_audio_flow_controller_set_config(MSAudioFlowController *ctl, const MSAudioFlowControlConfig *config);
MS2_PUBLIC void ms_audio_flow_controller_set_target(MSAudioFlowController *ctl, uint32_t samples_to_drop, uint32_t total_samples);
MS2_PUBLIC mblk_t *ms_audio_flow_controller_process(MSAudioFlowController *ctl, mblk_t *m);
......@@ -50,10 +61,22 @@ typedef struct _MSAudioFlowControlDropEvent{
} MSAudioFlowControlDropEvent;
/**
* Event sent by the filter each time some samples need to be dropped.
* Event than can be emitted by any filter each time some samples need to be dropped.
* @FIXME It is badly named. It should belong to MSFilter base class or to a specific audio interface.
**/
#define MS_AUDIO_FLOW_CONTROL_DROP_EVENT MS_FILTER_EVENT(MS_AUDIO_FLOW_CONTROL_ID, 0, MSAudioFlowControlDropEvent)
/**
* Set configuration of the flow controller. It can be changed at run-time.
*/
#define MS_AUDIO_FLOW_CONTROL_SET_CONFIG MS_FILTER_METHOD(MS_AUDIO_FLOW_CONTROL_ID, 0, MSAudioFlowControlConfig)
/**
* Request to drop samples.
*/
#define MS_AUDIO_FLOW_CONTROL_DROP MS_FILTER_METHOD(MS_AUDIO_FLOW_CONTROL_ID, 1, MSAudioFlowControlDropEvent)
#ifdef __cplusplus
}
......
......@@ -22,15 +22,26 @@
#include "mediastreamer2/msticker.h"
#include "mediastreamer2/flowcontrol.h"
#include <math.h>
void ms_audio_flow_controller_init(MSAudioFlowController *ctl)
{
static void ms_audio_flow_controller_reset(MSAudioFlowController *ctl){
ctl->target_samples = 0;
ctl->total_samples = 0;
ctl->current_pos = 0;
ctl->current_dropped = 0;
}
void ms_audio_flow_controller_init(MSAudioFlowController *ctl){
ctl->config.strategy = MSAudioFlowControlSoft;
ctl->config.silent_threshold = 0.02f;
ms_audio_flow_controller_reset(ctl);
}
void ms_audio_flow_controller_set_config(MSAudioFlowController *ctl, const MSAudioFlowControlConfig *cfg) {
ctl->config = *cfg;
ms_message("MSAudioFlowControl: configured with strategy=[%i] and silent_threshold=[%f].", cfg->strategy, cfg->silent_threshold);
}
void ms_audio_flow_controller_set_target(MSAudioFlowController *ctl, uint32_t samples_to_drop, uint32_t total_samples)
{
ctl->target_samples = samples_to_drop;
......@@ -79,6 +90,18 @@ static bool_t ms_audio_flow_controller_running(MSAudioFlowController *ctl) {
return (ctl->total_samples > 0) && (ctl->target_samples > 0);
}
static const float max_e = (32768* 0.7f); /* 0.7 - is RMS factor **/
static float compute_frame_power(int16_t *samples, uint32_t nsamples){
float acc = 0;
uint32_t i;
for (i = 0; i < nsamples; ++i){
int sample = samples[i];
acc += (float) (sample*sample);
}
return sqrtf(acc / (float)nsamples) / max_e;
}
mblk_t *ms_audio_flow_controller_process(MSAudioFlowController *ctl, mblk_t *m){
if (ms_audio_flow_controller_running(ctl)) {
uint32_t nsamples = (uint32_t)((m->b_wptr - m->b_rptr) / 2);
......@@ -86,19 +109,35 @@ mblk_t *ms_audio_flow_controller_process(MSAudioFlowController *ctl, mblk_t *m){
uint32_t todrop;
ctl->current_pos += nsamples;
th_dropped = (uint32_t)(((uint64_t)ctl->target_samples * (uint64_t)ctl->current_pos) / (uint64_t)ctl->total_samples);
todrop = (th_dropped > ctl->current_dropped) ? (th_dropped - ctl->current_dropped) : 0;
if (todrop > 0) {
if ((todrop * 8) < nsamples) {
discard_well_choosed_samples(m, nsamples, todrop);
} else {
ms_warning("Too many samples to drop, dropping entire frame.");
if (ctl->config.strategy == MSAudioFlowControlBasic){
if (ctl->current_dropped + nsamples <= ctl->target_samples){
freemsg(m);
ctl->current_dropped += nsamples;
//ms_message("MSAudioFlowControl: dropped %i samples in basic strategy.", (int) nsamples);
m = NULL;
todrop = nsamples;
}
/*ms_message("th_dropped=%i, current_dropped=%i, %i samples dropped", th_dropped, ctl->current_dropped, todrop);*/
ctl->current_dropped += todrop;
}else{
th_dropped = (uint32_t)(((uint64_t)ctl->target_samples * (uint64_t)ctl->current_pos) / (uint64_t)ctl->total_samples);
todrop = (th_dropped > ctl->current_dropped) ? (th_dropped - ctl->current_dropped) : 0;
if (todrop > 0) {
if (nsamples <= ctl->target_samples && compute_frame_power((int16_t*)m->b_rptr, nsamples) < ctl->config.silent_threshold){
/* This frame is almost silent, let's drop it entirely, it won't be noticeable.*/
//ms_message("MSAudioFlowControl: dropping silent frame.");
freemsg(m);
m = NULL;
todrop = nsamples;
}else if ((todrop * 8) < nsamples) {
/* eliminate samples at zero-crossing */
discard_well_choosed_samples(m, nsamples, todrop);
} else {
ms_warning("MSAudioFlowControl: too many samples to drop, dropping entire frame.");
freemsg(m);
m = NULL;
todrop = nsamples;
}
/*ms_message("th_dropped=%i, current_dropped=%i, %i samples dropped", th_dropped, ctl->current_dropped, todrop);*/
ctl->current_dropped += todrop;
}
}
if (ctl->current_pos >= ctl->total_samples) ctl->target_samples = 0; /*stop discarding*/
}
......@@ -117,24 +156,27 @@ typedef struct MSAudioFlowControlState {
static void ms_audio_flow_control_init(MSFilter *f) {
MSAudioFlowControlState *s = ms_new0(MSAudioFlowControlState, 1);
ms_audio_flow_controller_init(&s->afc);
f->data = s;
}
static void ms_audio_flow_control_preprocess(MSFilter *f) {
MSAudioFlowControlState *s = (MSAudioFlowControlState *)f->data;
ms_audio_flow_controller_init(&s->afc);
ms_audio_flow_controller_reset(&s->afc);
}
static void ms_audio_flow_control_process(MSFilter *f) {
MSAudioFlowControlState *s = (MSAudioFlowControlState *)f->data;
mblk_t *m;
ms_filter_lock(f);
while((m = ms_queue_get(f->inputs[0])) != NULL) {
m = ms_audio_flow_controller_process(&s->afc, m);
if (m) {
ms_queue_put(f->outputs[0], m);
}
}
ms_filter_unlock(f);
}
static void ms_audio_flow_control_postprocess(MSFilter *f) {
......@@ -147,20 +189,25 @@ static void ms_audio_flow_control_uninit(MSFilter *f) {
}
void ms_audio_flow_control_event_handler(void *user_data, MSFilter *source, unsigned int event, void *eventdata) {
if (event == MS_AUDIO_FLOW_CONTROL_DROP_EVENT) {
MSFilter *f = (MSFilter *)user_data;
MSAudioFlowControlState *s = (MSAudioFlowControlState *)f->data;
MSAudioFlowControlDropEvent *ev = (MSAudioFlowControlDropEvent *)eventdata;
if (!ms_audio_flow_controller_running(&s->afc)) {
ms_warning("Too much buffered audio signal, throwing out %u ms", ev->drop_ms);
ms_audio_flow_controller_set_target(&s->afc,
(ev->drop_ms * s->samplerate * s->nchannels) / 1000,
(ev->flow_control_interval_ms * s->samplerate * s->nchannels) / 1000);
}
}
static int ms_audio_flow_control_set_config(MSFilter *f, void *arg) {
MSAudioFlowControlState *s = (MSAudioFlowControlState *)f->data;
ms_audio_flow_controller_set_config(&s->afc, (MSAudioFlowControlConfig*) arg);
return 0;
}
static int ms_audio_flow_control_drop(MSFilter *f, void *arg) {
MSAudioFlowControlState *s = (MSAudioFlowControlState *)f->data;
MSAudioFlowControlDropEvent *ctl = (MSAudioFlowControlDropEvent *)arg;
ms_filter_lock(f);
if (!ms_audio_flow_controller_running(&s->afc)) {
ms_message("MSAudioFlowControl: requested to drop %i ms ", (int)ctl->drop_ms);
ms_audio_flow_controller_set_target(&s->afc,
(ctl->drop_ms * s->samplerate * s->nchannels) / 1000,
(ctl->flow_control_interval_ms * s->samplerate * s->nchannels) / 1000);
}
ms_filter_unlock(f);
return 0;
}
static int ms_audio_flow_control_set_sample_rate(MSFilter *f, void *arg) {
MSAudioFlowControlState *s = (MSAudioFlowControlState *)f->data;
......@@ -187,6 +234,8 @@ static int ms_audio_flow_control_get_nchannels(MSFilter *f, void *arg) {
}
static MSFilterMethod ms_audio_flow_control_methods[] = {
{ MS_AUDIO_FLOW_CONTROL_SET_CONFIG, ms_audio_flow_control_set_config },
{ MS_AUDIO_FLOW_CONTROL_DROP, ms_audio_flow_control_drop },
{ MS_FILTER_SET_SAMPLE_RATE, ms_audio_flow_control_set_sample_rate },
{ MS_FILTER_GET_SAMPLE_RATE, ms_audio_flow_control_get_sample_rate },
{ MS_FILTER_SET_NCHANNELS, ms_audio_flow_control_set_nchannels },
......
......@@ -37,6 +37,7 @@
#include "mediastreamer2/msvaddtx.h"
#include "mediastreamer2/msgenericplc.h"
#include "mediastreamer2/mseventqueue.h"
#include "mediastreamer2/flowcontrol.h"
#include "private.h"
#ifdef __ANDROID__
......@@ -813,6 +814,14 @@ static int get_usable_telephone_event(RtpProfile *profile, int clock_rate){
return fallback_pt;
}
static void ms_audio_flow_control_event_handler(void *user_data, MSFilter *source, unsigned int event, void *eventdata) {
if (event == MS_AUDIO_FLOW_CONTROL_DROP_EVENT) {
MSFilter *flow_controller = (MSFilter *)user_data;
MSAudioFlowControlDropEvent *ev = (MSAudioFlowControlDropEvent *)eventdata;
ms_filter_call_method(flow_controller, MS_AUDIO_FLOW_CONTROL_DROP, ev);
}
}
int audio_stream_start_from_io(AudioStream *stream, RtpProfile *profile, const char *rem_rtp_ip, int rem_rtp_port,
const char *rem_rtcp_ip, int rem_rtcp_port, int payload, const MSMediaStreamIO *io) {
RtpSession *rtps=stream->ms.sessions.rtp_session;
......
......@@ -104,7 +104,6 @@ void register_video_preset_high_fps(MSVideoPresetsManager *manager);
MSFilter *_ms_create_av_player(const char *filename, MSFactory* factory);
void video_recorder_handle_event(void *userdata, MSFilter *recorder, unsigned int event, void *event_arg);
void ms_audio_flow_control_event_handler(void *user_data, MSFilter *f, unsigned int event, void *eventdata);
void video_stream_enable_recording(VideoStream *stream, bool_t enabled);
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment