Commit b6387cdd authored by Ghislain MARY's avatar Ghislain MARY
Browse files

Add methods to enable AVPF in video encoders and decoders and use it in VP8 filter.

parent 038ff57f
......@@ -205,6 +205,8 @@ typedef enum _MSRecorderState MSRecorderState;
MS_FILTER_EVENT(MSFilterVideoDecoderInterface, 4, const MSVideoCodecRPSI *)
#define MS_VIDEO_DECODER_RESET_FIRST_IMAGE_NOTIFICATION \
MS_FILTER_METHOD_NO_ARG(MSFilterVideoDecoderInterface, 0)
#define MS_VIDEO_DECODER_ENABLE_AVPF \
MS_FILTER_METHOD(MSFilterVideoDecoderInterface, 1, bool_t)
/** Interface definitions for video capture */
#define MS_VIDEO_CAPTURE_SET_DEVICE_ORIENTATION \
......@@ -240,6 +242,8 @@ typedef enum _MSRecorderState MSRecorderState;
MS_FILTER_METHOD(MSFilterVideoEncoderInterface, 6, const MSVideoCodecSLI *)
#define MS_VIDEO_ENCODER_NOTIFY_RPSI \
MS_FILTER_METHOD(MSFilterVideoEncoderInterface, 7, const MSVideoCodecRPSI *)
#define MS_VIDEO_ENCODER_ENABLE_AVPF \
MS_FILTER_METHOD(MSFilterVideoEncoderInterface, 8, bool_t)
/** Interface definitions for audio capture */
/* Start numbering from the end for hacks */
......
......@@ -85,41 +85,54 @@ typedef struct EncState {
int last_fir_seq_nr;
uint8_t picture_id;
bool_t force_keyframe;
bool_t avpf_enabled;
bool_t ready;
} EncState;
static void enc_init(MSFilter *f) {
EncState *s = (EncState *)ms_new0(EncState, 1);
MSVideoSize vsize;
vpx_codec_err_t res;
vpx_codec_caps_t caps;
s->iface = vpx_codec_vp8_cx();
s->flags = 0;
ms_message("Using %s", vpx_codec_iface_name(s->iface));
if (ms_get_cpu_count() > 1) s->vconf_list = &multicore_vp8_conf_list[0];
else s->vconf_list = &vp8_conf_list[0];
MS_VIDEO_SIZE_ASSIGN(vsize, CIF);
s->vconf = ms_video_find_best_configuration_for_size(s->vconf_list, vsize);
s->frame_count = 0;
s->last_fir_seq_nr = -1;
s->picture_id = random() & 0x7F;
s->avpf_enabled = FALSE;
f->data = s;
}
static void enc_uninit(MSFilter *f) {
EncState *s = (EncState *)f->data;
ms_free(s);
}
static void enc_preprocess(MSFilter *f) {
EncState *s = (EncState *)f->data;
vpx_codec_err_t res;
vpx_codec_caps_t caps;
int cpuused;
/* Populate encoder configuration */
s->flags = 0;
caps = vpx_codec_get_caps(s->iface);
if (caps & VPX_CODEC_CAP_OUTPUT_PARTITION) {
if ((s->avpf_enabled == TRUE) && (caps & VPX_CODEC_CAP_OUTPUT_PARTITION)) {
s->flags |= VPX_CODEC_USE_OUTPUT_PARTITION;
}
res = vpx_codec_enc_config_default(s->iface, &s->cfg, 0);
if(res) {
if (res) {
ms_error("Failed to get config: %s", vpx_codec_err_to_string(res));
return;
}
if (ms_get_cpu_count() > 1) s->vconf_list = &multicore_vp8_conf_list[0];
else s->vconf_list = &vp8_conf_list[0];
MS_VIDEO_SIZE_ASSIGN(vsize, CIF);
s->vconf = ms_video_find_best_configuration_for_size(s->vconf_list, vsize);
s->frame_count = 0;
s->last_fir_seq_nr = -1;
s->cfg.g_w = s->vconf.vsize.width;
s->cfg.g_h = s->vconf.vsize.height;
/* encoder automatically places keyframes */
s->cfg.kf_mode = VPX_KF_AUTO;
s->cfg.rc_target_bitrate = ((float)s->vconf.required_bitrate)*0.92/1024.0; //0.9=take into account IP/UDP/RTP overhead, in average.
s->cfg.rc_target_bitrate = ((float)s->vconf.required_bitrate) * 0.92 / 1024.0; //0.9=take into account IP/UDP/RTP overhead, in average.
s->cfg.g_pass = VPX_RC_ONE_PASS; /* -p 1 */
s->cfg.g_timebase.num = 1;
s->cfg.g_timebase.den = s->vconf.fps;
......@@ -133,33 +146,18 @@ static void enc_init(MSFilter *f) {
s->cfg.rc_undershoot_pct = 95; /* --undershoot-pct=95 */
s->cfg.g_error_resilient = VPX_ERROR_RESILIENT_DEFAULT|VPX_ERROR_RESILIENT_PARTITIONS;
s->cfg.g_lag_in_frames = 0;
s->picture_id = random() & 0x7F;
vp8rtpfmt_packer_init(&s->packer);
f->data = s;
}
static void enc_uninit(MSFilter *f) {
EncState *s=(EncState*)f->data;
vp8rtpfmt_packer_uninit(&s->packer);
ms_free(s);
}
static void enc_preprocess(MSFilter *f) {
vpx_codec_err_t res;
EncState *s=(EncState*)f->data;
int cpuused = 11 - s->cfg.g_threads; /*cpu/quality tradeoff: positive values decrease CPU usage at the expense of quality*/
cpuused = 11 - s->cfg.g_threads; /*cpu/quality tradeoff: positive values decrease CPU usage at the expense of quality*/
if (cpuused < 7) cpuused = 7; /*values beneath 7 consume too much CPU*/
s->cfg.g_w = s->vconf.vsize.width;
s->cfg.g_h = s->vconf.vsize.height;
s->cfg.g_timebase.den=s->vconf.fps;
s->cfg.g_timebase.den = s->vconf.fps;
s->cfg.kf_max_dist = (unsigned int)(s->vconf.fps * 10); /* 1 keyframe each 10s. */
/* Initialize codec */
res = vpx_codec_enc_init(&s->codec, s->iface, &s->cfg, s->flags);
if (res) {
ms_error("vpx_codec_enc_init failed: %s (%s)", vpx_codec_err_to_string(res), vpx_codec_error_detail(&s->codec));
return;
}
vpx_codec_control(&s->codec, VP8E_SET_CPUUSED, cpuused);
vpx_codec_control(&s->codec, VP8E_SET_STATIC_THRESHOLD, 0);
......@@ -167,9 +165,12 @@ static void enc_preprocess(MSFilter *f) {
vpx_codec_control(&s->codec, VP8E_SET_MAX_INTRA_BITRATE_PCT, 400); /*limite iFrame size to 4 pframe*/
if (s->flags & VPX_CODEC_USE_OUTPUT_PARTITION) {
vpx_codec_control(&s->codec, VP8E_SET_TOKEN_PARTITIONS, 2); /* Output 4 partitions per frame */
} else {
vpx_codec_control(&s->codec, VP8E_SET_TOKEN_PARTITIONS, 0);
}
s->ready=TRUE;
vp8rtpfmt_packer_init(&s->packer);
s->ready = TRUE;
}
static void enc_process(MSFilter *f) {
......@@ -182,6 +183,14 @@ static void enc_process(MSFilter *f) {
YuvBuf yuv;
ms_filter_lock(f);
if (!s->ready) {
while ((im = ms_queue_get(f->inputs[0])) != NULL) {
freemsg(im);
}
return;
}
while((im=ms_queue_get(f->inputs[0]))!=NULL){
vpx_image_t img;
......@@ -239,13 +248,15 @@ static void enc_process(MSFilter *f) {
}
freemsg(im);
}
ms_filter_unlock(f);
}
static void enc_postprocess(MSFilter *f) {
EncState *s=(EncState*)f->data;
EncState *s = (EncState *)f->data;
if (s->ready) vpx_codec_destroy(&s->codec);
s->ready=FALSE;
vp8rtpfmt_packer_uninit(&s->packer);
s->ready = FALSE;
}
static int enc_set_configuration(MSFilter *f, void *data) {
......@@ -357,6 +368,12 @@ static int enc_get_configuration_list(MSFilter *f, void *data) {
return 0;
}
static int enc_enable_avpf(MSFilter *f, void *data) {
EncState *s = (EncState *)f->data;
s->avpf_enabled = *((bool_t *)data) ? TRUE : FALSE;
return 0;
}
static MSFilterMethod enc_methods[] = {
{ MS_FILTER_SET_VIDEO_SIZE, enc_set_vsize },
{ MS_FILTER_SET_FPS, enc_set_fps },
......@@ -371,10 +388,12 @@ static MSFilterMethod enc_methods[] = {
{ MS_VIDEO_ENCODER_NOTIFY_FIR, enc_notify_fir },
{ MS_VIDEO_ENCODER_GET_CONFIGURATION_LIST, enc_get_configuration_list },
{ MS_VIDEO_ENCODER_SET_CONFIGURATION, enc_set_configuration },
{ MS_VIDEO_ENCODER_ENABLE_AVPF, enc_enable_avpf },
{ 0, NULL }
};
#ifdef _MSC_VER
MSFilterDesc ms_vp8_enc_desc={
MS_VP8_ENC_ID,
"MSVp8Enc",
......@@ -390,7 +409,9 @@ MSFilterDesc ms_vp8_enc_desc={
enc_uninit,
enc_methods
};
#else
MSFilterDesc ms_vp8_enc_desc={
.id=MS_VP8_ENC_ID,
.name="MSVp8Enc",
......@@ -406,6 +427,7 @@ MSFilterDesc ms_vp8_enc_desc={
.uninit=enc_uninit,
.methods=enc_methods
};
#endif
MS_FILTER_DESC_EXPORT(ms_vp8_enc_desc)
......@@ -417,6 +439,7 @@ MS_FILTER_DESC_EXPORT(ms_vp8_enc_desc)
typedef struct DecState {
vpx_codec_ctx_t codec;
vpx_codec_iface_t *iface;
Vp8RtpFmtUnpackerCtx unpacker;
mblk_t *curframe;
long last_cseq; /*last receive sequence number, used to locate missing partition fragment*/
......@@ -428,19 +451,35 @@ typedef struct DecState {
MSQueue q;
MSAverageFPS fps;
bool_t first_image_decoded;
bool_t avpf_enabled;
} DecState;
static void dec_init(MSFilter *f) {
DecState *s=(DecState *)ms_new(DecState,1);
vpx_codec_flags_t flags = 0;
vpx_codec_iface_t *iface = vpx_codec_vp8_dx();
vpx_codec_caps_t caps = vpx_codec_get_caps(iface);
ms_message("Using %s", vpx_codec_iface_name(iface));
s->iface = vpx_codec_vp8_dx();
ms_message("Using %s", vpx_codec_iface_name(s->iface));
s->curframe = NULL;
s->last_error_reported_time = 0;
s->yuv_width = 0;
s->yuv_height = 0;
s->yuv_msg = 0;
ms_queue_init(&s->q);
s->first_image_decoded = FALSE;
s->avpf_enabled = FALSE;
f->data = s;
ms_video_init_average_fps(&s->fps, "VP8 decoder: FPS: %f");
}
static void dec_preprocess(MSFilter* f) {
DecState *s = (DecState *)f->data;
vpx_codec_flags_t flags = 0;
vpx_codec_caps_t caps = vpx_codec_get_caps(s->iface);
/* Initialize codec */
if (caps & VPX_CODEC_CAP_INPUT_FRAGMENTS) {
if ((s->avpf_enabled == TRUE) && (caps & VPX_CODEC_CAP_INPUT_FRAGMENTS)) {
flags |= VPX_CODEC_USE_INPUT_FRAGMENTS;
}
if (caps & VPX_CODEC_CAP_ERROR_CONCEALMENT) {
......@@ -451,26 +490,11 @@ static void dec_init(MSFilter *f) {
flags |= VPX_CODEC_USE_FRAME_THREADING;
}
#endif
if ((caps & VPX_CODEC_CAP_POSTPROC) && (ms_get_cpu_count() > 1)) {
flags |= VPX_CODEC_USE_POSTPROC;
}
if(vpx_codec_dec_init(&s->codec, iface, NULL, flags))
if(vpx_codec_dec_init(&s->codec, s->iface, NULL, flags))
ms_error("Failed to initialize decoder");
vp8rtpfmt_unpacker_init(&s->unpacker, f, (flags & VPX_CODEC_USE_INPUT_FRAGMENTS) ? TRUE : FALSE);
s->curframe = NULL;
s->last_error_reported_time = 0;
s->yuv_width = 0;
s->yuv_height = 0;
s->yuv_msg = 0;
ms_queue_init(&s->q);
s->first_image_decoded = FALSE;
f->data = s;
ms_video_init_average_fps(&s->fps, "VP8 decoder: FPS: %f");
}
vp8rtpfmt_unpacker_init(&s->unpacker, f, s->avpf_enabled, (flags & VPX_CODEC_USE_INPUT_FRAGMENTS) ? TRUE : FALSE);
static void dec_preprocess(MSFilter* f) {
DecState *s=(DecState*)f->data;
s->first_image_decoded = FALSE;
}
......@@ -478,7 +502,6 @@ static void dec_uninit(MSFilter *f) {
DecState *s=(DecState*)f->data;
vpx_codec_destroy(&s->codec);
vp8rtpfmt_unpacker_uninit(&s->unpacker);
if (s->curframe!=NULL)
freemsg(s->curframe);
if (s->yuv_msg)
......@@ -555,12 +578,23 @@ static void dec_process(MSFilter *f) {
}
}
static int reset_first_image(MSFilter* f, void *data) {
DecState *s=(DecState*)f->data;
static void dec_postprocess(MSFilter *f) {
DecState *s = (DecState *)f->data;
vp8rtpfmt_unpacker_uninit(&s->unpacker);
}
static int dec_reset_first_image(MSFilter* f, void *data) {
DecState *s = (DecState *)f->data;
s->first_image_decoded = FALSE;
return 0;
}
static int dec_enable_avpf(MSFilter *f, void *data) {
DecState *s = (DecState *)f->data;
s->avpf_enabled = *((bool_t *)data) ? TRUE : FALSE;
return 0;
}
static int dec_get_vsize(MSFilter *f, void *data) {
DecState *s = (DecState *)f->data;
MSVideoSize *vsize = (MSVideoSize *)data;
......@@ -574,13 +608,15 @@ static int dec_get_vsize(MSFilter *f, void *data) {
return 0;
}
static MSFilterMethod dec_methods[]={
{ MS_VIDEO_DECODER_RESET_FIRST_IMAGE_NOTIFICATION, reset_first_image },
{ MS_FILTER_GET_VIDEO_SIZE, dec_get_vsize },
{ 0 , NULL }
static MSFilterMethod dec_methods[] = {
{ MS_VIDEO_DECODER_RESET_FIRST_IMAGE_NOTIFICATION, dec_reset_first_image },
{ MS_VIDEO_DECODER_ENABLE_AVPF, dec_enable_avpf },
{ MS_FILTER_GET_VIDEO_SIZE, dec_get_vsize },
{ 0, NULL }
};
#ifdef _MSC_VER
MSFilterDesc ms_vp8_dec_desc={
MS_VP8_DEC_ID,
"MSVp8Dec",
......@@ -592,11 +628,13 @@ MSFilterDesc ms_vp8_dec_desc={
dec_init,
dec_preprocess,
dec_process,
NULL,
dec_postprocess,
dec_uninit,
dec_methods
};
#else
MSFilterDesc ms_vp8_dec_desc={
.id=MS_VP8_DEC_ID,
.name="MSVp8Dec",
......@@ -608,9 +646,11 @@ MSFilterDesc ms_vp8_dec_desc={
.init=dec_init,
.preprocess=dec_preprocess,
.process=dec_process,
.postprocess=NULL,
.postprocess=dec_postprocess,
.uninit=dec_uninit,
.methods=dec_methods
};
#endif
MS_FILTER_DESC_EXPORT(ms_vp8_dec_desc)
......@@ -367,6 +367,7 @@ int video_stream_start (VideoStream *stream, RtpProfile *profile, const char *re
MSVideoSize disp_size;
JBParameters jbp;
const int socket_buf_size=2000000;
bool_t avpf_enabled = FALSE;
if (cam==NULL){
cam=ms_web_cam_manager_get_default_cam (
......@@ -378,6 +379,7 @@ int video_stream_start (VideoStream *stream, RtpProfile *profile, const char *re
ms_error("videostream.c: undefined payload type.");
return -1;
}
if (pt->flags & PAYLOAD_TYPE_RTCP_FEEDBACK_ENABLED) avpf_enabled = TRUE;
if ((cam != NULL) && (cam->desc->encode_to_mime_type != NULL) && (cam->desc->encode_to_mime_type(cam, pt->mime_type) == TRUE)) {
stream->source_performs_encoding = TRUE;
......@@ -433,6 +435,7 @@ int video_stream_start (VideoStream *stream, RtpProfile *profile, const char *re
if (pt->send_fmtp){
ms_filter_call_method(stream->ms.encoder,MS_FILTER_ADD_FMTP,pt->send_fmtp);
}
ms_filter_call_method(stream->ms.encoder, MS_VIDEO_ENCODER_ENABLE_AVPF, &avpf_enabled);
if (stream->use_preview_window){
if (stream->rendercb==NULL){
stream->output2=ms_filter_new_from_name (stream->display_name);
......@@ -512,6 +515,7 @@ int video_stream_start (VideoStream *stream, RtpProfile *profile, const char *re
}
if (pt->recv_fmtp!=NULL)
ms_filter_call_method(stream->ms.decoder,MS_FILTER_ADD_FMTP,(void*)pt->recv_fmtp);
ms_filter_call_method(stream->ms.decoder, MS_VIDEO_DECODER_ENABLE_AVPF, &avpf_enabled);
/*force the decoder to output YUV420P */
format=MS_YUV420P;
......
......@@ -248,8 +248,11 @@ static void add_frame(Vp8RtpFmtUnpackerCtx *ctx, MSList **packets_list) {
/* There are no valid partitions in the frame. */
ms_warning("VP8 frame without any valid partition.");
ms_free(frame);
ms_filter_notify_no_arg(ctx->filter, MS_VIDEO_DECODER_DECODING_ERRORS);
ms_filter_notify_no_arg(ctx->filter, MS_VIDEO_DECODER_SEND_PLI);
if (ctx->avpf_enabled == TRUE) {
ms_filter_notify_no_arg(ctx->filter, MS_VIDEO_DECODER_SEND_PLI);
} else {
ms_filter_notify_no_arg(ctx->filter, MS_VIDEO_DECODER_DECODING_ERRORS);
}
}
}
ms_list_free(*packets_list);
......@@ -364,8 +367,11 @@ static void output_valid_partitions(Vp8RtpFmtUnpackerCtx *ctx, MSQueue *out) {
/* Drop frames until the first keyframe is successfully received. */
ms_warning("VP8 frame dropped because keyframe has not been received yet.");
frame->discarded = TRUE;
ms_filter_notify_no_arg(ctx->filter, MS_VIDEO_DECODER_DECODING_ERRORS);
ms_filter_notify_no_arg(ctx->filter, MS_VIDEO_DECODER_SEND_PLI);
if (ctx->avpf_enabled == TRUE) {
ms_filter_notify_no_arg(ctx->filter, MS_VIDEO_DECODER_SEND_PLI);
} else {
ms_filter_notify_no_arg(ctx->filter, MS_VIDEO_DECODER_DECODING_ERRORS);
}
}
} else if (is_frame_marker_present(frame) == TRUE) {
if (is_first_partition_present_in_frame(frame) == TRUE) {
......@@ -387,14 +393,20 @@ static void output_valid_partitions(Vp8RtpFmtUnpackerCtx *ctx, MSQueue *out) {
ms_warning("VP8 frame with some partitions missing/invalid.");
frame->discarded = TRUE;
}
ms_filter_notify_no_arg(ctx->filter, MS_VIDEO_DECODER_DECODING_ERRORS);
ms_filter_notify_no_arg(ctx->filter, MS_VIDEO_DECODER_SEND_PLI);
if (ctx->avpf_enabled == TRUE) {
ms_filter_notify_no_arg(ctx->filter, MS_VIDEO_DECODER_SEND_PLI);
} else {
ms_filter_notify_no_arg(ctx->filter, MS_VIDEO_DECODER_DECODING_ERRORS);
}
} else {
/* Drop the frame for which the first partition is missing. */
ms_warning("VP8 frame without first partition.");
frame->discarded = TRUE;
ms_filter_notify_no_arg(ctx->filter, MS_VIDEO_DECODER_DECODING_ERRORS);
ms_filter_notify_no_arg(ctx->filter, MS_VIDEO_DECODER_SEND_PLI);
if (ctx->avpf_enabled == TRUE) {
ms_filter_notify_no_arg(ctx->filter, MS_VIDEO_DECODER_SEND_PLI);
} else {
ms_filter_notify_no_arg(ctx->filter, MS_VIDEO_DECODER_DECODING_ERRORS);
}
}
} else {
/* The last packet of the frame has not been received.
......@@ -402,8 +414,11 @@ static void output_valid_partitions(Vp8RtpFmtUnpackerCtx *ctx, MSQueue *out) {
ms_warning("VP8 frame without last packet.");
// TODO: Try to get the missing packets at the next iteration of the filter.
frame->discarded = TRUE;
ms_filter_notify_no_arg(ctx->filter, MS_VIDEO_DECODER_DECODING_ERRORS);
ms_filter_notify_no_arg(ctx->filter, MS_VIDEO_DECODER_SEND_PLI);
if (ctx->avpf_enabled == TRUE) {
ms_filter_notify_no_arg(ctx->filter, MS_VIDEO_DECODER_SEND_PLI);
} else {
ms_filter_notify_no_arg(ctx->filter, MS_VIDEO_DECODER_DECODING_ERRORS);
}
}
}
}
......@@ -504,10 +519,11 @@ static Vp8RtpFmtErrorCode parse_payload_descriptor(Vp8RtpFmtPacket *packet) {
}
void vp8rtpfmt_unpacker_init(Vp8RtpFmtUnpackerCtx *ctx, MSFilter *f, bool_t output_partitions) {
void vp8rtpfmt_unpacker_init(Vp8RtpFmtUnpackerCtx *ctx, MSFilter *f, bool_t avpf_enabled, bool_t output_partitions) {
ctx->filter = f;
ctx->frames_list = NULL;
ms_queue_init(&ctx->output_queue);
ctx->avpf_enabled = avpf_enabled;
ctx->output_partitions = output_partitions;
ctx->valid_keyframe_received = FALSE;
ctx->initialized_last_ts = FALSE;
......
......@@ -90,6 +90,7 @@ extern "C"{
MSQueue output_queue;
uint32_t last_ts;
uint32_t ref_cseq;
bool_t avpf_enabled;
bool_t output_partitions;
bool_t valid_keyframe_received;
bool_t initialized_last_ts;
......@@ -105,7 +106,7 @@ extern "C"{
void vp8rtpfmt_packer_uninit(Vp8RtpFmtPackerCtx *ctx);
void vp8rtpfmt_packer_process(Vp8RtpFmtPackerCtx *ctx, MSList *in, MSQueue *out);
void vp8rtpfmt_unpacker_init(Vp8RtpFmtUnpackerCtx *ctx, MSFilter *f, bool_t output_partitions);
void vp8rtpfmt_unpacker_init(Vp8RtpFmtUnpackerCtx *ctx, MSFilter *f, bool_t avpf_enabled, bool_t output_partitions);
void vp8rtpfmt_unpacker_uninit(Vp8RtpFmtUnpackerCtx *ctx);
void vp8rtpfmt_unpacker_process(Vp8RtpFmtUnpackerCtx *ctx, MSQueue *inout);
uint32_t vp8rtpfmt_unpacker_calc_extended_cseq(Vp8RtpFmtUnpackerCtx *ctx, uint16_t cseq);
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment