Commit fd41cb43 authored by Justin Ruggles's avatar Justin Ruggles

avconv: improve sample format negotiation for decoder request

This takes into account planar vs. packed when comparing to find the closest
sample format match.
parent 6cb8c854
......@@ -269,22 +269,41 @@ static void update_sample_fmt(AVCodecContext *dec, AVCodec *dec_codec,
dec_codec->sample_fmts[0] != AV_SAMPLE_FMT_NONE &&
dec_codec->sample_fmts[1] != AV_SAMPLE_FMT_NONE) {
const enum AVSampleFormat *p;
int min_dec = -1, min_inc = -1;
int min_dec = INT_MAX, min_inc = INT_MAX;
enum AVSampleFormat dec_fmt = AV_SAMPLE_FMT_NONE;
enum AVSampleFormat inc_fmt = AV_SAMPLE_FMT_NONE;
/* find a matching sample format in the encoder */
for (p = dec_codec->sample_fmts; *p != AV_SAMPLE_FMT_NONE; p++) {
if (*p == enc->sample_fmt) {
dec->request_sample_fmt = *p;
return;
} else if (*p > enc->sample_fmt) {
min_inc = FFMIN(min_inc, *p - enc->sample_fmt);
} else
min_dec = FFMIN(min_dec, enc->sample_fmt - *p);
} else {
enum AVSampleFormat dfmt = av_get_packed_sample_fmt(*p);
enum AVSampleFormat efmt = av_get_packed_sample_fmt(enc->sample_fmt);
int fmt_diff = 32 * abs(dfmt - efmt);
if (av_sample_fmt_is_planar(*p) !=
av_sample_fmt_is_planar(enc->sample_fmt))
fmt_diff++;
if (dfmt == efmt) {
min_inc = fmt_diff;
inc_fmt = *p;
} else if (dfmt > efmt) {
if (fmt_diff < min_inc) {
min_inc = fmt_diff;
inc_fmt = *p;
}
} else {
if (fmt_diff < min_dec) {
min_dec = fmt_diff;
dec_fmt = *p;
}
}
}
}
/* if none match, provide the one that matches quality closest */
dec->request_sample_fmt = min_inc > 0 ? enc->sample_fmt + min_inc :
enc->sample_fmt - min_dec;
dec->request_sample_fmt = min_inc != INT_MAX ? inc_fmt : dec_fmt;
}
}
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment