rdopt.c 83.3 KB
Newer Older
John Koleszar's avatar
John Koleszar committed
1
/*
2
 *  Copyright (c) 2010 The WebM project authors. All Rights Reserved.
John Koleszar's avatar
John Koleszar committed
3
 *
4
 *  Use of this source code is governed by a BSD-style license
5 6
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
7
 *  in the file PATENTS.  All contributing project authors may
8
 *  be found in the AUTHORS file in the root of the source tree.
John Koleszar's avatar
John Koleszar committed
9 10 11 12 13 14 15
 */


#include <stdio.h>
#include <math.h>
#include <limits.h>
#include <assert.h>
16
#include "vpx_config.h"
17
#include "vp8_rtcd.h"
John Koleszar's avatar
John Koleszar committed
18
#include "vp8/common/pragmas.h"
John Koleszar's avatar
John Koleszar committed
19 20 21 22 23
#include "tokenize.h"
#include "treewriter.h"
#include "onyx_int.h"
#include "modecosts.h"
#include "encodeintra.h"
24
#include "pickinter.h"
John Koleszar's avatar
John Koleszar committed
25 26 27 28
#include "vp8/common/entropymode.h"
#include "vp8/common/reconinter.h"
#include "vp8/common/reconintra4x4.h"
#include "vp8/common/findnearmv.h"
29
#include "vp8/common/quant_common.h"
John Koleszar's avatar
John Koleszar committed
30 31
#include "encodemb.h"
#include "quantize.h"
32
#include "vp8/common/variance.h"
John Koleszar's avatar
John Koleszar committed
33
#include "mcomp.h"
Yunqing Wang's avatar
Yunqing Wang committed
34
#include "rdopt.h"
John Koleszar's avatar
John Koleszar committed
35
#include "vpx_mem/vpx_mem.h"
John Koleszar's avatar
John Koleszar committed
36
#include "vp8/common/systemdependent.h"
37 38 39
#if CONFIG_TEMPORAL_DENOISING
#include "denoising.h"
#endif
Scott LaVarnway's avatar
Scott LaVarnway committed
40 41
extern void vp8_update_zbin_extra(VP8_COMP *cpi, MACROBLOCK *x);

John Koleszar's avatar
John Koleszar committed
42 43
#define MAXF(a,b)            (((a) > (b)) ? (a) : (b))

44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62
typedef struct rate_distortion_struct
{
    int rate2;
    int rate_y;
    int rate_uv;
    int distortion2;
    int distortion_uv;
} RATE_DISTORTION;

typedef struct best_mode_struct
{
  int yrd;
  int rd;
  int intra_rd;
  MB_MODE_INFO mbmode;
  union b_mode_info bmodes[16];
  PARTITION_INFO partition;
} BEST_MODE;

63
static const int auto_speed_thresh[17] =
John Koleszar's avatar
John Koleszar committed
64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115
{
    1000,
    200,
    150,
    130,
    150,
    125,
    120,
    115,
    115,
    115,
    115,
    115,
    115,
    115,
    115,
    115,
    105
};

const MB_PREDICTION_MODE vp8_mode_order[MAX_MODES] =
{
    ZEROMV,
    DC_PRED,

    NEARESTMV,
    NEARMV,

    ZEROMV,
    NEARESTMV,

    ZEROMV,
    NEARESTMV,

    NEARMV,
    NEARMV,

    V_PRED,
    H_PRED,
    TM_PRED,

    NEWMV,
    NEWMV,
    NEWMV,

    SPLITMV,
    SPLITMV,
    SPLITMV,

    B_PRED,
};

116 117 118 119
/* This table determines the search order in reference frame priority order,
 * which may not necessarily match INTRA,LAST,GOLDEN,ARF
 */
const int vp8_ref_frame_order[MAX_MODES] =
John Koleszar's avatar
John Koleszar committed
120
{
121 122
    1,
    0,
John Koleszar's avatar
John Koleszar committed
123

124 125
    1,
    1,
John Koleszar's avatar
John Koleszar committed
126

127 128
    2,
    2,
John Koleszar's avatar
John Koleszar committed
129

130 131
    3,
    3,
John Koleszar's avatar
John Koleszar committed
132

133 134
    2,
    3,
John Koleszar's avatar
John Koleszar committed
135

136 137 138
    0,
    0,
    0,
John Koleszar's avatar
John Koleszar committed
139

140 141 142
    1,
    2,
    3,
John Koleszar's avatar
John Koleszar committed
143

144 145 146
    1,
    2,
    3,
John Koleszar's avatar
John Koleszar committed
147

148
    0,
John Koleszar's avatar
John Koleszar committed
149 150 151
};

static void fill_token_costs(
152 153
    int c[BLOCK_TYPES][COEF_BANDS][PREV_COEF_CONTEXTS][MAX_ENTROPY_TOKENS],
    const vp8_prob p[BLOCK_TYPES][COEF_BANDS][PREV_COEF_CONTEXTS][ENTROPY_NODES]
John Koleszar's avatar
John Koleszar committed
154 155 156 157 158 159 160 161
)
{
    int i, j, k;


    for (i = 0; i < BLOCK_TYPES; i++)
        for (j = 0; j < COEF_BANDS; j++)
            for (k = 0; k < PREV_COEF_CONTEXTS; k++)
162

John Koleszar's avatar
John Koleszar committed
163 164 165
                /* check for pt=0 and band > 1 if block type 0
                 * and 0 if blocktype 1
                 */
166 167
                if (k == 0 && j > (i == 0))
                    vp8_cost_tokens2(c[i][j][k], p [i][j][k], vp8_coef_tree, 2);
Jim Bankoski's avatar
Jim Bankoski committed
168
                else
169
                    vp8_cost_tokens(c[i][j][k], p [i][j][k], vp8_coef_tree);
John Koleszar's avatar
John Koleszar committed
170 171
}

172 173 174 175 176 177 178
static const int rd_iifactor[32] =
{
    4, 4, 3, 2, 1, 0, 0, 0,
    0, 0, 0, 0, 0, 0, 0, 0,
    0, 0, 0, 0, 0, 0, 0, 0,
    0, 0, 0, 0, 0, 0, 0, 0
};
John Koleszar's avatar
John Koleszar committed
179

180
/* values are now correlated to quantizer */
181
static const int sad_per_bit16lut[QINDEX_RANGE] =
John Koleszar's avatar
John Koleszar committed
182
{
Yaowu Xu's avatar
Yaowu Xu committed
183 184 185 186 187 188
    2,  2,  2,  2,  2,  2,  2,  2,
    2,  2,  2,  2,  2,  2,  2,  2,
    3,  3,  3,  3,  3,  3,  3,  3,
    3,  3,  3,  3,  3,  3,  4,  4,
    4,  4,  4,  4,  4,  4,  4,  4,
    4,  4,  5,  5,  5,  5,  5,  5,
189
    5,  5,  5,  5,  5,  5,  6,  6,
Yaowu Xu's avatar
Yaowu Xu committed
190 191 192 193 194 195 196 197 198
    6,  6,  6,  6,  6,  6,  6,  6,
    6,  6,  7,  7,  7,  7,  7,  7,
    7,  7,  7,  7,  7,  7,  8,  8,
    8,  8,  8,  8,  8,  8,  8,  8,
    8,  8,  9,  9,  9,  9,  9,  9,
    9,  9,  9,  9,  9,  9,  10, 10,
    10, 10, 10, 10, 10, 10, 11, 11,
    11, 11, 11, 11, 12, 12, 12, 12,
    12, 12, 13, 13, 13, 13, 14, 14
John Koleszar's avatar
John Koleszar committed
199
};
200
static const int sad_per_bit4lut[QINDEX_RANGE] =
John Koleszar's avatar
John Koleszar committed
201
{
Yaowu Xu's avatar
Yaowu Xu committed
202 203 204 205 206 207 208
    2,  2,  2,  2,  2,  2,  3,  3,
    3,  3,  3,  3,  3,  3,  3,  3,
    3,  3,  3,  3,  4,  4,  4,  4,
    4,  4,  4,  4,  4,  4,  5,  5,
    5,  5,  5,  5,  6,  6,  6,  6,
    6,  6,  6,  6,  6,  6,  6,  6,
    7,  7,  7,  7,  7,  7,  7,  7,
209
    7,  7,  7,  7,  7,  8,  8,  8,
Yaowu Xu's avatar
Yaowu Xu committed
210 211 212 213 214 215 216 217
    8,  8,  9,  9,  9,  9,  9,  9,
    10, 10, 10, 10, 10, 10, 10, 10,
    11, 11, 11, 11, 11, 11, 11, 11,
    12, 12, 12, 12, 12, 12, 12, 12,
    13, 13, 13, 13, 13, 13, 13, 14,
    14, 14, 14, 14, 15, 15, 15, 15,
    16, 16, 16, 16, 17, 17, 17, 18,
    18, 18, 19, 19, 19, 20, 20, 20,
John Koleszar's avatar
John Koleszar committed
218 219 220 221
};

void vp8cx_initialize_me_consts(VP8_COMP *cpi, int QIndex)
{
Yaowu Xu's avatar
Yaowu Xu committed
222 223
    cpi->mb.sadperbit16 =  sad_per_bit16lut[QIndex];
    cpi->mb.sadperbit4  =  sad_per_bit4lut[QIndex];
John Koleszar's avatar
John Koleszar committed
224 225
}

226
void vp8_initialize_rd_consts(VP8_COMP *cpi, MACROBLOCK *x, int Qvalue)
John Koleszar's avatar
John Koleszar committed
227 228 229
{
    int q;
    int i;
230
    double capped_q = (Qvalue < 160) ? (double)Qvalue : 160.0;
Jim Bankoski's avatar
Jim Bankoski committed
231
    double rdconst = 2.80;
John Koleszar's avatar
John Koleszar committed
232

John Koleszar's avatar
John Koleszar committed
233
    vp8_clear_system_state();
John Koleszar's avatar
John Koleszar committed
234

John Koleszar's avatar
John Koleszar committed
235 236 237
    /* Further tests required to see if optimum is different
     * for key frames, golden frames and arf frames.
     */
238
    cpi->RDMULT = (int)(rdconst * (capped_q * capped_q));
John Koleszar's avatar
John Koleszar committed
239

John Koleszar's avatar
John Koleszar committed
240
    /* Extend rate multiplier along side quantizer zbin increases */
241
    if (cpi->mb.zbin_over_quant  > 0)
242 243 244 245
    {
        double oq_factor;
        double modq;

John Koleszar's avatar
John Koleszar committed
246
        /* Experimental code using the same basic equation as used for Q above
247
         * The units of cpi->mb.zbin_over_quant are 1/128 of Q bin size
John Koleszar's avatar
John Koleszar committed
248
         */
249
        oq_factor = 1.0 + ((double)0.0015625 * cpi->mb.zbin_over_quant);
250 251 252
        modq = (int)((double)capped_q * oq_factor);
        cpi->RDMULT = (int)(rdconst * (modq * modq));
    }
John Koleszar's avatar
John Koleszar committed
253

Paul Wilkins's avatar
Paul Wilkins committed
254
    if (cpi->pass == 2 && (cpi->common.frame_type != KEY_FRAME))
John Koleszar's avatar
John Koleszar committed
255
    {
256
        if (cpi->twopass.next_iiratio > 31)
Paul Wilkins's avatar
Paul Wilkins committed
257
            cpi->RDMULT += (cpi->RDMULT * rd_iifactor[31]) >> 4;
John Koleszar's avatar
John Koleszar committed
258
        else
259 260
            cpi->RDMULT +=
                (cpi->RDMULT * rd_iifactor[cpi->twopass.next_iiratio]) >> 4;
John Koleszar's avatar
John Koleszar committed
261 262
    }

263
    cpi->mb.errorperbit = (cpi->RDMULT / 110);
264 265
    cpi->mb.errorperbit += (cpi->mb.errorperbit==0);

John Koleszar's avatar
John Koleszar committed
266 267
    vp8_set_speed_features(cpi);

268 269 270 271 272
    for (i = 0; i < MAX_MODES; i++)
    {
        x->mode_test_hit_counts[i] = 0;
    }

John Koleszar's avatar
John Koleszar committed
273 274 275 276 277 278 279 280 281 282 283 284 285 286
    q = (int)pow(Qvalue, 1.25);

    if (q < 8)
        q = 8;

    if (cpi->RDMULT > 1000)
    {
        cpi->RDDIV = 1;
        cpi->RDMULT /= 100;

        for (i = 0; i < MAX_MODES; i++)
        {
            if (cpi->sf.thresh_mult[i] < INT_MAX)
            {
287
                x->rd_threshes[i] = cpi->sf.thresh_mult[i] * q / 100;
John Koleszar's avatar
John Koleszar committed
288 289 290
            }
            else
            {
291
                x->rd_threshes[i] = INT_MAX;
John Koleszar's avatar
John Koleszar committed
292 293
            }

294
            cpi->rd_baseline_thresh[i] = x->rd_threshes[i];
John Koleszar's avatar
John Koleszar committed
295 296 297 298 299 300 301 302 303 304
        }
    }
    else
    {
        cpi->RDDIV = 100;

        for (i = 0; i < MAX_MODES; i++)
        {
            if (cpi->sf.thresh_mult[i] < (INT_MAX / q))
            {
305
                x->rd_threshes[i] = cpi->sf.thresh_mult[i] * q;
John Koleszar's avatar
John Koleszar committed
306 307 308
            }
            else
            {
309
                x->rd_threshes[i] = INT_MAX;
John Koleszar's avatar
John Koleszar committed
310 311
            }

312
            cpi->rd_baseline_thresh[i] = x->rd_threshes[i];
John Koleszar's avatar
John Koleszar committed
313 314 315
        }
    }

316
    {
John Koleszar's avatar
John Koleszar committed
317
      /* build token cost array for the type of frame we have now */
318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334
      FRAME_CONTEXT *l = &cpi->lfc_n;

      if(cpi->common.refresh_alt_ref_frame)
          l = &cpi->lfc_a;
      else if(cpi->common.refresh_golden_frame)
          l = &cpi->lfc_g;

      fill_token_costs(
          cpi->mb.token_costs,
          (const vp8_prob( *)[8][3][11]) l->coef_probs
      );
      /*
      fill_token_costs(
          cpi->mb.token_costs,
          (const vp8_prob( *)[8][3][11]) cpi->common.fc.coef_probs);
      */

John Koleszar's avatar
John Koleszar committed
335

John Koleszar's avatar
John Koleszar committed
336
      /* TODO make these mode costs depend on last,alt or gold too.  (jbb) */
337 338
      vp8_init_mode_costs(cpi);
    }
John Koleszar's avatar
John Koleszar committed
339 340 341 342 343

}

void vp8_auto_select_speed(VP8_COMP *cpi)
{
James Zern's avatar
James Zern committed
344
    int milliseconds_for_compress = (int)(1000000 / cpi->framerate);
John Koleszar's avatar
John Koleszar committed
345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381

    milliseconds_for_compress = milliseconds_for_compress * (16 - cpi->oxcf.cpu_used) / 16;

#if 0

    if (0)
    {
        FILE *f;

        f = fopen("speed.stt", "a");
        fprintf(f, " %8ld %10ld %10ld %10ld\n",
                cpi->common.current_video_frame, cpi->Speed, milliseconds_for_compress, cpi->avg_pick_mode_time);
        fclose(f);
    }

#endif

    if (cpi->avg_pick_mode_time < milliseconds_for_compress && (cpi->avg_encode_time - cpi->avg_pick_mode_time) < milliseconds_for_compress)
    {
        if (cpi->avg_pick_mode_time == 0)
        {
            cpi->Speed = 4;
        }
        else
        {
            if (milliseconds_for_compress * 100 < cpi->avg_encode_time * 95)
            {
                cpi->Speed          += 2;
                cpi->avg_pick_mode_time = 0;
                cpi->avg_encode_time = 0;

                if (cpi->Speed > 16)
                {
                    cpi->Speed = 16;
                }
            }

382
            if (milliseconds_for_compress * 100 > cpi->avg_encode_time * auto_speed_thresh[cpi->Speed])
John Koleszar's avatar
John Koleszar committed
383 384 385 386 387
            {
                cpi->Speed          -= 1;
                cpi->avg_pick_mode_time = 0;
                cpi->avg_encode_time = 0;

John Koleszar's avatar
John Koleszar committed
388 389
                /* In real-time mode, cpi->speed is in [4, 16]. */
                if (cpi->Speed < 4)
John Koleszar's avatar
John Koleszar committed
390
                {
John Koleszar's avatar
John Koleszar committed
391
                    cpi->Speed = 4;
John Koleszar's avatar
John Koleszar committed
392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469
                }
            }
        }
    }
    else
    {
        cpi->Speed += 4;

        if (cpi->Speed > 16)
            cpi->Speed = 16;


        cpi->avg_pick_mode_time = 0;
        cpi->avg_encode_time = 0;
    }
}

int vp8_block_error_c(short *coeff, short *dqcoeff)
{
    int i;
    int error = 0;

    for (i = 0; i < 16; i++)
    {
        int this_diff = coeff[i] - dqcoeff[i];
        error += this_diff * this_diff;
    }

    return error;
}

int vp8_mbblock_error_c(MACROBLOCK *mb, int dc)
{
    BLOCK  *be;
    BLOCKD *bd;
    int i, j;
    int berror, error = 0;

    for (i = 0; i < 16; i++)
    {
        be = &mb->block[i];
        bd = &mb->e_mbd.block[i];

        berror = 0;

        for (j = dc; j < 16; j++)
        {
            int this_diff = be->coeff[j] - bd->dqcoeff[j];
            berror += this_diff * this_diff;
        }

        error += berror;
    }

    return error;
}

int vp8_mbuverror_c(MACROBLOCK *mb)
{

    BLOCK  *be;
    BLOCKD *bd;


    int i;
    int error = 0;

    for (i = 16; i < 24; i++)
    {
        be = &mb->block[i];
        bd = &mb->e_mbd.block[i];

        error += vp8_block_error_c(be->coeff, bd->dqcoeff);
    }

    return error;
}

John Koleszar's avatar
John Koleszar committed
470
int VP8_UVSSE(MACROBLOCK *x)
John Koleszar's avatar
John Koleszar committed
471 472 473 474 475 476 477 478
{
    unsigned char *uptr, *vptr;
    unsigned char *upred_ptr = (*(x->block[16].base_src) + x->block[16].src);
    unsigned char *vpred_ptr = (*(x->block[20].base_src) + x->block[20].src);
    int uv_stride = x->block[16].src_stride;

    unsigned int sse1 = 0;
    unsigned int sse2 = 0;
479 480
    int mv_row = x->e_mbd.mode_info_context->mbmi.mv.as_mv.row;
    int mv_col = x->e_mbd.mode_info_context->mbmi.mv.as_mv.col;
John Koleszar's avatar
John Koleszar committed
481
    int offset;
Scott LaVarnway's avatar
Scott LaVarnway committed
482
    int pre_stride = x->e_mbd.pre.uv_stride;
John Koleszar's avatar
John Koleszar committed
483

484 485 486 487 488 489 490 491 492 493 494 495
    if (mv_row < 0)
        mv_row -= 1;
    else
        mv_row += 1;

    if (mv_col < 0)
        mv_col -= 1;
    else
        mv_col += 1;

    mv_row /= 2;
    mv_col /= 2;
John Koleszar's avatar
John Koleszar committed
496 497 498 499 500 501 502

    offset = (mv_row >> 3) * pre_stride + (mv_col >> 3);
    uptr = x->e_mbd.pre.u_buffer + offset;
    vptr = x->e_mbd.pre.v_buffer + offset;

    if ((mv_row | mv_col) & 7)
    {
John Koleszar's avatar
John Koleszar committed
503
        vp8_sub_pixel_variance8x8(uptr, pre_stride,
504
            mv_col & 7, mv_row & 7, upred_ptr, uv_stride, &sse2);
John Koleszar's avatar
John Koleszar committed
505
        vp8_sub_pixel_variance8x8(vptr, pre_stride,
506
            mv_col & 7, mv_row & 7, vpred_ptr, uv_stride, &sse1);
John Koleszar's avatar
John Koleszar committed
507 508 509 510
        sse2 += sse1;
    }
    else
    {
John Koleszar's avatar
John Koleszar committed
511
        vp8_variance8x8(uptr, pre_stride,
512
            upred_ptr, uv_stride, &sse2);
John Koleszar's avatar
John Koleszar committed
513
        vp8_variance8x8(vptr, pre_stride,
514
            vpred_ptr, uv_stride, &sse1);
John Koleszar's avatar
John Koleszar committed
515 516 517 518 519 520 521 522 523
        sse2 += sse1;
    }
    return sse2;

}

static int cost_coeffs(MACROBLOCK *mb, BLOCKD *b, int type, ENTROPY_CONTEXT *a, ENTROPY_CONTEXT *l)
{
    int c = !type;              /* start at coef 0, unless Y with Y2 */
524
    int eob = (int)(*b->eob);
John Koleszar's avatar
John Koleszar committed
525 526 527 528 529 530
    int pt ;    /* surrounding block/prev coef predictor */
    int cost = 0;
    short *qcoeff_ptr = b->qcoeff;

    VP8_COMBINEENTROPYCONTEXTS(pt, *a, *l);

531
    assert(eob <= 16);
John Koleszar's avatar
John Koleszar committed
532 533
    for (; c < eob; c++)
    {
534 535
        const int v = qcoeff_ptr[vp8_default_zig_zag1d[c]];
        const int t = vp8_dct_value_tokens_ptr[v].Token;
John Koleszar's avatar
John Koleszar committed
536 537 538 539 540 541 542 543
        cost += mb->token_costs [type] [vp8_coef_bands[c]] [pt] [t];
        cost += vp8_dct_value_cost_ptr[v];
        pt = vp8_prev_token_class[t];
    }

    if (c < 16)
        cost += mb->token_costs [type] [vp8_coef_bands[c]] [pt] [DCT_EOB_TOKEN];

John Koleszar's avatar
John Koleszar committed
544
    pt = (c != !type); /* is eob first coefficient; */
John Koleszar's avatar
John Koleszar committed
545 546 547 548 549
    *a = *l = pt;

    return cost;
}

Scott LaVarnway's avatar
Scott LaVarnway committed
550
static int vp8_rdcost_mby(MACROBLOCK *mb)
John Koleszar's avatar
John Koleszar committed
551 552 553 554
{
    int cost = 0;
    int b;
    MACROBLOCKD *x = &mb->e_mbd;
555 556 557 558 559 560
    ENTROPY_CONTEXT_PLANES t_above, t_left;
    ENTROPY_CONTEXT *ta;
    ENTROPY_CONTEXT *tl;

    vpx_memcpy(&t_above, mb->e_mbd.above_context, sizeof(ENTROPY_CONTEXT_PLANES));
    vpx_memcpy(&t_left, mb->e_mbd.left_context, sizeof(ENTROPY_CONTEXT_PLANES));
John Koleszar's avatar
John Koleszar committed
561

562 563
    ta = (ENTROPY_CONTEXT *)&t_above;
    tl = (ENTROPY_CONTEXT *)&t_left;
John Koleszar's avatar
John Koleszar committed
564 565

    for (b = 0; b < 16; b++)
Scott LaVarnway's avatar
Scott LaVarnway committed
566
        cost += cost_coeffs(mb, x->block + b, PLANE_TYPE_Y_NO_DC,
567
                    ta + vp8_block2above[b], tl + vp8_block2left[b]);
John Koleszar's avatar
John Koleszar committed
568

Scott LaVarnway's avatar
Scott LaVarnway committed
569
    cost += cost_coeffs(mb, x->block + 24, PLANE_TYPE_Y2,
Scott LaVarnway's avatar
Scott LaVarnway committed
570
                ta + vp8_block2above[24], tl + vp8_block2left[24]);
John Koleszar's avatar
John Koleszar committed
571 572 573 574

    return cost;
}

575 576
static void macro_block_yrd( MACROBLOCK *mb,
                             int *Rate,
577
                             int *Distortion)
578 579 580 581 582 583 584 585 586
{
    int b;
    MACROBLOCKD *const x = &mb->e_mbd;
    BLOCK   *const mb_y2 = mb->block + 24;
    BLOCKD *const x_y2  = x->block + 24;
    short *Y2DCPtr = mb_y2->src_diff;
    BLOCK *beptr;
    int d;

587
    vp8_subtract_mby( mb->src_diff, *(mb->block[0].base_src),
588
        mb->block[0].src_stride,  mb->e_mbd.predictor, 16);
589

John Koleszar's avatar
John Koleszar committed
590
    /* Fdct and building the 2nd order block */
591 592
    for (beptr = mb->block; beptr < mb->block + 16; beptr += 2)
    {
John Koleszar's avatar
John Koleszar committed
593
        mb->short_fdct8x4(beptr->src_diff, beptr->coeff, 32);
594 595 596 597
        *Y2DCPtr++ = beptr->coeff[0];
        *Y2DCPtr++ = beptr->coeff[16];
    }

John Koleszar's avatar
John Koleszar committed
598
    /* 2nd order fdct */
599 600
    mb->short_walsh4x4(mb_y2->src_diff, mb_y2->coeff, 8);

John Koleszar's avatar
John Koleszar committed
601
    /* Quantization */
602 603 604 605 606
    for (b = 0; b < 16; b++)
    {
        mb->quantize_b(&mb->block[b], &mb->e_mbd.block[b]);
    }

John Koleszar's avatar
John Koleszar committed
607
    /* DC predication and Quantization of 2nd Order block */
608 609
    mb->quantize_b(mb_y2, x_y2);

John Koleszar's avatar
John Koleszar committed
610
    /* Distortion */
611 612
    d = vp8_mbblock_error(mb, 1) << 2;
    d += vp8_block_error(mb_y2->coeff, x_y2->dqcoeff);
613 614 615

    *Distortion = (d >> 4);

John Koleszar's avatar
John Koleszar committed
616
    /* rate */
617 618
    *Rate = vp8_rdcost_mby(mb);
}
John Koleszar's avatar
John Koleszar committed
619

620
static void copy_predictor(unsigned char *dst, const unsigned char *predictor)
621
{
622 623 624 625 626 627
    const unsigned int *p = (const unsigned int *)predictor;
    unsigned int *d = (unsigned int *)dst;
    d[0] = p[0];
    d[4] = p[4];
    d[8] = p[8];
    d[12] = p[12];
628
}
629
static int rd_pick_intra4x4block(
John Koleszar's avatar
John Koleszar committed
630 631 632 633
    MACROBLOCK *x,
    BLOCK *be,
    BLOCKD *b,
    B_PREDICTION_MODE *best_mode,
634
    const int *bmode_costs,
John Koleszar's avatar
John Koleszar committed
635 636 637 638 639 640 641 642
    ENTROPY_CONTEXT *a,
    ENTROPY_CONTEXT *l,

    int *bestrate,
    int *bestratey,
    int *bestdistortion)
{
    B_PREDICTION_MODE mode;
643
    int best_rd = INT_MAX;
John Koleszar's avatar
John Koleszar committed
644 645 646 647 648
    int rate = 0;
    int distortion;

    ENTROPY_CONTEXT ta = *a, tempa = *a;
    ENTROPY_CONTEXT tl = *l, templ = *l;
649 650 651 652 653 654 655
    /*
     * The predictor buffer is a 2d buffer with a stride of 16.  Create
     * a temp buffer that meets the stride requirements, but we are only
     * interested in the left 4x4 block
     * */
    DECLARE_ALIGNED_ARRAY(16, unsigned char,  best_predictor, 16*4);
    DECLARE_ALIGNED_ARRAY(16, short, best_dqcoeff, 16);
Scott LaVarnway's avatar
Scott LaVarnway committed
656
    int dst_stride = x->e_mbd.dst.y_stride;
657 658 659 660 661
    unsigned char *dst = x->e_mbd.dst.y_buffer + b->offset;

    unsigned char *Above = dst - dst_stride;
    unsigned char *yleft = dst - 1;
    unsigned char top_left = Above[-1];
John Koleszar's avatar
John Koleszar committed
662 663 664 665 666 667

    for (mode = B_DC_PRED; mode <= B_HU_PRED; mode++)
    {
        int this_rd;
        int ratey;

668 669
        rate = bmode_costs[mode];

Johann's avatar
Johann committed
670 671
        vp8_intra4x4_predict(Above, yleft, dst_stride, mode,
                             b->predictor, 16, top_left);
672
        vp8_subtract_b(be, b, 16);
John Koleszar's avatar
John Koleszar committed
673
        x->short_fdct4x4(be->src_diff, be->coeff, 32);
674
        x->quantize_b(be, b);
John Koleszar's avatar
John Koleszar committed
675 676 677 678

        tempa = ta;
        templ = tl;

Scott LaVarnway's avatar
Scott LaVarnway committed
679
        ratey = cost_coeffs(x, b, PLANE_TYPE_Y_WITH_DC, &tempa, &templ);
John Koleszar's avatar
John Koleszar committed
680
        rate += ratey;
681
        distortion = vp8_block_error(be->coeff, b->dqcoeff) >> 2;
John Koleszar's avatar
John Koleszar committed
682 683 684 685 686 687 688 689 690 691 692 693

        this_rd = RDCOST(x->rdmult, x->rddiv, rate, distortion);

        if (this_rd < best_rd)
        {
            *bestrate = rate;
            *bestratey = ratey;
            *bestdistortion = distortion;
            best_rd = this_rd;
            *best_mode = mode;
            *a = tempa;
            *l = templ;
694 695
            copy_predictor(best_predictor, b->predictor);
            vpx_memcpy(best_dqcoeff, b->dqcoeff, 32);
John Koleszar's avatar
John Koleszar committed
696 697
        }
    }
698
    b->bmi.as_mode = *best_mode;
699

700
    vp8_short_idct4x4llm(best_dqcoeff, best_predictor, 16, dst, dst_stride);
John Koleszar's avatar
John Koleszar committed
701

702
    return best_rd;
John Koleszar's avatar
John Koleszar committed
703 704
}

705
static int rd_pick_intra4x4mby_modes(MACROBLOCK *mb, int *Rate,
706
                                     int *rate_y, int *Distortion, int best_rd)
John Koleszar's avatar
John Koleszar committed
707 708 709 710 711 712
{
    MACROBLOCKD *const xd = &mb->e_mbd;
    int i;
    int cost = mb->mbmode_cost [xd->frame_type] [B_PRED];
    int distortion = 0;
    int tot_rate_y = 0;
713
    int64_t total_rd = 0;
714 715 716
    ENTROPY_CONTEXT_PLANES t_above, t_left;
    ENTROPY_CONTEXT *ta;
    ENTROPY_CONTEXT *tl;
717
    const int *bmode_costs;
718 719 720 721 722 723

    vpx_memcpy(&t_above, mb->e_mbd.above_context, sizeof(ENTROPY_CONTEXT_PLANES));
    vpx_memcpy(&t_left, mb->e_mbd.left_context, sizeof(ENTROPY_CONTEXT_PLANES));

    ta = (ENTROPY_CONTEXT *)&t_above;
    tl = (ENTROPY_CONTEXT *)&t_left;
John Koleszar's avatar
John Koleszar committed
724

Scott LaVarnway's avatar
Scott LaVarnway committed
725
    intra_prediction_down_copy(xd, xd->dst.y_buffer - xd->dst.y_stride + 16);
John Koleszar's avatar
John Koleszar committed
726

727 728
    bmode_costs = mb->inter_bmode_costs;

John Koleszar's avatar
John Koleszar committed
729 730 731 732 733 734 735
    for (i = 0; i < 16; i++)
    {
        MODE_INFO *const mic = xd->mode_info_context;
        const int mis = xd->mode_info_stride;
        B_PREDICTION_MODE UNINITIALIZED_IS_SAFE(best_mode);
        int UNINITIALIZED_IS_SAFE(r), UNINITIALIZED_IS_SAFE(ry), UNINITIALIZED_IS_SAFE(d);

736 737
        if (mb->e_mbd.frame_type == KEY_FRAME)
        {
Scott LaVarnway's avatar
Scott LaVarnway committed
738 739
            const B_PREDICTION_MODE A = above_block_mode(mic, i, mis);
            const B_PREDICTION_MODE L = left_block_mode(mic, i);
740 741 742 743

            bmode_costs  = mb->bmode_costs[A][L];
        }

744
        total_rd += rd_pick_intra4x4block(
745
            mb, mb->block + i, xd->block + i, &best_mode, bmode_costs,
746 747
            ta + vp8_block2above[i],
            tl + vp8_block2left[i], &r, &ry, &d);
John Koleszar's avatar
John Koleszar committed
748 749 750 751

        cost += r;
        distortion += d;
        tot_rate_y += ry;
Scott LaVarnway's avatar
Scott LaVarnway committed
752 753

        mic->bmi[i].as_mode = best_mode;
754

755
        if(total_rd >= (int64_t)best_rd)
756
            break;
John Koleszar's avatar
John Koleszar committed
757 758
    }

759
    if(total_rd >= (int64_t)best_rd)
760
        return INT_MAX;
761

John Koleszar's avatar
John Koleszar committed
762
    *Rate = cost;
763
    *rate_y = tot_rate_y;
John Koleszar's avatar
John Koleszar committed
764 765 766 767
    *Distortion = distortion;

    return RDCOST(mb->rdmult, mb->rddiv, cost, distortion);
}
768 769


770
static int rd_pick_intra16x16mby_mode(MACROBLOCK *x,
771 772 773
                                      int *Rate,
                                      int *rate_y,
                                      int *Distortion)
John Koleszar's avatar
John Koleszar committed
774 775 776 777
{
    MB_PREDICTION_MODE mode;
    MB_PREDICTION_MODE UNINITIALIZED_IS_SAFE(mode_selected);
    int rate, ratey;
778
    int distortion;
John Koleszar's avatar
John Koleszar committed
779
    int best_rd = INT_MAX;
780
    int this_rd;
781
    MACROBLOCKD *xd = &x->e_mbd;
John Koleszar's avatar
John Koleszar committed
782

John Koleszar's avatar
John Koleszar committed
783
    /* Y Search for 16x16 intra prediction mode */
John Koleszar's avatar
John Koleszar committed
784 785
    for (mode = DC_PRED; mode <= TM_PRED; mode++)
    {
786
        xd->mode_info_context->mbmi.mode = mode;
John Koleszar's avatar
John Koleszar committed
787

788 789 790 791 792 793
        vp8_build_intra_predictors_mby_s(xd,
                                         xd->dst.y_buffer - xd->dst.y_stride,
                                         xd->dst.y_buffer - 1,
                                         xd->dst.y_stride,
                                         xd->predictor,
                                         16);
John Koleszar's avatar
John Koleszar committed
794

795
        macro_block_yrd(x, &ratey, &distortion);
796 797
        rate = ratey + x->mbmode_cost[xd->frame_type]
                                     [xd->mode_info_context->mbmi.mode];
John Koleszar's avatar
John Koleszar committed
798 799 800 801 802 803 804 805 806

        this_rd = RDCOST(x->rdmult, x->rddiv, rate, distortion);

        if (this_rd < best_rd)
        {
            mode_selected = mode;
            best_rd = this_rd;
            *Rate = rate;
            *rate_y = ratey;
807
            *Distortion = distortion;
John Koleszar's avatar
John Koleszar committed
808 809 810
        }
    }

811
    xd->mode_info_context->mbmi.mode = mode_selected;
John Koleszar's avatar
John Koleszar committed
812 813 814 815 816 817 818 819
    return best_rd;
}

static int rd_cost_mbuv(MACROBLOCK *mb)
{
    int b;
    int cost = 0;
    MACROBLOCKD *x = &mb->e_mbd;
820 821 822 823 824 825
    ENTROPY_CONTEXT_PLANES t_above, t_left;
    ENTROPY_CONTEXT *ta;
    ENTROPY_CONTEXT *tl;

    vpx_memcpy(&t_above, mb->e_mbd.above_context, sizeof(ENTROPY_CONTEXT_PLANES));
    vpx_memcpy(&t_left, mb->e_mbd.left_context, sizeof(ENTROPY_CONTEXT_PLANES));
John Koleszar's avatar
John Koleszar committed
826

827 828
    ta = (ENTROPY_CONTEXT *)&t_above;
    tl = (ENTROPY_CONTEXT *)&t_left;
John Koleszar's avatar
John Koleszar committed
829

Scott LaVarnway's avatar
Scott LaVarnway committed
830 831
    for (b = 16; b < 24; b++)
        cost += cost_coeffs(mb, x->block + b, PLANE_TYPE_UV,
832
                    ta + vp8_block2above[b], tl + vp8_block2left[b]);
John Koleszar's avatar
John Koleszar committed
833 834 835 836 837

    return cost;
}


838 839
static int rd_inter16x16_uv(VP8_COMP *cpi, MACROBLOCK *x, int *rate,
                            int *distortion, int fullpixel)
John Koleszar's avatar
John Koleszar committed
840
{
841
    vp8_build_inter16x16_predictors_mbuv(&x->e_mbd);
842
    vp8_subtract_mbuv(x->src_diff,
843 844
        x->src.u_buffer, x->src.v_buffer, x->src.uv_stride,
        &x->e_mbd.predictor[256], &x->e_mbd.predictor[320], 8);
John Koleszar's avatar
John Koleszar committed
845

846 847 848 849
    vp8_transform_mbuv(x);
    vp8_quantize_mbuv(x);

    *rate       = rd_cost_mbuv(x);
850
    *distortion = vp8_mbuverror(x) / 4;
851 852 853 854 855 856 857 858

    return RDCOST(x->rdmult, x->rddiv, *rate, *distortion);
}

static int rd_inter4x4_uv(VP8_COMP *cpi, MACROBLOCK *x, int *rate,
                          int *distortion, int fullpixel)
{
    vp8_build_inter4x4_predictors_mbuv(&x->e_mbd);
859
    vp8_subtract_mbuv(x->src_diff,
860 861
        x->src.u_buffer, x->src.v_buffer, x->src.uv_stride,
        &x->e_mbd.predictor[256], &x->e_mbd.predictor[320], 8);
862 863 864

    vp8_transform_mbuv(x);
    vp8_quantize_mbuv(x);
John Koleszar's avatar
John Koleszar committed
865 866

    *rate       = rd_cost_mbuv(x);
867
    *distortion = vp8_mbuverror(x) / 4;
John Koleszar's avatar
John Koleszar committed
868

869
    return RDCOST(x->rdmult, x->rddiv, *rate, *distortion);
John Koleszar's avatar
John Koleszar committed
870 871
}

872 873
static void rd_pick_intra_mbuv_mode(MACROBLOCK *x, int *rate,
                                    int *rate_tokenonly, int *distortion)
John Koleszar's avatar
John Koleszar committed
874 875 876 877 878 879
{
    MB_PREDICTION_MODE mode;
    MB_PREDICTION_MODE UNINITIALIZED_IS_SAFE(mode_selected);
    int best_rd = INT_MAX;
    int UNINITIALIZED_IS_SAFE(d), UNINITIALIZED_IS_SAFE(r);
    int rate_to;
880
    MACROBLOCKD *xd = &x->e_mbd;
John Koleszar's avatar
John Koleszar committed
881 882 883

    for (mode = DC_PRED; mode <= TM_PRED; mode++)
    {
884 885
        int this_rate;
        int this_distortion;
John Koleszar's avatar
John Koleszar committed
886 887
        int this_rd;

888 889 890 891 892 893 894 895 896 897 898 899
        xd->mode_info_context->mbmi.uv_mode = mode;

        vp8_build_intra_predictors_mbuv_s(xd,
                                          xd->dst.u_buffer - xd->dst.uv_stride,
                                          xd->dst.v_buffer - xd->dst.uv_stride,
                                          xd->dst.u_buffer - 1,
                                          xd->dst.v_buffer - 1,
                                          xd->dst.uv_stride,
                                          &xd->predictor[256], &xd->predictor[320],
                                          8);


900
        vp8_subtract_mbuv(x->src_diff,
901
                      x->src.u_buffer, x->src.v_buffer, x->src.uv_stride,
902
                      &xd->predictor[256], &xd->predictor[320], 8);
903 904
        vp8_transform_mbuv(x);
        vp8_quantize_mbuv(x);
John Koleszar's avatar
John Koleszar committed
905 906

        rate_to = rd_cost_mbuv(x);
907
        this_rate = rate_to + x->intra_uv_mode_cost[xd->frame_type][xd->mode_info_context->mbmi.uv_mode];
John Koleszar's avatar
John Koleszar committed
908

909
        this_distortion = vp8_mbuverror(x) / 4;
John Koleszar's avatar
John Koleszar committed
910

911
        this_rd = RDCOST(x->rdmult, x->rddiv, this_rate, this_distortion);
John Koleszar's avatar
John Koleszar committed
912 913 914 915

        if (this_rd < best_rd)
        {
            best_rd = this_rd;
916 917
            d = this_distortion;
            r = this_rate;
John Koleszar's avatar
John Koleszar committed
918 919 920 921 922 923 924 925
            *rate_tokenonly = rate_to;
            mode_selected = mode;
        }
    }

    *rate = r;
    *distortion = d;

926
    xd->mode_info_context->mbmi.uv_mode = mode_selected;
John Koleszar's avatar
John Koleszar committed
927 928 929 930 931 932 933
}

int vp8_cost_mv_ref(MB_PREDICTION_MODE m, const int near_mv_ref_ct[4])
{
    vp8_prob p [VP8_MVREFS-1];
    assert(NEARESTMV <= m  &&  m <= SPLITMV);
    vp8_mv_ref_probs(p, near_mv_ref_ct);
934
    return vp8_cost_token(vp8_mv_ref_tree, p,
Yaowu Xu's avatar
Yaowu Xu committed
935
                          vp8_mv_ref_encoding_array + (m - NEARESTMV));
John Koleszar's avatar
John Koleszar committed
936 937
}

Scott LaVarnway's avatar
Scott LaVarnway committed
938
void vp8_set_mbmode_and_mvs(MACROBLOCK *x, MB_PREDICTION_MODE mb, int_mv *mv)
John Koleszar's avatar
John Koleszar committed
939
{
940
    x->e_mbd.mode_info_context->mbmi.mode = mb;
Scott LaVarnway's avatar
Scott LaVarnway committed
941
    x->e_mbd.mode_info_context->mbmi.mv.as_int = mv->as_int;
John Koleszar's avatar
John Koleszar committed
942 943 944 945 946 947
}

static int labels2mode(
    MACROBLOCK *x,
    int const *labelings, int which_label,
    B_PREDICTION_MODE this_mode,
Scott LaVarnway's avatar
Scott LaVarnway committed
948
    int_mv *this_mv, int_mv *best_ref_mv,
John Koleszar's avatar
John Koleszar committed
949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980
    int *mvcost[2]
)
{
    MACROBLOCKD *const xd = & x->e_mbd;
    MODE_INFO *const mic = xd->mode_info_context;
    const int mis = xd->mode_info_stride;

    int cost = 0;
    int thismvcost = 0;

    /* We have to be careful retrieving previously-encoded motion vectors.
       Ones from this macroblock have to be pulled from the BLOCKD array
       as they have not yet made it to the bmi array in our MB_MODE_INFO. */

    int i = 0;

    do
    {
        BLOCKD *const d = xd->block + i;
        const int row = i >> 2,  col = i & 3;

        B_PREDICTION_MODE m;

        if (labelings[i] != which_label)
            continue;

        if (col  &&  labelings[i] == labelings[i-1])
            m = LEFT4X4;
        else if (row  &&  labelings[i] == labelings[i-4])
            m = ABOVE4X4;
        else
        {
John Koleszar's avatar
John Koleszar committed
981 982 983
            /* the only time we should do costing for new motion vector
             * or mode is when we are on a new label  (jbb May 08, 2007)
             */
John Koleszar's avatar
John Koleszar committed
984 985 986 987 988 989
            switch (m = this_mode)
            {
            case NEW4X4 :
                thismvcost  = vp8_mv_bit_cost(this_mv, best_ref_mv, mvcost, 102);
                break;
            case LEFT4X4:
Scott LaVarnway's avatar
Scott LaVarnway committed
990
                this_mv->as_int = col ? d[-1].bmi.mv.as_int : left_block_mv(mic, i);
John Koleszar's avatar
John Koleszar committed
991 992
                break;
            case ABOVE4X4:
Scott LaVarnway's avatar
Scott LaVarnway committed
993
                this_mv->as_int = row ? d[-4].bmi.mv.as_int : above_block_mv(mic, i, mis);
John Koleszar's avatar
John Koleszar committed
994 995
                break;
            case ZERO4X4:
Scott LaVarnway's avatar
Scott LaVarnway committed
996
                this_mv->as_int = 0;
John Koleszar's avatar
John Koleszar committed
997 998 999 1000 1001
                break;
            default:
                break;
            }

John Koleszar's avatar
John Koleszar committed
1002
            if (m == ABOVE4X4)  /* replace above with left if same */
John Koleszar's avatar
John Koleszar committed
1003
            {
Scott LaVarnway's avatar
Scott LaVarnway committed
1004
                int_mv left_mv;
Scott LaVarnway's avatar
Scott LaVarnway committed
1005

Scott LaVarnway's avatar
Scott LaVarnway committed
1006
                left_mv.as_int = col ? d[-1].bmi.mv.as_int :
Scott LaVarnway's avatar
Scott LaVarnway committed
1007
                                        left_block_mv(mic, i);
John Koleszar's avatar
John Koleszar committed
1008

Scott LaVarnway's avatar
Scott LaVarnway committed
1009
                if (left_mv.as_int == this_mv->as_int)
John Koleszar's avatar
John Koleszar committed
1010 1011 1012 1013 1014 1015
                    m = LEFT4X4;
            }

            cost = x->inter_bmode_costs[ m];
        }

Scott LaVarnway's avatar
Scott LaVarnway committed
1016
        d->bmi.mv.as_int = this_mv->as_int;
John Koleszar's avatar
John Koleszar committed
1017

1018 1019 1020
        x->partition_info->bmi[i].mode = m;
        x->partition_info->bmi[i].mv.as_int = this_mv->as_int;

John Koleszar's avatar
John Koleszar committed
1021 1022 1023 1024 1025 1026 1027
    }
    while (++i < 16);

    cost += thismvcost ;
    return cost;
}

1028 1029 1030
static int rdcost_mbsegment_y(MACROBLOCK *mb, const int *labels,
                              int which_label, ENTROPY_CONTEXT *ta,
                              ENTROPY_CONTEXT *tl)
John Koleszar's avatar
John Koleszar committed
1031 1032 1033 1034 1035 1036 1037
{
    int cost = 0;
    int b;
    MACROBLOCKD *x = &mb->e_mbd;

    for (b = 0; b < 16; b++)
        if (labels[ b] == which_label)
Scott LaVarnway's avatar
Scott LaVarnway committed
1038
            cost += cost_coeffs(mb, x->block + b, PLANE_TYPE_Y_WITH_DC,
1039