rdopt.c 79.8 KB
Newer Older
John Koleszar's avatar
John Koleszar committed
1
/*
2
 *  Copyright (c) 2010 The WebM project authors. All Rights Reserved.
John Koleszar's avatar
John Koleszar committed
3
 *
4
 *  Use of this source code is governed by a BSD-style license
5 6
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
7
 *  in the file PATENTS.  All contributing project authors may
8
 *  be found in the AUTHORS file in the root of the source tree.
John Koleszar's avatar
John Koleszar committed
9 10 11 12 13 14 15
 */


#include <stdio.h>
#include <math.h>
#include <limits.h>
#include <assert.h>
John Koleszar's avatar
John Koleszar committed
16
#include "vp8/common/pragmas.h"
John Koleszar's avatar
John Koleszar committed
17 18 19 20 21 22

#include "tokenize.h"
#include "treewriter.h"
#include "onyx_int.h"
#include "modecosts.h"
#include "encodeintra.h"
John Koleszar's avatar
John Koleszar committed
23 24 25 26 27
#include "vp8/common/entropymode.h"
#include "vp8/common/reconinter.h"
#include "vp8/common/reconintra.h"
#include "vp8/common/reconintra4x4.h"
#include "vp8/common/findnearmv.h"
John Koleszar's avatar
John Koleszar committed
28 29
#include "encodemb.h"
#include "quantize.h"
John Koleszar's avatar
John Koleszar committed
30 31
#include "vp8/common/idct.h"
#include "vp8/common/g_common.h"
John Koleszar's avatar
John Koleszar committed
32 33
#include "variance.h"
#include "mcomp.h"
Yunqing Wang's avatar
Yunqing Wang committed
34
#include "rdopt.h"
John Koleszar's avatar
John Koleszar committed
35 36
#include "vpx_mem/vpx_mem.h"
#include "dct.h"
John Koleszar's avatar
John Koleszar committed
37
#include "vp8/common/systemdependent.h"
John Koleszar's avatar
John Koleszar committed
38 39 40 41 42 43 44 45

#if CONFIG_RUNTIME_CPU_DETECT
#define IF_RTCD(x)  (x)
#else
#define IF_RTCD(x)  NULL
#endif


Scott LaVarnway's avatar
Scott LaVarnway committed
46 47 48
extern void vp8cx_mb_init_quantizer(VP8_COMP *cpi, MACROBLOCK *x);
extern void vp8_update_zbin_extra(VP8_COMP *cpi, MACROBLOCK *x);

John Koleszar's avatar
John Koleszar committed
49 50
#define MAXF(a,b)            (((a) > (b)) ? (a) : (b))

51
static const int auto_speed_thresh[17] =
John Koleszar's avatar
John Koleszar committed
52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151
{
    1000,
    200,
    150,
    130,
    150,
    125,
    120,
    115,
    115,
    115,
    115,
    115,
    115,
    115,
    115,
    115,
    105
};

const MB_PREDICTION_MODE vp8_mode_order[MAX_MODES] =
{
    ZEROMV,
    DC_PRED,

    NEARESTMV,
    NEARMV,

    ZEROMV,
    NEARESTMV,

    ZEROMV,
    NEARESTMV,

    NEARMV,
    NEARMV,

    V_PRED,
    H_PRED,
    TM_PRED,

    NEWMV,
    NEWMV,
    NEWMV,

    SPLITMV,
    SPLITMV,
    SPLITMV,

    B_PRED,
};

const MV_REFERENCE_FRAME vp8_ref_frame_order[MAX_MODES] =
{
    LAST_FRAME,
    INTRA_FRAME,

    LAST_FRAME,
    LAST_FRAME,

    GOLDEN_FRAME,
    GOLDEN_FRAME,

    ALTREF_FRAME,
    ALTREF_FRAME,

    GOLDEN_FRAME,
    ALTREF_FRAME,

    INTRA_FRAME,
    INTRA_FRAME,
    INTRA_FRAME,

    LAST_FRAME,
    GOLDEN_FRAME,
    ALTREF_FRAME,

    LAST_FRAME,
    GOLDEN_FRAME,
    ALTREF_FRAME,

    INTRA_FRAME,
};

static void fill_token_costs(
    unsigned int c      [BLOCK_TYPES] [COEF_BANDS] [PREV_COEF_CONTEXTS] [vp8_coef_tokens],
    const vp8_prob p    [BLOCK_TYPES] [COEF_BANDS] [PREV_COEF_CONTEXTS] [vp8_coef_tokens-1]
)
{
    int i, j, k;


    for (i = 0; i < BLOCK_TYPES; i++)
        for (j = 0; j < COEF_BANDS; j++)
            for (k = 0; k < PREV_COEF_CONTEXTS; k++)

                vp8_cost_tokens((int *)(c [i][j][k]), p [i][j][k], vp8_coef_tree);

}

Paul Wilkins's avatar
Paul Wilkins committed
152
static int rd_iifactor [ 32 ] =  {    4,   4,   3,   2,   1,   0,   0,   0,
John Koleszar's avatar
John Koleszar committed
153 154 155 156 157 158
                                      0,   0,   0,   0,   0,   0,   0,   0,
                                      0,   0,   0,   0,   0,   0,   0,   0,
                                      0,   0,   0,   0,   0,   0,   0,   0,
                                 };


159 160
/* values are now correlated to quantizer */
static int sad_per_bit16lut[QINDEX_RANGE] =
John Koleszar's avatar
John Koleszar committed
161
{
162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177
    5,  5,  5,  5,  5,  5,  6,  6,
    6,  6,  6,  6,  6,  7,  7,  7,
    7,  7,  7,  7,  8,  8,  8,  8,
    8,  8,  8,  8,  8,  8,  9,  9,
    9,  9,  9,  9, 10, 10, 10, 10,
    10, 10, 11, 11, 11, 11, 11, 11,
    12, 12, 12, 12, 12, 12, 12, 13,
    13, 13, 13, 13, 13, 14, 14, 14,
    14, 14, 15, 15, 15, 15, 15, 15,
    16, 16, 16, 16, 16, 16, 17, 17,
    17, 17, 17, 17, 17, 18, 18, 18,
    18, 18, 19, 19, 19, 19, 19, 19,
    20, 20, 20, 21, 21, 21, 21, 22,
    22, 22, 23, 23, 23, 24, 24, 24,
    25, 25, 26, 26, 27, 27, 27, 28,
    28, 28, 29, 29, 30, 30, 31, 31
John Koleszar's avatar
John Koleszar committed
178
};
179
static int sad_per_bit4lut[QINDEX_RANGE] =
John Koleszar's avatar
John Koleszar committed
180
{
181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196
    5,  5,  5,  5,  5,  5,  7,  7,
    7,  7,  7,  7,  7,  8,  8,  8,
    8,  8,  8,  8,  10, 10, 10, 10,
    10, 10, 10, 10, 10, 10, 11, 11,
    11, 11, 11, 11, 13, 13, 13, 13,
    13, 13, 14, 14, 14, 14, 14, 14,
    16, 16, 16, 16, 16, 16, 16, 17,
    17, 17, 17, 17, 17, 19, 19, 19,
    19, 19, 20, 20, 20, 20, 20, 20,
    22, 22, 22, 22, 22, 22, 23, 23,
    23, 23, 23, 23, 23, 25, 25, 25,
    25, 25, 26, 26, 26, 26, 26, 26,
    28, 28, 28, 29, 29, 29, 29, 31,
    31, 31, 32, 32, 32, 34, 34, 34,
    35, 35, 37, 37, 38, 38, 38, 40,
    40, 40, 41, 41, 43, 43, 44, 44,
John Koleszar's avatar
John Koleszar committed
197 198 199 200 201 202 203 204 205 206 207 208
};

void vp8cx_initialize_me_consts(VP8_COMP *cpi, int QIndex)
{
    cpi->mb.sadperbit16 =  sad_per_bit16lut[QIndex];
    cpi->mb.sadperbit4  =  sad_per_bit4lut[QIndex];
}

void vp8_initialize_rd_consts(VP8_COMP *cpi, int Qvalue)
{
    int q;
    int i;
209
    double capped_q = (Qvalue < 160) ? (double)Qvalue : 160.0;
210
    double rdconst = 2.70;
John Koleszar's avatar
John Koleszar committed
211 212 213

    vp8_clear_system_state();  //__asm emms;

214 215 216 217 218
    // Further tests required to see if optimum is different
    // for key frames, golden frames and arf frames.
    // if (cpi->common.refresh_golden_frame ||
    //     cpi->common.refresh_alt_ref_frame)
    cpi->RDMULT = (int)(rdconst * (capped_q * capped_q));
John Koleszar's avatar
John Koleszar committed
219

220 221 222 223 224 225 226 227 228 229 230 231
    // Extend rate multiplier along side quantizer zbin increases
    if (cpi->zbin_over_quant  > 0)
    {
        double oq_factor;
        double modq;

        // Experimental code using the same basic equation as used for Q above
        // The units of cpi->zbin_over_quant are 1/128 of Q bin size
        oq_factor = 1.0 + ((double)0.0015625 * cpi->zbin_over_quant);
        modq = (int)((double)capped_q * oq_factor);
        cpi->RDMULT = (int)(rdconst * (modq * modq));
    }
John Koleszar's avatar
John Koleszar committed
232

Paul Wilkins's avatar
Paul Wilkins committed
233
    if (cpi->pass == 2 && (cpi->common.frame_type != KEY_FRAME))
John Koleszar's avatar
John Koleszar committed
234
    {
235
        if (cpi->twopass.next_iiratio > 31)
Paul Wilkins's avatar
Paul Wilkins committed
236
            cpi->RDMULT += (cpi->RDMULT * rd_iifactor[31]) >> 4;
John Koleszar's avatar
John Koleszar committed
237
        else
238 239
            cpi->RDMULT +=
                (cpi->RDMULT * rd_iifactor[cpi->twopass.next_iiratio]) >> 4;
John Koleszar's avatar
John Koleszar committed
240 241 242
    }

    cpi->mb.errorperbit = (cpi->RDMULT / 100);
243 244
    cpi->mb.errorperbit += (cpi->mb.errorperbit==0);

John Koleszar's avatar
John Koleszar committed
245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320
    vp8_set_speed_features(cpi);

    q = (int)pow(Qvalue, 1.25);

    if (q < 8)
        q = 8;

    if (cpi->RDMULT > 1000)
    {
        cpi->RDDIV = 1;
        cpi->RDMULT /= 100;

        for (i = 0; i < MAX_MODES; i++)
        {
            if (cpi->sf.thresh_mult[i] < INT_MAX)
            {
                cpi->rd_threshes[i] = cpi->sf.thresh_mult[i] * q / 100;
            }
            else
            {
                cpi->rd_threshes[i] = INT_MAX;
            }

            cpi->rd_baseline_thresh[i] = cpi->rd_threshes[i];
        }
    }
    else
    {
        cpi->RDDIV = 100;

        for (i = 0; i < MAX_MODES; i++)
        {
            if (cpi->sf.thresh_mult[i] < (INT_MAX / q))
            {
                cpi->rd_threshes[i] = cpi->sf.thresh_mult[i] * q;
            }
            else
            {
                cpi->rd_threshes[i] = INT_MAX;
            }

            cpi->rd_baseline_thresh[i] = cpi->rd_threshes[i];
        }
    }

    fill_token_costs(
        cpi->mb.token_costs,
        (const vp8_prob( *)[8][3][11]) cpi->common.fc.coef_probs
    );

    vp8_init_mode_costs(cpi);

}

void vp8_auto_select_speed(VP8_COMP *cpi)
{
    int milliseconds_for_compress = (int)(1000000 / cpi->oxcf.frame_rate);

    milliseconds_for_compress = milliseconds_for_compress * (16 - cpi->oxcf.cpu_used) / 16;

#if 0

    if (0)
    {
        FILE *f;

        f = fopen("speed.stt", "a");
        fprintf(f, " %8ld %10ld %10ld %10ld\n",
                cpi->common.current_video_frame, cpi->Speed, milliseconds_for_compress, cpi->avg_pick_mode_time);
        fclose(f);
    }

#endif

    /*
    // this is done during parameter valid check
Johann's avatar
Johann committed
321 322 323 324
    if( cpi->oxcf.cpu_used > 16)
        cpi->oxcf.cpu_used = 16;
    if( cpi->oxcf.cpu_used < -16)
        cpi->oxcf.cpu_used = -16;
John Koleszar's avatar
John Koleszar committed
325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346
    */

    if (cpi->avg_pick_mode_time < milliseconds_for_compress && (cpi->avg_encode_time - cpi->avg_pick_mode_time) < milliseconds_for_compress)
    {
        if (cpi->avg_pick_mode_time == 0)
        {
            cpi->Speed = 4;
        }
        else
        {
            if (milliseconds_for_compress * 100 < cpi->avg_encode_time * 95)
            {
                cpi->Speed          += 2;
                cpi->avg_pick_mode_time = 0;
                cpi->avg_encode_time = 0;

                if (cpi->Speed > 16)
                {
                    cpi->Speed = 16;
                }
            }

347
            if (milliseconds_for_compress * 100 > cpi->avg_encode_time * auto_speed_thresh[cpi->Speed])
John Koleszar's avatar
John Koleszar committed
348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458
            {
                cpi->Speed          -= 1;
                cpi->avg_pick_mode_time = 0;
                cpi->avg_encode_time = 0;

                // In real-time mode, cpi->speed is in [4, 16].
                if (cpi->Speed < 4)        //if ( cpi->Speed < 0 )
                {
                    cpi->Speed = 4;        //cpi->Speed = 0;
                }
            }
        }
    }
    else
    {
        cpi->Speed += 4;

        if (cpi->Speed > 16)
            cpi->Speed = 16;


        cpi->avg_pick_mode_time = 0;
        cpi->avg_encode_time = 0;
    }
}

int vp8_block_error_c(short *coeff, short *dqcoeff)
{
    int i;
    int error = 0;

    for (i = 0; i < 16; i++)
    {
        int this_diff = coeff[i] - dqcoeff[i];
        error += this_diff * this_diff;
    }

    return error;
}

int vp8_mbblock_error_c(MACROBLOCK *mb, int dc)
{
    BLOCK  *be;
    BLOCKD *bd;
    int i, j;
    int berror, error = 0;

    for (i = 0; i < 16; i++)
    {
        be = &mb->block[i];
        bd = &mb->e_mbd.block[i];

        berror = 0;

        for (j = dc; j < 16; j++)
        {
            int this_diff = be->coeff[j] - bd->dqcoeff[j];
            berror += this_diff * this_diff;
        }

        error += berror;
    }

    return error;
}

int vp8_mbuverror_c(MACROBLOCK *mb)
{

    BLOCK  *be;
    BLOCKD *bd;


    int i;
    int error = 0;

    for (i = 16; i < 24; i++)
    {
        be = &mb->block[i];
        bd = &mb->e_mbd.block[i];

        error += vp8_block_error_c(be->coeff, bd->dqcoeff);
    }

    return error;
}

int VP8_UVSSE(MACROBLOCK *x, const vp8_variance_rtcd_vtable_t *rtcd)
{
    unsigned char *uptr, *vptr;
    unsigned char *upred_ptr = (*(x->block[16].base_src) + x->block[16].src);
    unsigned char *vpred_ptr = (*(x->block[20].base_src) + x->block[20].src);
    int uv_stride = x->block[16].src_stride;

    unsigned int sse1 = 0;
    unsigned int sse2 = 0;
    int mv_row;
    int mv_col;
    int offset;
    int pre_stride = x->e_mbd.block[16].pre_stride;

    vp8_build_uvmvs(&x->e_mbd, 0);
    mv_row = x->e_mbd.block[16].bmi.mv.as_mv.row;
    mv_col = x->e_mbd.block[16].bmi.mv.as_mv.col;

    offset = (mv_row >> 3) * pre_stride + (mv_col >> 3);
    uptr = x->e_mbd.pre.u_buffer + offset;
    vptr = x->e_mbd.pre.v_buffer + offset;

    if ((mv_row | mv_col) & 7)
    {
459 460 461 462
        VARIANCE_INVOKE(rtcd, subpixvar8x8)(uptr, pre_stride,
            mv_col & 7, mv_row & 7, upred_ptr, uv_stride, &sse2);
        VARIANCE_INVOKE(rtcd, subpixvar8x8)(vptr, pre_stride,
            mv_col & 7, mv_row & 7, vpred_ptr, uv_stride, &sse1);
John Koleszar's avatar
John Koleszar committed
463 464 465 466
        sse2 += sse1;
    }
    else
    {
467 468 469 470
        VARIANCE_INVOKE(rtcd, var8x8)(uptr, pre_stride,
            upred_ptr, uv_stride, &sse2);
        VARIANCE_INVOKE(rtcd, var8x8)(vptr, pre_stride,
            vpred_ptr, uv_stride, &sse1);
John Koleszar's avatar
John Koleszar committed
471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508
        sse2 += sse1;
    }
    return sse2;

}

static int cost_coeffs(MACROBLOCK *mb, BLOCKD *b, int type, ENTROPY_CONTEXT *a, ENTROPY_CONTEXT *l)
{
    int c = !type;              /* start at coef 0, unless Y with Y2 */
    int eob = b->eob;
    int pt ;    /* surrounding block/prev coef predictor */
    int cost = 0;
    short *qcoeff_ptr = b->qcoeff;

    VP8_COMBINEENTROPYCONTEXTS(pt, *a, *l);

# define QC( I)  ( qcoeff_ptr [vp8_default_zig_zag1d[I]] )

    for (; c < eob; c++)
    {
        int v = QC(c);
        int t = vp8_dct_value_tokens_ptr[v].Token;
        cost += mb->token_costs [type] [vp8_coef_bands[c]] [pt] [t];
        cost += vp8_dct_value_cost_ptr[v];
        pt = vp8_prev_token_class[t];
    }

# undef QC

    if (c < 16)
        cost += mb->token_costs [type] [vp8_coef_bands[c]] [pt] [DCT_EOB_TOKEN];

    pt = (c != !type); // is eob first coefficient;
    *a = *l = pt;

    return cost;
}

Scott LaVarnway's avatar
Scott LaVarnway committed
509
static int vp8_rdcost_mby(MACROBLOCK *mb)
John Koleszar's avatar
John Koleszar committed
510 511 512 513
{
    int cost = 0;
    int b;
    MACROBLOCKD *x = &mb->e_mbd;
514 515 516 517 518 519
    ENTROPY_CONTEXT_PLANES t_above, t_left;
    ENTROPY_CONTEXT *ta;
    ENTROPY_CONTEXT *tl;

    vpx_memcpy(&t_above, mb->e_mbd.above_context, sizeof(ENTROPY_CONTEXT_PLANES));
    vpx_memcpy(&t_left, mb->e_mbd.left_context, sizeof(ENTROPY_CONTEXT_PLANES));
John Koleszar's avatar
John Koleszar committed
520

521 522
    ta = (ENTROPY_CONTEXT *)&t_above;
    tl = (ENTROPY_CONTEXT *)&t_left;
John Koleszar's avatar
John Koleszar committed
523 524

    for (b = 0; b < 16; b++)
Scott LaVarnway's avatar
Scott LaVarnway committed
525
        cost += cost_coeffs(mb, x->block + b, PLANE_TYPE_Y_NO_DC,
526
                    ta + vp8_block2above[b], tl + vp8_block2left[b]);
John Koleszar's avatar
John Koleszar committed
527

Scott LaVarnway's avatar
Scott LaVarnway committed
528
    cost += cost_coeffs(mb, x->block + 24, PLANE_TYPE_Y2,
Scott LaVarnway's avatar
Scott LaVarnway committed
529
                ta + vp8_block2above[24], tl + vp8_block2left[24]);
John Koleszar's avatar
John Koleszar committed
530 531 532 533

    return cost;
}

534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578
static void macro_block_yrd( MACROBLOCK *mb,
                             int *Rate,
                             int *Distortion,
                             const vp8_encodemb_rtcd_vtable_t *rtcd)
{
    int b;
    MACROBLOCKD *const x = &mb->e_mbd;
    BLOCK   *const mb_y2 = mb->block + 24;
    BLOCKD *const x_y2  = x->block + 24;
    short *Y2DCPtr = mb_y2->src_diff;
    BLOCK *beptr;
    int d;

    ENCODEMB_INVOKE(rtcd, submby)( mb->src_diff, mb->src.y_buffer,
                                   mb->e_mbd.predictor, mb->src.y_stride );

    // Fdct and building the 2nd order block
    for (beptr = mb->block; beptr < mb->block + 16; beptr += 2)
    {
        mb->vp8_short_fdct8x4(beptr->src_diff, beptr->coeff, 32);
        *Y2DCPtr++ = beptr->coeff[0];
        *Y2DCPtr++ = beptr->coeff[16];
    }

    // 2nd order fdct
    mb->short_walsh4x4(mb_y2->src_diff, mb_y2->coeff, 8);

    // Quantization
    for (b = 0; b < 16; b++)
    {
        mb->quantize_b(&mb->block[b], &mb->e_mbd.block[b]);
    }

    // DC predication and Quantization of 2nd Order block
    mb->quantize_b(mb_y2, x_y2);

    // Distortion
    d = ENCODEMB_INVOKE(rtcd, mberr)(mb, 1) << 2;
    d += ENCODEMB_INVOKE(rtcd, berr)(mb_y2->coeff, x_y2->dqcoeff);

    *Distortion = (d >> 4);

    // rate
    *Rate = vp8_rdcost_mby(mb);
}
John Koleszar's avatar
John Koleszar committed
579

580
static void copy_predictor(unsigned char *dst, const unsigned char *predictor)
581
{
582 583 584 585 586 587
    const unsigned int *p = (const unsigned int *)predictor;
    unsigned int *d = (unsigned int *)dst;
    d[0] = p[0];
    d[4] = p[4];
    d[8] = p[8];
    d[12] = p[12];
588
}
589
static int rd_pick_intra4x4block(
John Koleszar's avatar
John Koleszar committed
590 591 592 593 594
    VP8_COMP *cpi,
    MACROBLOCK *x,
    BLOCK *be,
    BLOCKD *b,
    B_PREDICTION_MODE *best_mode,
595
    unsigned int *bmode_costs,
John Koleszar's avatar
John Koleszar committed
596 597 598 599 600 601 602 603
    ENTROPY_CONTEXT *a,
    ENTROPY_CONTEXT *l,

    int *bestrate,
    int *bestratey,
    int *bestdistortion)
{
    B_PREDICTION_MODE mode;
604
    int best_rd = INT_MAX;
John Koleszar's avatar
John Koleszar committed
605 606 607 608 609
    int rate = 0;
    int distortion;

    ENTROPY_CONTEXT ta = *a, tempa = *a;
    ENTROPY_CONTEXT tl = *l, templ = *l;
610 611 612 613 614 615 616
    /*
     * The predictor buffer is a 2d buffer with a stride of 16.  Create
     * a temp buffer that meets the stride requirements, but we are only
     * interested in the left 4x4 block
     * */
    DECLARE_ALIGNED_ARRAY(16, unsigned char,  best_predictor, 16*4);
    DECLARE_ALIGNED_ARRAY(16, short, best_dqcoeff, 16);
John Koleszar's avatar
John Koleszar committed
617 618 619 620 621 622

    for (mode = B_DC_PRED; mode <= B_HU_PRED; mode++)
    {
        int this_rd;
        int ratey;

623 624
        rate = bmode_costs[mode];

625 626
        RECON_INVOKE(&cpi->rtcd.common->recon, intra4x4_predict)
                     (b, mode, b->predictor);
627 628 629
        ENCODEMB_INVOKE(IF_RTCD(&cpi->rtcd.encodemb), subb)(be, b, 16);
        x->vp8_short_fdct4x4(be->src_diff, be->coeff, 32);
        x->quantize_b(be, b);
John Koleszar's avatar
John Koleszar committed
630 631 632 633

        tempa = ta;
        templ = tl;

Scott LaVarnway's avatar
Scott LaVarnway committed
634
        ratey = cost_coeffs(x, b, PLANE_TYPE_Y_WITH_DC, &tempa, &templ);
John Koleszar's avatar
John Koleszar committed
635 636 637 638 639 640 641 642 643 644 645 646 647 648
        rate += ratey;
        distortion = ENCODEMB_INVOKE(IF_RTCD(&cpi->rtcd.encodemb), berr)(be->coeff, b->dqcoeff) >> 2;

        this_rd = RDCOST(x->rdmult, x->rddiv, rate, distortion);

        if (this_rd < best_rd)
        {
            *bestrate = rate;
            *bestratey = ratey;
            *bestdistortion = distortion;
            best_rd = this_rd;
            *best_mode = mode;
            *a = tempa;
            *l = templ;
649 650
            copy_predictor(best_predictor, b->predictor);
            vpx_memcpy(best_dqcoeff, b->dqcoeff, 32);
John Koleszar's avatar
John Koleszar committed
651 652 653
        }
    }
    b->bmi.mode = (B_PREDICTION_MODE)(*best_mode);
654

655 656
    IDCT_INVOKE(IF_RTCD(&cpi->rtcd.common->idct), idct16)(best_dqcoeff, b->diff, 32);
    RECON_INVOKE(IF_RTCD(&cpi->rtcd.common->recon), recon)(best_predictor, b->diff, *(b->base_dst) + b->dst, b->dst_stride);
John Koleszar's avatar
John Koleszar committed
657

658
    return best_rd;
John Koleszar's avatar
John Koleszar committed
659 660
}

661 662
int vp8_rd_pick_intra4x4mby_modes(VP8_COMP *cpi, MACROBLOCK *mb, int *Rate,
                                  int *rate_y, int *Distortion, int best_rd)
John Koleszar's avatar
John Koleszar committed
663 664 665 666 667 668
{
    MACROBLOCKD *const xd = &mb->e_mbd;
    int i;
    int cost = mb->mbmode_cost [xd->frame_type] [B_PRED];
    int distortion = 0;
    int tot_rate_y = 0;
669
    long long total_rd = 0;
670 671 672
    ENTROPY_CONTEXT_PLANES t_above, t_left;
    ENTROPY_CONTEXT *ta;
    ENTROPY_CONTEXT *tl;
673
    unsigned int *bmode_costs;
674 675 676 677 678 679

    vpx_memcpy(&t_above, mb->e_mbd.above_context, sizeof(ENTROPY_CONTEXT_PLANES));
    vpx_memcpy(&t_left, mb->e_mbd.left_context, sizeof(ENTROPY_CONTEXT_PLANES));

    ta = (ENTROPY_CONTEXT *)&t_above;
    tl = (ENTROPY_CONTEXT *)&t_left;
John Koleszar's avatar
John Koleszar committed
680 681 682

    vp8_intra_prediction_down_copy(xd);

683 684
    bmode_costs = mb->inter_bmode_costs;

John Koleszar's avatar
John Koleszar committed
685 686 687 688 689 690 691
    for (i = 0; i < 16; i++)
    {
        MODE_INFO *const mic = xd->mode_info_context;
        const int mis = xd->mode_info_stride;
        B_PREDICTION_MODE UNINITIALIZED_IS_SAFE(best_mode);
        int UNINITIALIZED_IS_SAFE(r), UNINITIALIZED_IS_SAFE(ry), UNINITIALIZED_IS_SAFE(d);

692 693
        if (mb->e_mbd.frame_type == KEY_FRAME)
        {
Scott LaVarnway's avatar
Scott LaVarnway committed
694 695
            const B_PREDICTION_MODE A = above_block_mode(mic, i, mis);
            const B_PREDICTION_MODE L = left_block_mode(mic, i);
696 697 698 699

            bmode_costs  = mb->bmode_costs[A][L];
        }

700
        total_rd += rd_pick_intra4x4block(
701
            cpi, mb, mb->block + i, xd->block + i, &best_mode, bmode_costs,
702 703
            ta + vp8_block2above[i],
            tl + vp8_block2left[i], &r, &ry, &d);
John Koleszar's avatar
John Koleszar committed
704 705 706 707

        cost += r;
        distortion += d;
        tot_rate_y += ry;
Scott LaVarnway's avatar
Scott LaVarnway committed
708 709

        mic->bmi[i].as_mode = best_mode;
710

711 712
        if(total_rd >= (long long)best_rd)
            break;
John Koleszar's avatar
John Koleszar committed
713 714
    }

715 716
    if(total_rd >= (long long)best_rd)
        return INT_MAX;
717

John Koleszar's avatar
John Koleszar committed
718 719 720 721 722 723
    *Rate = cost;
    *rate_y += tot_rate_y;
    *Distortion = distortion;

    return RDCOST(mb->rdmult, mb->rddiv, cost, distortion);
}
724 725 726 727 728
int vp8_rd_pick_intra16x16mby_mode(VP8_COMP *cpi,
                                   MACROBLOCK *x,
                                   int *Rate,
                                   int *rate_y,
                                   int *Distortion)
John Koleszar's avatar
John Koleszar committed
729 730 731 732
{
    MB_PREDICTION_MODE mode;
    MB_PREDICTION_MODE UNINITIALIZED_IS_SAFE(mode_selected);
    int rate, ratey;
733
    int distortion;
John Koleszar's avatar
John Koleszar committed
734
    int best_rd = INT_MAX;
735
    int this_rd;
John Koleszar's avatar
John Koleszar committed
736 737 738 739

    //Y Search for 16x16 intra prediction mode
    for (mode = DC_PRED; mode <= TM_PRED; mode++)
    {
740
        x->e_mbd.mode_info_context->mbmi.mode = mode;
John Koleszar's avatar
John Koleszar committed
741

742 743
        RECON_INVOKE(&cpi->common.rtcd.recon, build_intra_predictors_mby)
            (&x->e_mbd);
John Koleszar's avatar
John Koleszar committed
744

745 746 747
        macro_block_yrd(x, &ratey, &distortion, IF_RTCD(&cpi->rtcd.encodemb));
        rate = ratey + x->mbmode_cost[x->e_mbd.frame_type]
                                     [x->e_mbd.mode_info_context->mbmi.mode];
John Koleszar's avatar
John Koleszar committed
748 749 750 751 752 753 754 755 756

        this_rd = RDCOST(x->rdmult, x->rddiv, rate, distortion);

        if (this_rd < best_rd)
        {
            mode_selected = mode;
            best_rd = this_rd;
            *Rate = rate;
            *rate_y = ratey;
757
            *Distortion = distortion;
John Koleszar's avatar
John Koleszar committed
758 759 760
        }
    }

761
    x->e_mbd.mode_info_context->mbmi.mode = mode_selected;
John Koleszar's avatar
John Koleszar committed
762 763 764 765 766 767 768 769
    return best_rd;
}

static int rd_cost_mbuv(MACROBLOCK *mb)
{
    int b;
    int cost = 0;
    MACROBLOCKD *x = &mb->e_mbd;
770 771 772 773 774 775
    ENTROPY_CONTEXT_PLANES t_above, t_left;
    ENTROPY_CONTEXT *ta;
    ENTROPY_CONTEXT *tl;

    vpx_memcpy(&t_above, mb->e_mbd.above_context, sizeof(ENTROPY_CONTEXT_PLANES));
    vpx_memcpy(&t_left, mb->e_mbd.left_context, sizeof(ENTROPY_CONTEXT_PLANES));
John Koleszar's avatar
John Koleszar committed
776

777 778
    ta = (ENTROPY_CONTEXT *)&t_above;
    tl = (ENTROPY_CONTEXT *)&t_left;
John Koleszar's avatar
John Koleszar committed
779

Scott LaVarnway's avatar
Scott LaVarnway committed
780 781
    for (b = 16; b < 24; b++)
        cost += cost_coeffs(mb, x->block + b, PLANE_TYPE_UV,
782
                    ta + vp8_block2above[b], tl + vp8_block2left[b]);
John Koleszar's avatar
John Koleszar committed
783 784 785 786 787 788 789 790 791 792 793 794 795 796

    return cost;
}


static int vp8_rd_inter_uv(VP8_COMP *cpi, MACROBLOCK *x, int *rate, int *distortion, int fullpixel)
{
    vp8_build_uvmvs(&x->e_mbd, fullpixel);
    vp8_encode_inter16x16uvrd(IF_RTCD(&cpi->rtcd), x);


    *rate       = rd_cost_mbuv(x);
    *distortion = ENCODEMB_INVOKE(&cpi->rtcd.encodemb, mbuverr)(x) / 4;

797
    return RDCOST(x->rdmult, x->rddiv, *rate, *distortion);
John Koleszar's avatar
John Koleszar committed
798 799
}

Paul Wilkins's avatar
Paul Wilkins committed
800
void vp8_rd_pick_intra_mbuv_mode(VP8_COMP *cpi, MACROBLOCK *x, int *rate, int *rate_tokenonly, int *distortion)
John Koleszar's avatar
John Koleszar committed
801 802 803 804 805 806 807 808 809 810 811 812 813
{
    MB_PREDICTION_MODE mode;
    MB_PREDICTION_MODE UNINITIALIZED_IS_SAFE(mode_selected);
    int best_rd = INT_MAX;
    int UNINITIALIZED_IS_SAFE(d), UNINITIALIZED_IS_SAFE(r);
    int rate_to;

    for (mode = DC_PRED; mode <= TM_PRED; mode++)
    {
        int rate;
        int distortion;
        int this_rd;

814
        x->e_mbd.mode_info_context->mbmi.uv_mode = mode;
815 816
        RECON_INVOKE(&cpi->rtcd.common->recon, build_intra_predictors_mbuv)
                     (&x->e_mbd);
817 818 819 820 821
        ENCODEMB_INVOKE(IF_RTCD(&cpi->rtcd.encodemb), submbuv)(x->src_diff,
                      x->src.u_buffer, x->src.v_buffer, x->e_mbd.predictor,
                      x->src.uv_stride);
        vp8_transform_mbuv(x);
        vp8_quantize_mbuv(x);
John Koleszar's avatar
John Koleszar committed
822 823

        rate_to = rd_cost_mbuv(x);
824
        rate = rate_to + x->intra_uv_mode_cost[x->e_mbd.frame_type][x->e_mbd.mode_info_context->mbmi.uv_mode];
John Koleszar's avatar
John Koleszar committed
825

826
        distortion = ENCODEMB_INVOKE(&cpi->rtcd.encodemb, mbuverr)(x) / 4;
John Koleszar's avatar
John Koleszar committed
827

828
        this_rd = RDCOST(x->rdmult, x->rddiv, rate, distortion);
John Koleszar's avatar
John Koleszar committed
829 830 831 832 833 834 835 836 837 838 839 840 841 842

        if (this_rd < best_rd)
        {
            best_rd = this_rd;
            d = distortion;
            r = rate;
            *rate_tokenonly = rate_to;
            mode_selected = mode;
        }
    }

    *rate = r;
    *distortion = d;

843
    x->e_mbd.mode_info_context->mbmi.uv_mode = mode_selected;
John Koleszar's avatar
John Koleszar committed
844 845 846 847 848 849 850
}

int vp8_cost_mv_ref(MB_PREDICTION_MODE m, const int near_mv_ref_ct[4])
{
    vp8_prob p [VP8_MVREFS-1];
    assert(NEARESTMV <= m  &&  m <= SPLITMV);
    vp8_mv_ref_probs(p, near_mv_ref_ct);
851 852
    return vp8_cost_token(vp8_mv_ref_tree, p,
                          vp8_mv_ref_encoding_array - NEARESTMV + m);
John Koleszar's avatar
John Koleszar committed
853 854
}

Scott LaVarnway's avatar
Scott LaVarnway committed
855
void vp8_set_mbmode_and_mvs(MACROBLOCK *x, MB_PREDICTION_MODE mb, int_mv *mv)
John Koleszar's avatar
John Koleszar committed
856
{
857
    x->e_mbd.mode_info_context->mbmi.mode = mb;
Scott LaVarnway's avatar
Scott LaVarnway committed
858
    x->e_mbd.mode_info_context->mbmi.mv.as_int = mv->as_int;
John Koleszar's avatar
John Koleszar committed
859 860 861 862 863 864
}

static int labels2mode(
    MACROBLOCK *x,
    int const *labelings, int which_label,
    B_PREDICTION_MODE this_mode,
Scott LaVarnway's avatar
Scott LaVarnway committed
865
    int_mv *this_mv, int_mv *best_ref_mv,
John Koleszar's avatar
John Koleszar committed
866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905
    int *mvcost[2]
)
{
    MACROBLOCKD *const xd = & x->e_mbd;
    MODE_INFO *const mic = xd->mode_info_context;
    const int mis = xd->mode_info_stride;

    int cost = 0;
    int thismvcost = 0;

    /* We have to be careful retrieving previously-encoded motion vectors.
       Ones from this macroblock have to be pulled from the BLOCKD array
       as they have not yet made it to the bmi array in our MB_MODE_INFO. */

    int i = 0;

    do
    {
        BLOCKD *const d = xd->block + i;
        const int row = i >> 2,  col = i & 3;

        B_PREDICTION_MODE m;

        if (labelings[i] != which_label)
            continue;

        if (col  &&  labelings[i] == labelings[i-1])
            m = LEFT4X4;
        else if (row  &&  labelings[i] == labelings[i-4])
            m = ABOVE4X4;
        else
        {
            // the only time we should do costing for new motion vector or mode
            // is when we are on a new label  (jbb May 08, 2007)
            switch (m = this_mode)
            {
            case NEW4X4 :
                thismvcost  = vp8_mv_bit_cost(this_mv, best_ref_mv, mvcost, 102);
                break;
            case LEFT4X4:
Scott LaVarnway's avatar
Scott LaVarnway committed
906
                this_mv->as_int = col ? d[-1].bmi.mv.as_int : left_block_mv(mic, i);
John Koleszar's avatar
John Koleszar committed
907 908
                break;
            case ABOVE4X4:
Scott LaVarnway's avatar
Scott LaVarnway committed
909
                this_mv->as_int = row ? d[-4].bmi.mv.as_int : above_block_mv(mic, i, mis);
John Koleszar's avatar
John Koleszar committed
910 911
                break;
            case ZERO4X4:
Scott LaVarnway's avatar
Scott LaVarnway committed
912
                this_mv->as_int = 0;
John Koleszar's avatar
John Koleszar committed
913 914 915 916 917 918 919
                break;
            default:
                break;
            }

            if (m == ABOVE4X4)  // replace above with left if same
            {
Scott LaVarnway's avatar
Scott LaVarnway committed
920
                int_mv left_mv;
Scott LaVarnway's avatar
Scott LaVarnway committed
921

Scott LaVarnway's avatar
Scott LaVarnway committed
922
                left_mv.as_int = col ? d[-1].bmi.mv.as_int :
Scott LaVarnway's avatar
Scott LaVarnway committed
923
                                        left_block_mv(mic, i);
John Koleszar's avatar
John Koleszar committed
924

Scott LaVarnway's avatar
Scott LaVarnway committed
925
                if (left_mv.as_int == this_mv->as_int)
John Koleszar's avatar
John Koleszar committed
926 927 928 929 930 931
                    m = LEFT4X4;
            }

            cost = x->inter_bmode_costs[ m];
        }

Scott LaVarnway's avatar
Scott LaVarnway committed
932
        d->bmi.mv.as_int = this_mv->as_int;
John Koleszar's avatar
John Koleszar committed
933

934 935 936
        x->partition_info->bmi[i].mode = m;
        x->partition_info->bmi[i].mv.as_int = this_mv->as_int;

John Koleszar's avatar
John Koleszar committed
937 938 939 940 941 942 943
    }
    while (++i < 16);

    cost += thismvcost ;
    return cost;
}

944 945 946
static int rdcost_mbsegment_y(MACROBLOCK *mb, const int *labels,
                              int which_label, ENTROPY_CONTEXT *ta,
                              ENTROPY_CONTEXT *tl)
John Koleszar's avatar
John Koleszar committed
947 948 949 950 951 952 953
{
    int cost = 0;
    int b;
    MACROBLOCKD *x = &mb->e_mbd;

    for (b = 0; b < 16; b++)
        if (labels[ b] == which_label)
Scott LaVarnway's avatar
Scott LaVarnway committed
954
            cost += cost_coeffs(mb, x->block + b, PLANE_TYPE_Y_WITH_DC,
955 956
                                ta + vp8_block2above[b],
                                tl + vp8_block2left[b]);
John Koleszar's avatar
John Koleszar committed
957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975

    return cost;

}
static unsigned int vp8_encode_inter_mb_segment(MACROBLOCK *x, int const *labels, int which_label, const vp8_encodemb_rtcd_vtable_t *rtcd)
{
    int i;
    unsigned int distortion = 0;

    for (i = 0; i < 16; i++)
    {
        if (labels[i] == which_label)
        {
            BLOCKD *bd = &x->e_mbd.block[i];
            BLOCK *be = &x->block[i];


            vp8_build_inter_predictors_b(bd, 16, x->e_mbd.subpixel_predict);
            ENCODEMB_INVOKE(rtcd, subb)(be, bd, 16);
Yaowu Xu's avatar
Yaowu Xu committed
976
            x->vp8_short_fdct4x4(be->src_diff, be->coeff, 32);
John Koleszar's avatar
John Koleszar committed
977 978 979

            // set to 0 no way to account for 2nd order DC so discount
            //be->coeff[0] = 0;
980
            x->quantize_b(be, bd);
John Koleszar's avatar
John Koleszar committed
981 982 983 984 985 986 987 988

            distortion += ENCODEMB_INVOKE(rtcd, berr)(be->coeff, bd->dqcoeff);
        }
    }

    return distortion;
}

989 990 991 992 993

static const unsigned int segmentation_to_sseshift[4] = {3, 3, 2, 0};


typedef struct
John Koleszar's avatar
John Koleszar committed
994
{
Scott LaVarnway's avatar
Scott LaVarnway committed
995 996
  int_mv *ref_mv;
  int_mv mvp;
997 998 999 1000 1001 1002 1003

  int segment_rd;
  int segment_num;
  int r;
  int d;
  int segment_yrate;
  B_PREDICTION_MODE modes[16];
1004
  int_mv mvs[16];
1005 1006 1007 1008 1009
  unsigned char eobs[16];

  int mvthresh;
  int *mdcounts;

Scott LaVarnway's avatar
Scott LaVarnway committed
1010
  int_mv sv_mvp[4];     // save 4 mvp from 8x8
1011 1012
  int sv_istep[2];  // save 2 initial step_param for 16x8/8x16

1013 1014 1015
} BEST_SEG_INFO;


1016 1017
static void rd_check_segment(VP8_COMP *cpi, MACROBLOCK *x,
                             BEST_SEG_INFO *bsi, unsigned int segmentation)
1018 1019
{
    int i;
John Koleszar's avatar
John Koleszar committed
1020 1021 1022
    int const *labels;
    int br = 0;
    int bd = 0;
1023
    B_PREDICTION_MODE this_mode;
John Koleszar's avatar
John Koleszar committed
1024

1025

1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041
    int label_count;
    int this_segment_rd = 0;
    int label_mv_thresh;
    int rate = 0;
    int sbr = 0;
    int sbd = 0;
    int segmentyrate = 0;

    vp8_variance_fn_ptr_t *v_fn_ptr;

    ENTROPY_CONTEXT_PLANES t_above, t_left;
    ENTROPY_CONTEXT *ta;
    ENTROPY_CONTEXT *tl;
    ENTROPY_CONTEXT_PLANES t_above_b, t_left_b;
    ENTROPY_CONTEXT *ta_b;
    ENTROPY_CONTEXT *tl_b;
John Koleszar's avatar
John Koleszar committed
1042

1043 1044
    vpx_memcpy(&t_above, x->e_mbd.above_context, sizeof(ENTROPY_CONTEXT_PLANES));
    vpx_memcpy(&t_left, x->e_mbd.left_context, sizeof(ENTROPY_CONTEXT_PLANES));
John Koleszar's avatar
John Koleszar committed
1045

1046 1047 1048 1049 1050 1051 1052
    ta = (ENTROPY_CONTEXT *)&t_above;
    tl = (ENTROPY_CONTEXT *)&t_left;
    ta_b = (ENTROPY_CONTEXT *)&t_above_b;
    tl_b = (ENTROPY_CONTEXT *)&t_left_b;

    br = 0;
    bd = 0;
1053

1054 1055 1056
    v_fn_ptr = &cpi->fn_ptr[segmentation];
    labels = vp8_mbsplits[segmentation];
    label_count = vp8_mbsplit_count[segmentation];
1057

1058 1059 1060 1061 1062 1063 1064 1065 1066
    // 64 makes this threshold really big effectively
    // making it so that we very rarely check mvs on
    // segments.   setting this to 1 would make mv thresh
    // roughly equal to what it is for macroblocks
    label_mv_thresh = 1 * bsi->mvthresh / label_count ;

    // Segmentation method overheads
    rate = vp8_cost_token(vp8_mbsplit_tree, vp8_mbsplit_probs, vp8_mbsplit_encodings + segmentation);
    rate += vp8_cost_mv_ref(SPLITMV, bsi->mdcounts);
1067
    this_segment_rd += RDCOST(x->rdmult, x->rddiv, rate, 0);
1068 1069 1070
    br += rate;

    for (i = 0; i < label_count; i++)
John Koleszar's avatar
John Koleszar committed
1071
    {
Scott LaVarnway's avatar
Scott LaVarnway committed
1072
        int_mv mode_mv[B_MODE_COUNT];
1073 1074 1075
        int best_label_rd = INT_MAX;
        B_PREDICTION_MODE mode_selected = ZERO4X4;
        int bestlabelyrate = 0;
John Koleszar's avatar
John Koleszar committed
1076

1077 1078 1079 1080 1081 1082 1083 1084 1085
        // search for the best motion vector on this segment
        for (this_mode = LEFT4X4; this_mode <= NEW4X4 ; this_mode ++)
        {
            int this_rd;
            int distortion;
            int labelyrate;
            ENTROPY_CONTEXT_PLANES t_above_s, t_left_s;
            ENTROPY_CONTEXT *ta_s;
            ENTROPY_CONTEXT *tl_s;
John Koleszar's avatar
John Koleszar committed
1086

1087 1088
            vpx_memcpy(&t_above_s, &t_above, sizeof(ENTROPY_CONTEXT_PLANES));
            vpx_memcpy(&t_left_s, &t_left, sizeof(ENTROPY_CONTEXT_PLANES));
John Koleszar's avatar
John Koleszar committed
1089

1090 1091
            ta_s = (ENTROPY_CONTEXT *)&t_above_s;
            tl_s = (ENTROPY_CONTEXT *)&t_left_s;
John Koleszar's avatar
John Koleszar committed
1092

1093
            if (this_mode == NEW4X4)
John Koleszar's avatar
John Koleszar committed
1094
            {
1095
                int sseshift;
John Koleszar's avatar
John Koleszar committed
1096
                int num00;
1097
                int step_param = 0;
1098
                int further_steps;
1099 1100 1101
                int n;
                int thissme;
                int bestsme = INT_MAX;
Scott LaVarnway's avatar
Scott LaVarnway committed
1102
                int_mv  temp_mv;
1103 1104
                BLOCK *c;
                BLOCKD *e;
1105

1106 1107 1108
                // Is the best so far sufficiently good that we cant justify doing and new motion search.
                if (best_label_rd < label_mv_thresh)
                    break;
John Koleszar's avatar
John Koleszar committed
1109

1110 1111 1112 1113
                if(cpi->compressor_speed)
                {
                    if (segmentation == BLOCK_8X16 || segmentation == BLOCK_16X8)
                    {
Scott LaVarnway's avatar
Scott LaVarnway committed
1114 1115 1116
                        bsi->mvp.as_int = bsi->sv_mvp[i].as_int;
                        if (i==1 && segmentation == BLOCK_16X8)
                          bsi->mvp.as_int = bsi->sv_mvp[2].as_int;
1117 1118 1119 1120 1121 1122 1123

                        step_param = bsi->sv_istep[i];
                    }

                    // use previous block's result as next block's MV predictor.
                    if (segmentation == BLOCK_4X4 && i>0)
                    {
Scott LaVarnway's avatar
Scott LaVarnway committed
1124 1125 1126
                        bsi->mvp.as_int = x->e_mbd.block[i-1].bmi.mv.as_int;
                        if (i==4 || i==8 || i==12)
                            bsi->mvp.as_int = x->e_mbd.block[i-4].bmi.mv.as_int;
1127 1128 1129 1130 1131 1132
                        step_param = 2;
                    }
                }

                further_steps = (MAX_MVSEARCH_STEPS - 1) - step_param;

John Koleszar's avatar
John Koleszar committed
1133
                {
1134
                    int sadpb = x->sadperbit4;
John Koleszar's avatar
John Koleszar committed
1135

1136
                    // find first label
1137
                    n = vp8_mbsplit_offset[segmentation][i];
John Koleszar's avatar
John Koleszar committed
1138

1139 1140
                    c = &x->block[n];
                    e = &x->e_mbd.block[n];
John Koleszar's avatar
John Koleszar committed
1141

1142
                    {
Scott LaVarnway's avatar
Scott LaVarnway committed
1143
                        bestsme = cpi->diamond_search_sad(x, c, e, &bsi->mvp,
1144
                                                          &mode_mv[NEW4X4], step_param,
1145
                                                          sadpb / 2, &num00, v_fn_ptr, x->mvcost, bsi->ref_mv);
John Koleszar's avatar
John Koleszar committed
1146

1147 1148
                        n = num00;
                        num00 = 0;
John Koleszar's avatar
John Koleszar committed
1149

1150
                        while (n < further_steps)
John Koleszar's avatar
John Koleszar committed
1151
                        {
1152
                            n++;
John Koleszar's avatar
John Koleszar committed
1153

1154 1155
                            if (num00)
                                num00--;
John Koleszar's avatar
John Koleszar committed
1156 1157
                            else
                            {
Scott LaVarnway's avatar
Scott LaVarnway committed
1158
                                thissme = cpi->diamond_search_sad(x, c, e, &bsi->mvp,
1159
                                                                  &temp_mv, step_param + n,
1160
                                                                  sadpb / 2, &num00, v_fn_ptr, x->mvcost, bsi->ref_mv);
1161 1162 1163 1164

                                if (thissme < bestsme)
                                {
                                    bestsme = thissme;
Scott LaVarnway's avatar
Scott LaVarnway committed
1165
                                    mode_mv[NEW4X4].as_int = temp_mv.as_int;
1166
                                }
John Koleszar's avatar
John Koleszar committed
1167 1168 1169 1170
                            }
                        }
                    }

1171 1172 1173 1174
                    sseshift = segmentation_to_sseshift[segmentation];

                    // Should we do a full search (best quality only)
                    if ((cpi->compressor_speed == 0) && (bestsme >> sseshift) > 4000)
John Koleszar's avatar
John Koleszar committed
1175
                    {
Scott LaVarnway's avatar
Scott LaVarnway committed
1176
                        int_mv full_mvp;
Yunqing Wang's avatar
Yunqing Wang committed
1177

Scott LaVarnway's avatar
Scott LaVarnway committed
1178 1179
                        full_mvp.as_mv.row = bsi->mvp.as_mv.row >>3;
                        full_mvp.as_mv.col = bsi->mvp.as_mv.col >>3;
Yunqing Wang's avatar
Yunqing Wang committed
1180 1181

                        thissme = cpi->full_search_sad(x, c, e, &full_mvp,
1182
                                                       sadpb / 4, 16, v_fn_ptr, x->mvcost, bsi->ref_mv);
1183 1184 1185 1186

                        if (thissme < bestsme)
                        {
                            bestsme = thissme;
Scott LaVarnway's avatar
Scott LaVarnway committed
1187
                            mode_mv[NEW4X4].as_int = e->bmi.mv.as_int;
1188
                        }
John Koleszar's avatar
John Koleszar committed
1189
                        else
1190 1191
                        {
                            // The full search result is actually worse so re-instate the previous best vector
Scott LaVarnway's avatar
Scott LaVarnway committed
1192
                            e->bmi.mv.as_int = mode_mv[NEW4X4].as_int;
1193
                        }
John Koleszar's avatar
John Koleszar committed
1194 1195 1196
                    }
                }

1197
                if (bestsme < INT_MAX)
John Koleszar's avatar
John Koleszar committed
1198
                {
1199
                    int distortion;
1200
                    unsigned int sse;
1201

1202 1203
                    if (!cpi->common.full_pixel)
                        cpi->find_fractional_mv_step(x, c, e, &mode_mv[NEW4X4],
1204
                                                     bsi->ref_mv, x->errorperbit / 2, v_fn_ptr, x-><