vp9_pickmode.c 30.7 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11
/*
 *  Copyright (c) 2014 The WebM project authors. All Rights Reserved.
 *
 *  Use of this source code is governed by a BSD-style license
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
 *  in the file PATENTS.  All contributing project authors may
 *  be found in the AUTHORS file in the root of the source tree.
 */

#include <assert.h>
12 13 14
#include <limits.h>
#include <math.h>
#include <stdio.h>
15

16 17 18 19
#include "./vp9_rtcd.h"

#include "vpx_mem/vpx_mem.h"

20
#include "vp9/common/vp9_blockd.h"
21 22
#include "vp9/common/vp9_common.h"
#include "vp9/common/vp9_mvref_common.h"
23 24
#include "vp9/common/vp9_reconinter.h"
#include "vp9/common/vp9_reconintra.h"
25

Dmitry Kovalev's avatar
Dmitry Kovalev committed
26
#include "vp9/encoder/vp9_encoder.h"
27
#include "vp9/encoder/vp9_pickmode.h"
28
#include "vp9/encoder/vp9_ratectrl.h"
29
#include "vp9/encoder/vp9_rd.h"
30

31 32 33 34 35 36
typedef struct {
  uint8_t *data;
  int stride;
  int in_use;
} PRED_BUFFER;

37
static int mv_refs_rt(const VP9_COMMON *cm, const MACROBLOCKD *xd,
38 39 40 41
                      const TileInfo *const tile,
                      MODE_INFO *mi, MV_REFERENCE_FRAME ref_frame,
                      int_mv *mv_ref_list,
                      int mi_row, int mi_col) {
42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60
  const int *ref_sign_bias = cm->ref_frame_sign_bias;
  int i, refmv_count = 0;

  const POSITION *const mv_ref_search = mv_ref_blocks[mi->mbmi.sb_type];

  int different_ref_found = 0;
  int context_counter = 0;
  int const_motion = 0;

  // Blank the reference vector list
  vpx_memset(mv_ref_list, 0, sizeof(*mv_ref_list) * MAX_MV_REF_CANDIDATES);

  // The nearest 2 blocks are treated differently
  // if the size < 8x8 we get the mv from the bmi substructure,
  // and we also need to keep a mode count.
  for (i = 0; i < 2; ++i) {
    const POSITION *const mv_ref = &mv_ref_search[i];
    if (is_inside(tile, mi_col, mi_row, cm->mi_rows, mv_ref)) {
      const MODE_INFO *const candidate_mi = xd->mi[mv_ref->col + mv_ref->row *
hkuang's avatar
hkuang committed
61
                                                   xd->mi_stride].src_mi;
62 63 64 65 66 67
      const MB_MODE_INFO *const candidate = &candidate_mi->mbmi;
      // Keep counts for entropy encoding.
      context_counter += mode_2_counter[candidate->mode];
      different_ref_found = 1;

      if (candidate->ref_frame[0] == ref_frame)
68 69
        ADD_MV_REF_LIST(get_sub_block_mv(candidate_mi, 0, mv_ref->col, -1),
                        refmv_count, mv_ref_list, Done);
70 71 72 73 74 75 76 77 78 79 80 81
    }
  }

  const_motion = 1;

  // Check the rest of the neighbors in much the same way
  // as before except we don't need to keep track of sub blocks or
  // mode counts.
  for (; i < MVREF_NEIGHBOURS && !refmv_count; ++i) {
    const POSITION *const mv_ref = &mv_ref_search[i];
    if (is_inside(tile, mi_col, mi_row, cm->mi_rows, mv_ref)) {
      const MB_MODE_INFO *const candidate = &xd->mi[mv_ref->col + mv_ref->row *
hkuang's avatar
hkuang committed
82
                                                    xd->mi_stride].src_mi->mbmi;
83 84 85
      different_ref_found = 1;

      if (candidate->ref_frame[0] == ref_frame)
86
        ADD_MV_REF_LIST(candidate->mv[0], refmv_count, mv_ref_list, Done);
87 88 89 90 91 92 93 94 95 96 97
    }
  }

  // Since we couldn't find 2 mvs from the same reference frame
  // go back through the neighbors and find motion vectors from
  // different reference frames.
  if (different_ref_found && !refmv_count) {
    for (i = 0; i < MVREF_NEIGHBOURS; ++i) {
      const POSITION *mv_ref = &mv_ref_search[i];
      if (is_inside(tile, mi_col, mi_row, cm->mi_rows, mv_ref)) {
        const MB_MODE_INFO *const candidate = &xd->mi[mv_ref->col + mv_ref->row
hkuang's avatar
hkuang committed
98
                                              * xd->mi_stride].src_mi->mbmi;
99 100

        // If the candidate is INTRA we don't want to consider its mv.
101 102
        IF_DIFF_REF_FRAME_ADD_MV(candidate, ref_frame, ref_sign_bias,
                                 refmv_count, mv_ref_list, Done);
103 104 105 106 107 108 109 110 111 112 113 114 115 116 117
      }
    }
  }

 Done:

  mi->mbmi.mode_context[ref_frame] = counter_to_context[context_counter];

  // Clamp vectors
  for (i = 0; i < MAX_MV_REF_CANDIDATES; ++i)
    clamp_mv_ref(&mv_ref_list[i].as_mv, xd);

  return const_motion;
}

118 119 120 121
static int combined_motion_search(VP9_COMP *cpi, MACROBLOCK *x,
                                  BLOCK_SIZE bsize, int mi_row, int mi_col,
                                  int_mv *tmp_mv, int *rate_mv,
                                  int64_t best_rd_sofar) {
122
  MACROBLOCKD *xd = &x->e_mbd;
hkuang's avatar
hkuang committed
123
  MB_MODE_INFO *mbmi = &xd->mi[0].src_mi->mbmi;
124
  struct buf_2d backup_yv12[MAX_MB_PLANE] = {{0, 0}};
125 126
  const int step_param = cpi->sf.mv.fullpel_search_step_param;
  const int sadpb = x->sadperbit16;
127
  MV mvp_full;
128
  const int ref = mbmi->ref_frame[0];
129
  const MV ref_mv = mbmi->ref_mvs[ref][0].as_mv;
130 131 132 133 134 135 136
  int dis;
  int rate_mode;
  const int tmp_col_min = x->mv_col_min;
  const int tmp_col_max = x->mv_col_max;
  const int tmp_row_min = x->mv_row_min;
  const int tmp_row_max = x->mv_row_max;
  int rv = 0;
137
  int cost_list[5];
138 139
  const YV12_BUFFER_CONFIG *scaled_ref_frame = vp9_get_scaled_ref_frame(cpi,
                                                                        ref);
Yaowu Xu's avatar
Yaowu Xu committed
140 141 142 143
  if (cpi->common.show_frame &&
      (x->pred_mv_sad[ref] >> 3) > x->pred_mv_sad[LAST_FRAME])
    return rv;

144 145 146 147 148 149 150
  if (scaled_ref_frame) {
    int i;
    // Swap out the reference frame for a version that's been scaled to
    // match the resolution of the current frame, allowing the existing
    // motion search code to be used without additional modifications.
    for (i = 0; i < MAX_MB_PLANE; i++)
      backup_yv12[i] = xd->plane[i].pre[0];
151
    vp9_setup_pre_planes(xd, 0, scaled_ref_frame, mi_row, mi_col, NULL);
152
  }
153
  vp9_set_mv_search_range(x, &ref_mv);
154

Yaowu Xu's avatar
Yaowu Xu committed
155 156 157 158
  assert(x->mv_best_ref_index[ref] <= 2);
  if (x->mv_best_ref_index[ref] < 2)
    mvp_full = mbmi->ref_mvs[ref][x->mv_best_ref_index[ref]].as_mv;
  else
Dmitry Kovalev's avatar
Dmitry Kovalev committed
159
    mvp_full = x->pred_mv[ref];
160 161 162 163

  mvp_full.col >>= 3;
  mvp_full.row >>= 3;

164
  vp9_full_pixel_search(cpi, x, bsize, &mvp_full, step_param, sadpb,
165
                        cond_cost_list(cpi, cost_list),
166
                        &ref_mv, &tmp_mv->as_mv, INT_MAX, 0);
167

168 169 170 171 172
  x->mv_col_min = tmp_col_min;
  x->mv_col_max = tmp_col_max;
  x->mv_row_min = tmp_row_min;
  x->mv_row_max = tmp_row_max;

173 174 175
  // calculate the bit cost on motion vector
  mvp_full.row = tmp_mv->as_mv.row * 8;
  mvp_full.col = tmp_mv->as_mv.col * 8;
176

177 178
  *rate_mv = vp9_mv_bit_cost(&mvp_full, &ref_mv,
                             x->nmvjointcost, x->mvcost, MV_COST_WEIGHT);
179

180 181 182 183 184 185 186 187 188 189 190 191
  rate_mode = cpi->inter_mode_cost[mbmi->mode_context[ref]]
                                  [INTER_OFFSET(NEWMV)];
  rv = !(RDCOST(x->rdmult, x->rddiv, (*rate_mv + rate_mode), 0) >
         best_rd_sofar);

  if (rv) {
    cpi->find_fractional_mv_step(x, &tmp_mv->as_mv, &ref_mv,
                                 cpi->common.allow_high_precision_mv,
                                 x->errorperbit,
                                 &cpi->fn_ptr[bsize],
                                 cpi->sf.mv.subpel_force_stop,
                                 cpi->sf.mv.subpel_iters_per_step,
192
                                 cond_cost_list(cpi, cost_list),
193
                                 x->nmvjointcost, x->mvcost,
194
                                 &dis, &x->pred_sse[ref], NULL, 0, 0);
195
    x->pred_mv[ref] = tmp_mv->as_mv;
196 197 198 199 200 201 202
  }

  if (scaled_ref_frame) {
    int i;
    for (i = 0; i < MAX_MB_PLANE; i++)
      xd->plane[i].pre[0] = backup_yv12[i];
  }
203
  return rv;
204 205
}

206

207 208
static void model_rd_for_sb_y(VP9_COMP *cpi, BLOCK_SIZE bsize,
                              MACROBLOCK *x, MACROBLOCKD *xd,
209 210
                              int *out_rate_sum, int64_t *out_dist_sum,
                              unsigned int *var_y, unsigned int *sse_y) {
211 212 213 214
  // Note our transform coeffs are 8 times an orthogonal transform.
  // Hence quantizer step is also 8 times. To get effective quantizer
  // we need to divide by 8 before sending to modeling function.
  unsigned int sse;
215 216
  int rate;
  int64_t dist;
217 218
  struct macroblock_plane *const p = &x->plane[0];
  struct macroblockd_plane *const pd = &xd->plane[0];
219 220
  const uint32_t dc_quant = pd->dequant[0];
  const uint32_t ac_quant = pd->dequant[1];
221 222
  unsigned int var = cpi->fn_ptr[bsize].vf(p->src.buf, p->src.stride,
                                           pd->dst.buf, pd->dst.stride, &sse);
223 224 225
  *var_y = var;
  *sse_y = sse;

226
  if (sse < dc_quant * dc_quant >> 6)
227
    x->skip_txfm[0] = 1;
228
  else if (var < ac_quant * ac_quant >> 6)
229
    x->skip_txfm[0] = 2;
230
  else
231
    x->skip_txfm[0] = 0;
232

233 234
  if (cpi->common.tx_mode == TX_MODE_SELECT) {
    if (sse > (var << 2))
hkuang's avatar
hkuang committed
235
      xd->mi[0].src_mi->mbmi.tx_size =
236 237
          MIN(max_txsize_lookup[bsize],
              tx_mode_to_biggest_tx_size[cpi->common.tx_mode]);
238
    else
hkuang's avatar
hkuang committed
239
      xd->mi[0].src_mi->mbmi.tx_size = TX_8X8;
240 241 242 243

    if (cpi->sf.partition_search_type == VAR_BASED_PARTITION &&
        xd->mi[0].src_mi->mbmi.tx_size > TX_16X16)
      xd->mi[0].src_mi->mbmi.tx_size = TX_16X16;
244
  } else {
hkuang's avatar
hkuang committed
245
    xd->mi[0].src_mi->mbmi.tx_size =
246 247
        MIN(max_txsize_lookup[bsize],
            tx_mode_to_biggest_tx_size[cpi->common.tx_mode]);
248 249
  }

250 251 252 253 254 255 256 257 258
#if CONFIG_VP9_HIGHBITDEPTH
  if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) {
    vp9_model_rd_from_var_lapndz(sse - var, 1 << num_pels_log2_lookup[bsize],
                                 dc_quant >> (xd->bd - 5), &rate, &dist);
  } else {
    vp9_model_rd_from_var_lapndz(sse - var, 1 << num_pels_log2_lookup[bsize],
                                 dc_quant >> 3, &rate, &dist);
  }
#else
259 260
  vp9_model_rd_from_var_lapndz(sse - var, 1 << num_pels_log2_lookup[bsize],
                               dc_quant >> 3, &rate, &dist);
261 262
#endif  // CONFIG_VP9_HIGHBITDEPTH

263
  *out_rate_sum = rate >> 1;
264
  *out_dist_sum = dist << 3;
265

266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287
#if CONFIG_VP9_HIGHBITDEPTH
  if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) {
    vp9_model_rd_from_var_lapndz(var,
                                 1 << num_pels_log2_lookup[bsize],
                                 ac_quant >> (xd->bd - 5),
                                 &rate,
                                 &dist);
  } else {
    vp9_model_rd_from_var_lapndz(var,
                                 1 << num_pels_log2_lookup[bsize],
                                 ac_quant >> 3,
                                 &rate,
                                 &dist);
  }
#else
  vp9_model_rd_from_var_lapndz(var,
                               1 << num_pels_log2_lookup[bsize],
                               ac_quant >> 3,
                               &rate,
                               &dist);
#endif  // CONFIG_VP9_HIGHBITDEPTH

288 289
  *out_rate_sum += rate;
  *out_dist_sum += dist << 4;
290 291
}

292 293 294 295 296 297 298 299 300 301 302 303 304
static int get_pred_buffer(PRED_BUFFER *p, int len) {
  int i;

  for (i = 0; i < len; i++) {
    if (!p[i].in_use) {
      p[i].in_use = 1;
      return i;
    }
  }
  return -1;
}

static void free_pred_buffer(PRED_BUFFER *p) {
305 306
  if (p != NULL)
    p->in_use = 0;
307 308
}

309 310 311 312 313 314 315 316
static void encode_breakout_test(VP9_COMP *cpi, MACROBLOCK *x,
                                 BLOCK_SIZE bsize, int mi_row, int mi_col,
                                 MV_REFERENCE_FRAME ref_frame,
                                 PREDICTION_MODE this_mode,
                                 unsigned int var_y, unsigned int sse_y,
                                 struct buf_2d yv12_mb[][MAX_MB_PLANE],
                                 int *rate, int64_t *dist) {
  MACROBLOCKD *xd = &x->e_mbd;
hkuang's avatar
hkuang committed
317
  MB_MODE_INFO *mbmi = &xd->mi[0].src_mi->mbmi;
318 319 320 321 322 323 324 325 326 327 328 329 330 331 332

  const BLOCK_SIZE uv_size = get_plane_block_size(bsize, &xd->plane[1]);
  unsigned int var = var_y, sse = sse_y;
  // Skipping threshold for ac.
  unsigned int thresh_ac;
  // Skipping threshold for dc.
  unsigned int thresh_dc;
  if (x->encode_breakout > 0) {
    // Set a maximum for threshold to avoid big PSNR loss in low bit rate
    // case. Use extreme low threshold for static frames to limit
    // skipping.
    const unsigned int max_thresh = 36000;
    // The encode_breakout input
    const unsigned int min_thresh =
        MIN(((unsigned int)x->encode_breakout << 4), max_thresh);
333 334 335
#if CONFIG_VP9_HIGHBITDEPTH
    const int shift = 2 * xd->bd - 16;
#endif
336 337 338

    // Calculate threshold according to dequant value.
    thresh_ac = (xd->plane[0].dequant[1] * xd->plane[0].dequant[1]) / 9;
339 340 341 342 343
#if CONFIG_VP9_HIGHBITDEPTH
    if ((xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) && shift > 0) {
      thresh_ac = ROUND_POWER_OF_TWO(thresh_ac, shift);
    }
#endif  // CONFIG_VP9_HIGHBITDEPTH
344 345 346 347
    thresh_ac = clamp(thresh_ac, min_thresh, max_thresh);

    // Adjust ac threshold according to partition size.
    thresh_ac >>=
348
        8 - (b_width_log2_lookup[bsize] + b_height_log2_lookup[bsize]);
349 350

    thresh_dc = (xd->plane[0].dequant[0] * xd->plane[0].dequant[0] >> 6);
351 352 353 354 355
#if CONFIG_VP9_HIGHBITDEPTH
    if ((xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) && shift > 0) {
      thresh_dc = ROUND_POWER_OF_TWO(thresh_dc, shift);
    }
#endif  // CONFIG_VP9_HIGHBITDEPTH
356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391
  } else {
    thresh_ac = 0;
    thresh_dc = 0;
  }

  // Y skipping condition checking for ac and dc.
  if (var <= thresh_ac && (sse - var) <= thresh_dc) {
    unsigned int sse_u, sse_v;
    unsigned int var_u, var_v;

    // Skip UV prediction unless breakout is zero (lossless) to save
    // computation with low impact on the result
    if (x->encode_breakout == 0) {
      xd->plane[1].pre[0] = yv12_mb[ref_frame][1];
      xd->plane[2].pre[0] = yv12_mb[ref_frame][2];
      vp9_build_inter_predictors_sbuv(xd, mi_row, mi_col, bsize);
    }

    var_u = cpi->fn_ptr[uv_size].vf(x->plane[1].src.buf,
                                    x->plane[1].src.stride,
                                    xd->plane[1].dst.buf,
                                    xd->plane[1].dst.stride, &sse_u);

    // U skipping condition checking
    if ((var_u * 4 <= thresh_ac) && (sse_u - var_u <= thresh_dc)) {
      var_v = cpi->fn_ptr[uv_size].vf(x->plane[2].src.buf,
                                      x->plane[2].src.stride,
                                      xd->plane[2].dst.buf,
                                      xd->plane[2].dst.stride, &sse_v);

      // V skipping condition checking
      if ((var_v * 4 <= thresh_ac) && (sse_v - var_v <= thresh_dc)) {
        x->skip = 1;

        // The cost of skip bit needs to be added.
        *rate = cpi->inter_mode_cost[mbmi->mode_context[ref_frame]]
392
                                    [INTER_OFFSET(this_mode)];
393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408

        // More on this part of rate
        // rate += vp9_cost_bit(vp9_get_skip_prob(cm, xd), 1);

        // Scaling factor for SSE from spatial domain to frequency
        // domain is 16. Adjust distortion accordingly.
        // TODO(yunqingwang): In this function, only y-plane dist is
        // calculated.
        *dist = (sse << 4);  // + ((sse_u + sse_v) << 4);

        // *disable_skip = 1;
      }
    }
  }
}

409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441
struct estimate_block_intra_args {
  VP9_COMP *cpi;
  MACROBLOCK *x;
  PREDICTION_MODE mode;
  int rate;
  int64_t dist;
};

static void estimate_block_intra(int plane, int block, BLOCK_SIZE plane_bsize,
                                 TX_SIZE tx_size, void *arg) {
  struct estimate_block_intra_args* const args = arg;
  VP9_COMP *const cpi = args->cpi;
  MACROBLOCK *const x = args->x;
  MACROBLOCKD *const xd = &x->e_mbd;
  struct macroblock_plane *const p = &x->plane[0];
  struct macroblockd_plane *const pd = &xd->plane[0];
  const BLOCK_SIZE bsize_tx = txsize_to_bsize[tx_size];
  uint8_t *const src_buf_base = p->src.buf;
  uint8_t *const dst_buf_base = pd->dst.buf;
  const int src_stride = p->src.stride;
  const int dst_stride = pd->dst.stride;
  int i, j;
  int rate;
  int64_t dist;
  unsigned int var_y, sse_y;
  txfrm_block_to_raster_xy(plane_bsize, tx_size, block, &i, &j);
  assert(plane == 0);
  (void) plane;

  p->src.buf = &src_buf_base[4 * (j * src_stride + i)];
  pd->dst.buf = &dst_buf_base[4 * (j * dst_stride + i)];
  // Use source buffer as an approximation for the fully reconstructed buffer.
  vp9_predict_intra_block(xd, block >> (2 * tx_size),
442
                          b_width_log2_lookup[plane_bsize],
443 444 445 446 447 448 449 450 451 452 453 454
                          tx_size, args->mode,
                          p->src.buf, src_stride,
                          pd->dst.buf, dst_stride,
                          i, j, 0);
  // This procedure assumes zero offset from p->src.buf and pd->dst.buf.
  model_rd_for_sb_y(cpi, bsize_tx, x, xd, &rate, &dist, &var_y, &sse_y);
  p->src.buf = src_buf_base;
  pd->dst.buf = dst_buf_base;
  args->rate += rate;
  args->dist += dist;
}

455 456
static const THR_MODES mode_idx[MAX_REF_FRAMES][4] = {
  {THR_DC, THR_H_PRED, THR_V_PRED},
457 458 459 460 461
  {THR_NEARESTMV, THR_NEARMV, THR_ZEROMV, THR_NEWMV},
  {THR_NEARESTG, THR_NEARG, THR_ZEROG, THR_NEWG},
  {THR_NEARESTA, THR_NEARA, THR_ZEROA, THR_NEWA},
};

462 463
// TODO(jingning) placeholder for inter-frame non-RD mode decision.
// this needs various further optimizations. to be continued..
464
void vp9_pick_inter_mode(VP9_COMP *cpi, MACROBLOCK *x,
465
                         TileDataEnc *tile_data,
466 467
                         int mi_row, int mi_col, RD_COST *rd_cost,
                         BLOCK_SIZE bsize, PICK_MODE_CONTEXT *ctx) {
468
  VP9_COMMON *const cm = &cpi->common;
469
  TileInfo *const tile_info = &tile_data->tile_info;
470
  MACROBLOCKD *const xd = &x->e_mbd;
hkuang's avatar
hkuang committed
471
  MB_MODE_INFO *const mbmi = &xd->mi[0].src_mi->mbmi;
472
  struct macroblockd_plane *const pd = &xd->plane[0];
473
  PREDICTION_MODE best_mode = ZEROMV;
474
  MV_REFERENCE_FRAME ref_frame, best_ref_frame = LAST_FRAME;
475
  TX_SIZE best_tx_size = MIN(max_txsize_lookup[bsize],
476
                             tx_mode_to_biggest_tx_size[cm->tx_mode]);
477
  INTERP_FILTER best_pred_filter = EIGHTTAP;
478 479 480 481
  int_mv frame_mv[MB_MODE_COUNT][MAX_REF_FRAMES];
  struct buf_2d yv12_mb[4][MAX_MB_PLANE];
  static const int flag_list[4] = { 0, VP9_LAST_FLAG, VP9_GOLD_FLAG,
                                    VP9_ALT_FLAG };
482
  RD_COST this_rdc, best_rdc;
483
  uint8_t skip_txfm = 0;
484 485 486
  // var_y and sse_y are saved to be used in skipping checking
  unsigned int var_y = UINT_MAX;
  unsigned int sse_y = UINT_MAX;
487 488 489 490
  // Reduce the intra cost penalty for small blocks (<=16x16).
  const int reduction_fac =
      (cpi->sf.partition_search_type == VAR_BASED_PARTITION &&
       bsize <= BLOCK_16X16) ? 4 : 1;
491
  const int intra_cost_penalty = vp9_get_intra_cost_penalty(
492
      cm->base_qindex, cm->y_dc_delta_q, cm->bit_depth) / reduction_fac;
493 494
  const int64_t inter_mode_thresh = RDCOST(x->rdmult, x->rddiv,
                                           intra_cost_penalty, 0);
495
  const int intra_mode_cost = 50;
496

497
  const int8_t segment_id = mbmi->segment_id;
Dmitry Kovalev's avatar
Dmitry Kovalev committed
498
  const int *const rd_threshes = cpi->rd.threshes[segment_id][bsize];
499
  const int *const rd_thresh_freq_fact = tile_data->thresh_freq_fact[bsize];
Yaowu Xu's avatar
Yaowu Xu committed
500
  INTERP_FILTER filter_ref;
501
  const int bsl = mi_width_log2_lookup[bsize];
502
  const int pred_filter_search = cm->interp_filter == SWITCHABLE ?
503 504
      (((mi_row + mi_col) >> bsl) +
       get_chessboard_index(cm->current_video_frame)) & 0x1 : 0;
505
  int const_motion[MAX_REF_FRAMES] = { 0 };
506 507
  const int bh = num_4x4_blocks_high_lookup[bsize] << 2;
  const int bw = num_4x4_blocks_wide_lookup[bsize] << 2;
508 509
  // For speed 6, the result of interp filter is reused later in actual encoding
  // process.
510 511 512
  // tmp[3] points to dst buffer, and the other 3 point to allocated buffers.
  PRED_BUFFER tmp[4];
  DECLARE_ALIGNED_ARRAY(16, uint8_t, pred_buf, 3 * 64 * 64);
513 514 515
#if CONFIG_VP9_HIGHBITDEPTH
  DECLARE_ALIGNED_ARRAY(16, uint16_t, pred_buf_16, 3 * 64 * 64);
#endif
516 517 518
  struct buf_2d orig_dst = pd->dst;
  PRED_BUFFER *best_pred = NULL;
  PRED_BUFFER *this_mode_pred = NULL;
519
  const int pixels_in_block = bh * bw;
520
  int reuse_inter_pred = cpi->sf.reuse_inter_pred_sby && ctx->pred_pixel_ready;
521

522
  if (reuse_inter_pred) {
523
    int i;
524
    for (i = 0; i < 3; i++) {
525 526 527 528 529 530 531 532
#if CONFIG_VP9_HIGHBITDEPTH
      if (cm->use_highbitdepth)
        tmp[i].data = CONVERT_TO_BYTEPTR(&pred_buf_16[pixels_in_block * i]);
      else
        tmp[i].data = &pred_buf[pixels_in_block * i];
#else
      tmp[i].data = &pred_buf[pixels_in_block * i];
#endif  // CONFIG_VP9_HIGHBITDEPTH
533 534 535 536 537 538 539 540
      tmp[i].stride = bw;
      tmp[i].in_use = 0;
    }
    tmp[3].data = pd->dst.buf;
    tmp[3].stride = pd->dst.stride;
    tmp[3].in_use = 0;
  }

541 542
  x->skip_encode = cpi->sf.skip_encode_frame && x->q_index < QIDX_SKIP_THRESH;
  x->skip = 0;
543

Yaowu Xu's avatar
Yaowu Xu committed
544 545 546 547 548 549 550
  if (xd->up_available)
    filter_ref = xd->mi[-xd->mi_stride].src_mi->mbmi.interp_filter;
  else if (xd->left_available)
    filter_ref = xd->mi[-1].src_mi->mbmi.interp_filter;
  else
    filter_ref = cm->interp_filter;

551
  // initialize mode decisions
552 553 554
  vp9_rd_cost_reset(&best_rdc);
  vp9_rd_cost_reset(&this_rdc);
  vp9_rd_cost_reset(rd_cost);
555 556 557 558 559
  vpx_memset(mbmi, 0, sizeof(MB_MODE_INFO));
  mbmi->sb_type = bsize;
  mbmi->ref_frame[0] = NONE;
  mbmi->ref_frame[1] = NONE;
  mbmi->tx_size = MIN(max_txsize_lookup[bsize],
560 561 562
                      tx_mode_to_biggest_tx_size[cm->tx_mode]);
  mbmi->interp_filter = cm->interp_filter == SWITCHABLE ?
                        EIGHTTAP : cm->interp_filter;
563
  mbmi->segment_id = segment_id;
564

565
  for (ref_frame = LAST_FRAME; ref_frame <= GOLDEN_FRAME; ++ref_frame) {
566
    PREDICTION_MODE this_mode;
567
    x->pred_mv_sad[ref_frame] = INT_MAX;
568 569 570
    frame_mv[NEWMV][ref_frame].as_int = INVALID_MV;
    frame_mv[ZEROMV][ref_frame].as_int = 0;

571
    if (cpi->ref_frame_flags & flag_list[ref_frame]) {
572 573 574 575 576 577
      const YV12_BUFFER_CONFIG *yv12 = get_ref_frame_buffer(cpi, ref_frame);
      int_mv *const candidates = mbmi->ref_mvs[ref_frame];
      const struct scale_factors *const sf = &cm->frame_refs[ref_frame - 1].sf;
      vp9_setup_pred_block(xd, yv12_mb[ref_frame], yv12, mi_row, mi_col,
                           sf, sf);

578
      if (!cm->error_resilient_mode)
579
        vp9_find_mv_refs(cm, xd, tile_info, xd->mi[0].src_mi, ref_frame,
580 581
                         candidates, mi_row, mi_col);
      else
582 583
        const_motion[ref_frame] = mv_refs_rt(cm, xd, tile_info,
                                             xd->mi[0].src_mi,
584 585 586 587 588 589 590 591 592 593
                                             ref_frame, candidates,
                                             mi_row, mi_col);

      vp9_find_best_ref_mvs(xd, cm->allow_high_precision_mv, candidates,
                            &frame_mv[NEARESTMV][ref_frame],
                            &frame_mv[NEARMV][ref_frame]);

      if (!vp9_is_scaled(sf) && bsize >= BLOCK_8X8)
        vp9_mv_pred(cpi, x, yv12_mb[ref_frame][0].buf, yv12->y_stride,
                    ref_frame, bsize);
594
    } else {
595
      continue;
596
    }
597 598 599 600 601 602 603

    // Select prediction reference frames.
    xd->plane[0].pre[0] = yv12_mb[ref_frame][0];

    clamp_mv2(&frame_mv[NEARESTMV][ref_frame].as_mv, xd);
    clamp_mv2(&frame_mv[NEARMV][ref_frame].as_mv, xd);

604 605
    mbmi->ref_frame[0] = ref_frame;

606
    for (this_mode = NEARESTMV; this_mode <= NEWMV; ++this_mode) {
607
      int rate_mv = 0;
608
      int mode_rd_thresh;
609
      int mode_index = mode_idx[ref_frame][INTER_OFFSET(this_mode)];
610

611
      if (const_motion[ref_frame] && this_mode == NEARMV)
612 613
        continue;

614
      if (!(cpi->sf.inter_mode_mask[bsize] & (1 << this_mode)))
615
        continue;
Jim Bankoski's avatar
Jim Bankoski committed
616

617
      mode_rd_thresh = rd_threshes[mode_index];
618
      if (rd_less_than_thresh(best_rdc.rdcost, mode_rd_thresh,
619
                              rd_thresh_freq_fact[mode_index]))
620 621
        continue;

Jim Bankoski's avatar
Jim Bankoski committed
622
      if (this_mode == NEWMV) {
623 624
        if (ref_frame > LAST_FRAME)
          continue;
625
        if (cpi->sf.partition_search_type != VAR_BASED_PARTITION &&
626
            this_rdc.rdcost < (int64_t)(1 << num_pels_log2_lookup[bsize]))
627
          continue;
628 629
        if (!combined_motion_search(cpi, x, bsize, mi_row, mi_col,
                                    &frame_mv[NEWMV][ref_frame],
630
                                    &rate_mv, best_rdc.rdcost))
Jim Bankoski's avatar
Jim Bankoski committed
631 632 633
          continue;
      }

634 635 636
      if (this_mode != NEARESTMV &&
          frame_mv[this_mode][ref_frame].as_int ==
              frame_mv[NEARESTMV][ref_frame].as_int)
637
        continue;
638

639 640
      mbmi->mode = this_mode;
      mbmi->mv[0].as_int = frame_mv[this_mode][ref_frame].as_int;
641

642 643 644
      // Search for the best prediction filter type, when the resulting
      // motion vector is at sub-pixel accuracy level for luma component, i.e.,
      // the last three bits are all zeros.
645
      if (reuse_inter_pred) {
646
        if (!this_mode_pred) {
647 648 649 650 651 652 653 654
          this_mode_pred = &tmp[3];
        } else {
          this_mode_pred = &tmp[get_pred_buffer(tmp, 3)];
          pd->dst.buf = this_mode_pred->data;
          pd->dst.stride = bw;
        }
      }

655
      if ((this_mode == NEWMV || filter_ref == SWITCHABLE) &&
656
          pred_filter_search &&
657 658 659 660
          ((mbmi->mv[0].as_mv.row & 0x07) != 0 ||
           (mbmi->mv[0].as_mv.col & 0x07) != 0)) {
        int pf_rate[3];
        int64_t pf_dist[3];
661 662
        unsigned int pf_var[3];
        unsigned int pf_sse[3];
663
        TX_SIZE pf_tx_size[3];
664 665
        int64_t best_cost = INT64_MAX;
        INTERP_FILTER best_filter = SWITCHABLE, filter;
666
        PRED_BUFFER *current_pred = this_mode_pred;
667 668 669 670 671 672

        for (filter = EIGHTTAP; filter <= EIGHTTAP_SHARP; ++filter) {
          int64_t cost;
          mbmi->interp_filter = filter;
          vp9_build_inter_predictors_sby(xd, mi_row, mi_col, bsize);
          model_rd_for_sb_y(cpi, bsize, x, xd, &pf_rate[filter],
673
                            &pf_dist[filter], &pf_var[filter], &pf_sse[filter]);
674 675 676
          cost = RDCOST(x->rdmult, x->rddiv,
                        vp9_get_switchable_rate(cpi) + pf_rate[filter],
                        pf_dist[filter]);
677
          pf_tx_size[filter] = mbmi->tx_size;
678
          if (cost < best_cost) {
679 680
            best_filter = filter;
            best_cost = cost;
681
            skip_txfm = x->skip_txfm[0];
682

683
            if (reuse_inter_pred) {
684 685 686 687 688 689 690 691 692 693 694
              if (this_mode_pred != current_pred) {
                free_pred_buffer(this_mode_pred);
                this_mode_pred = current_pred;
              }

              if (filter < EIGHTTAP_SHARP) {
                current_pred = &tmp[get_pred_buffer(tmp, 3)];
                pd->dst.buf = current_pred->data;
                pd->dst.stride = bw;
              }
            }
695
          }
696 697
        }

698
        if (reuse_inter_pred && this_mode_pred != current_pred)
699 700
          free_pred_buffer(current_pred);

701
        mbmi->interp_filter = best_filter;
702
        mbmi->tx_size = pf_tx_size[mbmi->interp_filter];
703 704
        this_rdc.rate = pf_rate[mbmi->interp_filter];
        this_rdc.dist = pf_dist[mbmi->interp_filter];
705 706
        var_y = pf_var[mbmi->interp_filter];
        sse_y = pf_sse[mbmi->interp_filter];
707
        x->skip_txfm[0] = skip_txfm;
708 709 710
      } else {
        mbmi->interp_filter = (filter_ref == SWITCHABLE) ? EIGHTTAP: filter_ref;
        vp9_build_inter_predictors_sby(xd, mi_row, mi_col, bsize);
711 712
        model_rd_for_sb_y(cpi, bsize, x, xd, &this_rdc.rate, &this_rdc.dist,
                          &var_y, &sse_y);
713 714
      }

715 716
      this_rdc.rate += rate_mv;
      this_rdc.rate += cpi->inter_mode_cost[mbmi->mode_context[ref_frame]]
717
                                  [INTER_OFFSET(this_mode)];
718 719
      this_rdc.rdcost = RDCOST(x->rdmult, x->rddiv,
                               this_rdc.rate, this_rdc.dist);
720

721 722
      // Skipping checking: test to see if this block can be reconstructed by
      // prediction only.
723
      if (cpi->allow_encode_breakout) {
724
        encode_breakout_test(cpi, x, bsize, mi_row, mi_col, ref_frame,
725 726
                             this_mode, var_y, sse_y, yv12_mb,
                             &this_rdc.rate, &this_rdc.dist);
727
        if (x->skip) {
728 729 730
          this_rdc.rate += rate_mv;
          this_rdc.rdcost = RDCOST(x->rdmult, x->rddiv,
                                   this_rdc.rate, this_rdc.dist);
731 732 733
        }
      }

734
#if CONFIG_VP9_TEMPORAL_DENOISING
735
      if (cpi->oxcf.noise_sensitivity > 0) {
736
        vp9_denoiser_update_frame_stats(mbmi, sse_y, this_mode, ctx);
737
      }
738 739
#else
      (void)ctx;
Tim Kopp's avatar
Tim Kopp committed
740 741
#endif

742 743
      if (this_rdc.rdcost < best_rdc.rdcost || x->skip) {
        best_rdc = this_rdc;
744
        best_mode = this_mode;
745
        best_pred_filter = mbmi->interp_filter;
746
        best_tx_size = mbmi->tx_size;
747
        best_ref_frame = ref_frame;
748
        skip_txfm = x->skip_txfm[0];
749

750
        if (reuse_inter_pred) {
751
          free_pred_buffer(best_pred);
752 753 754
          best_pred = this_mode_pred;
        }
      } else {
755
        if (reuse_inter_pred)
756
          free_pred_buffer(this_mode_pred);
757
      }
758 759 760

      if (x->skip)
        break;
761
    }
762 763 764

    // Check that a prediction mode has been selected.
    assert(best_rdc.rdcost < INT64_MAX);
765 766
  }

767 768
  // If best prediction is not in dst buf, then copy the prediction block from
  // temp buf to dst buf.
769
  if (best_pred != NULL && reuse_inter_pred &&
770
      best_pred->data != orig_dst.buf) {
771
    pd->dst = orig_dst;
772 773
#if CONFIG_VP9_HIGHBITDEPTH
    if (cm->use_highbitdepth) {
774 775
      vp9_highbd_convolve_copy(best_pred->data, bw, pd->dst.buf, pd->dst.stride,
                               NULL, 0, NULL, 0, bw, bh, xd->bd);
776 777 778 779 780
    } else {
      vp9_convolve_copy(best_pred->data, bw, pd->dst.buf, pd->dst.stride,
                        NULL, 0, NULL, 0, bw, bh);
    }
#else
781 782
    vp9_convolve_copy(best_pred->data, bw, pd->dst.buf, pd->dst.stride, NULL, 0,
                      NULL, 0, bw, bh);
783
#endif  // CONFIG_VP9_HIGHBITDEPTH
784
  }
Tim Kopp's avatar
Tim Kopp committed
785

786
  mbmi->mode          = best_mode;
787
  mbmi->interp_filter = best_pred_filter;
788 789 790
  mbmi->tx_size       = best_tx_size;
  mbmi->ref_frame[0]  = best_ref_frame;
  mbmi->mv[0].as_int  = frame_mv[best_mode][best_ref_frame].as_int;
hkuang's avatar
hkuang committed
791
  xd->mi[0].src_mi->bmi[0].as_mv[0].as_int = mbmi->mv[0].as_int;
792
  x->skip_txfm[0] = skip_txfm;
793

794 795
  // Perform intra prediction search, if the best SAD is above a certain
  // threshold.
796
  if (!x->skip && best_rdc.rdcost > inter_mode_thresh &&
797
      bsize <= cpi->sf.max_intra_bsize) {
798
    PREDICTION_MODE this_mode;
799 800 801 802
    struct estimate_block_intra_args args = { cpi, x, DC_PRED, 0, 0 };
    const TX_SIZE intra_tx_size =
        MIN(max_txsize_lookup[bsize],
            tx_mode_to_biggest_tx_size[cpi->common.tx_mode]);
803

804
    if (reuse_inter_pred) {
805 806 807
      pd->dst.buf = tmp[0].data;
      pd->dst.stride = bw;
    }
808

809
    for (this_mode = DC_PRED; this_mode <= DC_PRED; ++this_mode) {
810 811 812 813 814 815 816 817
      const TX_SIZE saved_tx_size = mbmi->tx_size;
      args.mode = this_mode;
      args.rate = 0;
      args.dist = 0;
      mbmi->tx_size = intra_tx_size;
      vp9_foreach_transformed_block_in_plane(xd, bsize, 0,
                                             estimate_block_intra, &args);
      mbmi->tx_size = saved_tx_size;
818 819 820 821 822 823 824 825 826
      this_rdc.rate = args.rate;
      this_rdc.dist = args.dist;
      this_rdc.rate += cpi->mbmode_cost[this_mode];
      this_rdc.rate += intra_cost_penalty;
      this_rdc.rdcost = RDCOST(x->rdmult, x->rddiv,
                               this_rdc.rate, this_rdc.dist);

      if (this_rdc.rdcost + intra_mode_cost < best_rdc.rdcost) {
        best_rdc = this_rdc;
827
        mbmi->mode = this_mode;
828
        mbmi->tx_size = intra_tx_size;
829 830
        mbmi->ref_frame[0] = INTRA_FRAME;
        mbmi->uv_mode = this_mode;
831
        mbmi->mv[0].as_int = INVALID_MV;
832
      } else {
833
        x->skip_txfm[0] = skip_txfm;
834 835
      }
    }
836
    if (reuse_inter_pred)
837
      pd->dst = orig_dst;
838
  }
839

840
  if (is_inter_block(mbmi))
841
    vp9_update_rd_thresh_fact(tile_data->thresh_freq_fact,
842 843
                            cpi->sf.adaptive_rd_thresh, bsize,
                            mode_idx[best_ref_frame][INTER_OFFSET(mbmi->mode)]);
844
  else
845 846
    vp9_update_rd_thresh_fact(tile_data->thresh_freq_fact,
                              cpi->sf.adaptive_rd_thresh, bsize,
847
                              mode_idx[INTRA_FRAME][mbmi->mode]);
848

849
  *rd_cost = best_rdc;
850
}