vp9_encoder.c 143 KB
Newer Older
John Koleszar's avatar
John Koleszar committed
1
/*
2
 * Copyright (c) 2010 The WebM project authors. All Rights Reserved.
John Koleszar's avatar
John Koleszar committed
3
 *
4
 *  Use of this source code is governed by a BSD-style license
5 6
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
7
 *  in the file PATENTS.  All contributing project authors may
8
 *  be found in the AUTHORS file in the root of the source tree.
John Koleszar's avatar
John Koleszar committed
9 10
 */

Jim Bankoski's avatar
Jim Bankoski committed
11 12 13 14 15 16
#include <math.h>
#include <stdio.h>
#include <limits.h>

#include "./vpx_config.h"
#include "./vpx_scale_rtcd.h"
17 18
#include "vpx/internal/vpx_psnr.h"
#include "vpx_ports/vpx_timer.h"
John Koleszar's avatar
John Koleszar committed
19

Jim Bankoski's avatar
Jim Bankoski committed
20
#include "vp9/common/vp9_alloccommon.h"
21
#include "vp9/common/vp9_filter.h"
22
#include "vp9/common/vp9_idct.h"
Jim Bankoski's avatar
Jim Bankoski committed
23 24 25
#if CONFIG_VP9_POSTPROC
#include "vp9/common/vp9_postproc.h"
#endif
26
#include "vp9/common/vp9_reconinter.h"
27
#include "vp9/common/vp9_reconintra.h"
28
#include "vp9/common/vp9_systemdependent.h"
Jim Bankoski's avatar
Jim Bankoski committed
29
#include "vp9/common/vp9_tile_common.h"
30

31
#include "vp9/encoder/vp9_aq_complexity.h"
Marco Paniconi's avatar
Marco Paniconi committed
32 33
#include "vp9/encoder/vp9_aq_cyclicrefresh.h"
#include "vp9/encoder/vp9_aq_variance.h"
34
#include "vp9/encoder/vp9_bitstream.h"
35
#include "vp9/encoder/vp9_context_tree.h"
36
#include "vp9/encoder/vp9_encodeframe.h"
37
#include "vp9/encoder/vp9_encodemv.h"
38
#include "vp9/encoder/vp9_ethread.h"
39
#include "vp9/encoder/vp9_firstpass.h"
Jim Bankoski's avatar
Jim Bankoski committed
40
#include "vp9/encoder/vp9_mbgraph.h"
Dmitry Kovalev's avatar
Dmitry Kovalev committed
41
#include "vp9/encoder/vp9_encoder.h"
Jim Bankoski's avatar
Jim Bankoski committed
42
#include "vp9/encoder/vp9_picklpf.h"
43
#include "vp9/encoder/vp9_ratectrl.h"
44
#include "vp9/encoder/vp9_rd.h"
Jim Bankoski's avatar
Jim Bankoski committed
45
#include "vp9/encoder/vp9_segmentation.h"
Yaowu Xu's avatar
Yaowu Xu committed
46
#include "vp9/encoder/vp9_speed_features.h"
Dmitry Kovalev's avatar
Dmitry Kovalev committed
47 48 49
#if CONFIG_INTERNAL_STATS
#include "vp9/encoder/vp9_ssim.h"
#endif
50
#include "vp9/encoder/vp9_temporal_filter.h"
51
#include "vp9/encoder/vp9_resize.h"
52
#include "vp9/encoder/vp9_svc_layercontext.h"
Marco's avatar
Marco committed
53
#include "vp9/encoder/vp9_skin_detection.h"
Paul Wilkins's avatar
Paul Wilkins committed
54

55 56
#define AM_SEGMENT_ID_INACTIVE 7
#define AM_SEGMENT_ID_ACTIVE 0
57

58 59
#define SHARP_FILTER_QTHRESH 0          /* Q threshold for 8-tap sharp filter */

Jim Bankoski's avatar
Jim Bankoski committed
60 61 62 63 64 65
#define ALTREF_HIGH_PRECISION_MV 1      // Whether to use high precision mv
                                         //  for altref computation.
#define HIGH_PRECISION_MV_QTHRESH 200   // Q threshold for high precision
                                         // mv. Choose a very high value for
                                         // now so that HIGH_PRECISION is always
                                         // chosen.
John Koleszar's avatar
John Koleszar committed
66
// #define OUTPUT_YUV_REC
John Koleszar's avatar
John Koleszar committed
67

68
#ifdef OUTPUT_YUV_DENOISED
69
FILE *yuv_denoised_file = NULL;
70
#endif
Marco's avatar
Marco committed
71 72 73
#ifdef OUTPUT_YUV_SKINMAP
FILE *yuv_skinmap_file = NULL;
#endif
74 75 76
#ifdef OUTPUT_YUV_REC
FILE *yuv_rec_file;
#endif
John Koleszar's avatar
John Koleszar committed
77 78 79

#if 0
FILE *framepsnr;
Yaowu Xu's avatar
Yaowu Xu committed
80
FILE *kf_list;
John Koleszar's avatar
John Koleszar committed
81 82 83
FILE *keyfile;
#endif

84
static INLINE void Scale2Ratio(VPX_SCALING mode, int *hr, int *hs) {
85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109
  switch (mode) {
    case NORMAL:
      *hr = 1;
      *hs = 1;
      break;
    case FOURFIVE:
      *hr = 4;
      *hs = 5;
      break;
    case THREEFIVE:
      *hr = 3;
      *hs = 5;
    break;
    case ONETWO:
      *hr = 1;
      *hs = 2;
    break;
    default:
      *hr = 1;
      *hs = 1;
       assert(0);
      break;
  }
}

110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174
// Mark all inactive blocks as active. Other segmentation features may be set
// so memset cannot be used, instead only inactive blocks should be reset.
void vp9_suppress_active_map(VP9_COMP *cpi) {
  unsigned char *const seg_map = cpi->segmentation_map;
  int i;
  if (cpi->active_map.enabled || cpi->active_map.update)
    for (i = 0; i < cpi->common.mi_rows * cpi->common.mi_cols; ++i)
      if (seg_map[i] == AM_SEGMENT_ID_INACTIVE)
        seg_map[i] = AM_SEGMENT_ID_ACTIVE;
}

void vp9_apply_active_map(VP9_COMP *cpi) {
  struct segmentation *const seg = &cpi->common.seg;
  unsigned char *const seg_map = cpi->segmentation_map;
  const unsigned char *const active_map = cpi->active_map.map;
  int i;

  assert(AM_SEGMENT_ID_ACTIVE == CR_SEGMENT_ID_BASE);

  if (cpi->active_map.update) {
    if (cpi->active_map.enabled) {
      for (i = 0; i < cpi->common.mi_rows * cpi->common.mi_cols; ++i)
        if (seg_map[i] == AM_SEGMENT_ID_ACTIVE) seg_map[i] = active_map[i];
      vp9_enable_segmentation(seg);
      vp9_enable_segfeature(seg, AM_SEGMENT_ID_INACTIVE, SEG_LVL_SKIP);
    } else {
      vp9_disable_segfeature(seg, AM_SEGMENT_ID_INACTIVE, SEG_LVL_SKIP);
      if (seg->enabled) {
        seg->update_data = 1;
        seg->update_map = 1;
      }
    }
    cpi->active_map.update = 0;
  }
}

int vp9_set_active_map(VP9_COMP* cpi,
                       unsigned char* new_map_16x16,
                       int rows,
                       int cols) {
  if (rows == cpi->common.mb_rows && cols == cpi->common.mb_cols) {
    unsigned char *const active_map_8x8 = cpi->active_map.map;
    const int mi_rows = cpi->common.mi_rows;
    const int mi_cols = cpi->common.mi_cols;
    cpi->active_map.update = 1;
    if (new_map_16x16) {
      int r, c;
      for (r = 0; r < mi_rows; ++r) {
        for (c = 0; c < mi_cols; ++c) {
          active_map_8x8[r * mi_cols + c] =
              new_map_16x16[(r >> 1) * cols + (c >> 1)]
                  ? AM_SEGMENT_ID_ACTIVE
                  : AM_SEGMENT_ID_INACTIVE;
        }
      }
      cpi->active_map.enabled = 1;
    } else {
      cpi->active_map.enabled = 0;
    }
    return 0;
  } else {
    return -1;
  }
}

175
void vp9_set_high_precision_mv(VP9_COMP *cpi, int allow_high_precision_mv) {
176
  MACROBLOCK *const mb = &cpi->td.mb;
Deb Mukherjee's avatar
Deb Mukherjee committed
177
  cpi->common.allow_high_precision_mv = allow_high_precision_mv;
178
  if (cpi->common.allow_high_precision_mv) {
179 180 181 182 183 184 185
    mb->mvcost = mb->nmvcost_hp;
    mb->mvsadcost = mb->nmvsadcost_hp;
  } else {
    mb->mvcost = mb->nmvcost;
    mb->mvsadcost = mb->nmvsadcost;
  }
}
Paul Wilkins's avatar
Paul Wilkins committed
186

187 188 189 190 191 192 193
static void setup_frame(VP9_COMP *cpi) {
  VP9_COMMON *const cm = &cpi->common;
  // Set up entropy context depending on frame type. The decoder mandates
  // the use of the default context, index 0, for keyframes and inter
  // frames where the error_resilient_mode or intra_only flag is set. For
  // other inter-frames the encoder currently uses only two contexts;
  // context 1 for ALTREF frames and context 0 for the others.
194 195 196 197 198 199 200
  if (frame_is_intra_only(cm) || cm->error_resilient_mode) {
    vp9_setup_past_independence(cm);
  } else {
    if (!cpi->use_svc)
      cm->frame_context_idx = cpi->refresh_alt_ref_frame;
  }

201
  if (cm->frame_type == KEY_FRAME) {
202
    if (!is_two_pass_svc(cpi))
203
      cpi->refresh_golden_frame = 1;
204
    cpi->refresh_alt_ref_frame = 1;
205
    vp9_zero(cpi->interp_filter_selected);
206
  } else {
207
    *cm->fc = cm->frame_contexts[cm->frame_context_idx];
208
    vp9_zero(cpi->interp_filter_selected[0]);
209 210 211
  }
}

212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252
static void vp9_enc_setup_mi(VP9_COMMON *cm) {
  int i;
  cm->mi = cm->mip + cm->mi_stride + 1;
  vpx_memset(cm->mip, 0, cm->mi_stride * (cm->mi_rows + 1) * sizeof(*cm->mip));
  cm->prev_mi = cm->prev_mip + cm->mi_stride + 1;
  // Clear top border row
  vpx_memset(cm->prev_mip, 0, sizeof(*cm->prev_mip) * cm->mi_stride);
  // Clear left border column
  for (i = 1; i < cm->mi_rows + 1; ++i)
    vpx_memset(&cm->prev_mip[i * cm->mi_stride], 0, sizeof(*cm->prev_mip));
}

static int vp9_enc_alloc_mi(VP9_COMMON *cm, int mi_size) {
  cm->mip = vpx_calloc(mi_size, sizeof(*cm->mip));
  if (!cm->mip)
    return 1;
  cm->prev_mip = vpx_calloc(mi_size, sizeof(*cm->prev_mip));
  if (!cm->prev_mip)
    return 1;
  cm->mi_alloc_size = mi_size;
  return 0;
}

static void vp9_enc_free_mi(VP9_COMMON *cm) {
  vpx_free(cm->mip);
  cm->mip = NULL;
  vpx_free(cm->prev_mip);
  cm->prev_mip = NULL;
}

static void vp9_swap_mi_and_prev_mi(VP9_COMMON *cm) {
  // Current mip will be the prev_mip for the next frame.
  MODE_INFO *temp = cm->prev_mip;
  cm->prev_mip = cm->mip;
  cm->mip = temp;

  // Update the upper left visible macroblock ptrs.
  cm->mi = cm->mip + cm->mi_stride + 1;
  cm->prev_mi = cm->prev_mip + cm->mi_stride + 1;
}

253
void vp9_initialize_enc(void) {
254
  static volatile int init_done = 0;
John Koleszar's avatar
John Koleszar committed
255 256

  if (!init_done) {
257 258
    vp9_rtcd();
    vp9_init_intra_predictors();
259
    vp9_init_me_luts();
260
    vp9_rc_init_minq_luts();
261
    vp9_entropy_mv_init();
262
    vp9_temporal_filter_init();
John Koleszar's avatar
John Koleszar committed
263 264
    init_done = 1;
  }
John Koleszar's avatar
John Koleszar committed
265 266
}

267
static void dealloc_compressor_data(VP9_COMP *cpi) {
Dmitry Kovalev's avatar
Dmitry Kovalev committed
268
  VP9_COMMON *const cm = &cpi->common;
269
  int i;
Dmitry Kovalev's avatar
Dmitry Kovalev committed
270

271 272
  vpx_free(cpi->tile_data);
  cpi->tile_data = NULL;
273

274
  // Delete sementation map
John Koleszar's avatar
John Koleszar committed
275
  vpx_free(cpi->segmentation_map);
Dmitry Kovalev's avatar
Dmitry Kovalev committed
276
  cpi->segmentation_map = NULL;
John Koleszar's avatar
John Koleszar committed
277
  vpx_free(cpi->coding_context.last_frame_seg_map_copy);
Dmitry Kovalev's avatar
Dmitry Kovalev committed
278
  cpi->coding_context.last_frame_seg_map_copy = NULL;
John Koleszar's avatar
John Koleszar committed
279

Jingning Han's avatar
Jingning Han committed
280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299
  vpx_free(cpi->nmvcosts[0]);
  vpx_free(cpi->nmvcosts[1]);
  cpi->nmvcosts[0] = NULL;
  cpi->nmvcosts[1] = NULL;

  vpx_free(cpi->nmvcosts_hp[0]);
  vpx_free(cpi->nmvcosts_hp[1]);
  cpi->nmvcosts_hp[0] = NULL;
  cpi->nmvcosts_hp[1] = NULL;

  vpx_free(cpi->nmvsadcosts[0]);
  vpx_free(cpi->nmvsadcosts[1]);
  cpi->nmvsadcosts[0] = NULL;
  cpi->nmvsadcosts[1] = NULL;

  vpx_free(cpi->nmvsadcosts_hp[0]);
  vpx_free(cpi->nmvsadcosts_hp[1]);
  cpi->nmvsadcosts_hp[0] = NULL;
  cpi->nmvsadcosts_hp[1] = NULL;

300 301 302
  vp9_cyclic_refresh_free(cpi->cyclic_refresh);
  cpi->cyclic_refresh = NULL;

303 304 305
  vpx_free(cpi->active_map.map);
  cpi->active_map.map = NULL;

Adrian Grange's avatar
Adrian Grange committed
306
  vp9_free_ref_frame_buffers(cm);
307
  vp9_free_context_buffers(cm);
John Koleszar's avatar
John Koleszar committed
308

309 310
  vp9_free_frame_buffer(&cpi->last_frame_uf);
  vp9_free_frame_buffer(&cpi->scaled_source);
311
  vp9_free_frame_buffer(&cpi->scaled_last_source);
312
  vp9_free_frame_buffer(&cpi->alt_ref_buffer);
313
  vp9_lookahead_destroy(cpi->lookahead);
John Koleszar's avatar
John Koleszar committed
314

315 316
  vpx_free(cpi->tile_tok[0][0]);
  cpi->tile_tok[0][0] = 0;
John Koleszar's avatar
John Koleszar committed
317

318
  vp9_free_pc_tree(&cpi->td);
319

320 321 322 323 324 325
  for (i = 0; i < cpi->svc.number_spatial_layers; ++i) {
    LAYER_CONTEXT *const lc = &cpi->svc.layer_context[i];
    vpx_free(lc->rc_twopass_stats_in.buf);
    lc->rc_twopass_stats_in.buf = NULL;
    lc->rc_twopass_stats_in.sz = 0;
  }
326 327 328 329 330

  if (cpi->source_diff_var != NULL) {
    vpx_free(cpi->source_diff_var);
    cpi->source_diff_var = NULL;
  }
331

332 333 334 335 336
  for (i = 0; i < MAX_LAG_BUFFERS; ++i) {
    vp9_free_frame_buffer(&cpi->svc.scaled_frames[i]);
  }
  vpx_memset(&cpi->svc.scaled_frames[0], 0,
             MAX_LAG_BUFFERS * sizeof(cpi->svc.scaled_frames[0]));
337 338 339

  vp9_free_frame_buffer(&cpi->svc.empty_frame.img);
  vpx_memset(&cpi->svc.empty_frame, 0, sizeof(cpi->svc.empty_frame));
John Koleszar's avatar
John Koleszar committed
340 341
}

342 343 344 345 346 347 348 349
static void save_coding_context(VP9_COMP *cpi) {
  CODING_CONTEXT *const cc = &cpi->coding_context;
  VP9_COMMON *cm = &cpi->common;

  // Stores a snapshot of key state variables which can subsequently be
  // restored with a call to vp9_restore_coding_context. These functions are
  // intended for use in a re-code loop in vp9_compress_frame where the
  // quantizer value is adjusted between loop iterations.
350
  vp9_copy(cc->nmvjointcost,  cpi->td.mb.nmvjointcost);
Jingning Han's avatar
Jingning Han committed
351 352 353 354 355 356 357 358 359

  vpx_memcpy(cc->nmvcosts[0], cpi->nmvcosts[0],
             MV_VALS * sizeof(*cpi->nmvcosts[0]));
  vpx_memcpy(cc->nmvcosts[1], cpi->nmvcosts[1],
             MV_VALS * sizeof(*cpi->nmvcosts[1]));
  vpx_memcpy(cc->nmvcosts_hp[0], cpi->nmvcosts_hp[0],
             MV_VALS * sizeof(*cpi->nmvcosts_hp[0]));
  vpx_memcpy(cc->nmvcosts_hp[1], cpi->nmvcosts_hp[1],
             MV_VALS * sizeof(*cpi->nmvcosts_hp[1]));
360 361 362 363 364 365 366 367 368

  vp9_copy(cc->segment_pred_probs, cm->seg.pred_probs);

  vpx_memcpy(cpi->coding_context.last_frame_seg_map_copy,
             cm->last_frame_seg_map, (cm->mi_rows * cm->mi_cols));

  vp9_copy(cc->last_ref_lf_deltas, cm->lf.last_ref_deltas);
  vp9_copy(cc->last_mode_lf_deltas, cm->lf.last_mode_deltas);

369
  cc->fc = *cm->fc;
370 371 372 373 374 375 376 377
}

static void restore_coding_context(VP9_COMP *cpi) {
  CODING_CONTEXT *const cc = &cpi->coding_context;
  VP9_COMMON *cm = &cpi->common;

  // Restore key state variables to the snapshot state stored in the
  // previous call to vp9_save_coding_context.
378
  vp9_copy(cpi->td.mb.nmvjointcost, cc->nmvjointcost);
Jingning Han's avatar
Jingning Han committed
379 380 381 382 383 384 385 386 387

  vpx_memcpy(cpi->nmvcosts[0], cc->nmvcosts[0],
             MV_VALS * sizeof(*cc->nmvcosts[0]));
  vpx_memcpy(cpi->nmvcosts[1], cc->nmvcosts[1],
             MV_VALS * sizeof(*cc->nmvcosts[1]));
  vpx_memcpy(cpi->nmvcosts_hp[0], cc->nmvcosts_hp[0],
             MV_VALS * sizeof(*cc->nmvcosts_hp[0]));
  vpx_memcpy(cpi->nmvcosts_hp[1], cc->nmvcosts_hp[1],
             MV_VALS * sizeof(*cc->nmvcosts_hp[1]));
388 389 390 391 392 393 394 395 396 397

  vp9_copy(cm->seg.pred_probs, cc->segment_pred_probs);

  vpx_memcpy(cm->last_frame_seg_map,
             cpi->coding_context.last_frame_seg_map_copy,
             (cm->mi_rows * cm->mi_cols));

  vp9_copy(cm->lf.last_ref_deltas, cc->last_ref_lf_deltas);
  vp9_copy(cm->lf.last_mode_deltas, cc->last_mode_lf_deltas);

398
  *cm->fc = cc->fc;
399 400
}

401
static void configure_static_seg_features(VP9_COMP *cpi) {
Dmitry Kovalev's avatar
Dmitry Kovalev committed
402
  VP9_COMMON *const cm = &cpi->common;
403
  const RATE_CONTROL *const rc = &cpi->rc;
Dmitry Kovalev's avatar
Dmitry Kovalev committed
404
  struct segmentation *const seg = &cm->seg;
405

406
  int high_q = (int)(rc->avg_q > 48.0);
John Koleszar's avatar
John Koleszar committed
407
  int qi_delta;
408

John Koleszar's avatar
John Koleszar committed
409 410 411
  // Disable and clear down for KF
  if (cm->frame_type == KEY_FRAME) {
    // Clear down the global segmentation map
412
    vpx_memset(cpi->segmentation_map, 0, cm->mi_rows * cm->mi_cols);
413 414
    seg->update_map = 0;
    seg->update_data = 0;
John Koleszar's avatar
John Koleszar committed
415
    cpi->static_mb_pct = 0;
416

John Koleszar's avatar
John Koleszar committed
417
    // Disable segmentation
418
    vp9_disable_segmentation(seg);
Paul Wilkins's avatar
Paul Wilkins committed
419

John Koleszar's avatar
John Koleszar committed
420
    // Clear down the segment features.
421
    vp9_clearall_segfeatures(seg);
422 423
  } else if (cpi->refresh_alt_ref_frame) {
    // If this is an alt ref frame
John Koleszar's avatar
John Koleszar committed
424
    // Clear down the global segmentation map
425
    vpx_memset(cpi->segmentation_map, 0, cm->mi_rows * cm->mi_cols);
426 427
    seg->update_map = 0;
    seg->update_data = 0;
John Koleszar's avatar
John Koleszar committed
428
    cpi->static_mb_pct = 0;
Paul Wilkins's avatar
Paul Wilkins committed
429

John Koleszar's avatar
John Koleszar committed
430
    // Disable segmentation and individual segment features by default
431
    vp9_disable_segmentation(seg);
432
    vp9_clearall_segfeatures(seg);
Paul Wilkins's avatar
Paul Wilkins committed
433

John Koleszar's avatar
John Koleszar committed
434 435
    // Scan frames from current to arf frame.
    // This function re-enables segmentation if appropriate.
436
    vp9_update_mbgraph_stats(cpi);
Paul Wilkins's avatar
Paul Wilkins committed
437

John Koleszar's avatar
John Koleszar committed
438 439
    // If segmentation was enabled set those features needed for the
    // arf itself.
440 441 442
    if (seg->enabled) {
      seg->update_map = 1;
      seg->update_data = 1;
Paul Wilkins's avatar
Paul Wilkins committed
443

444 445
      qi_delta = vp9_compute_qdelta(rc, rc->avg_q, rc->avg_q * 0.875,
                                    cm->bit_depth);
446
      vp9_set_segdata(seg, 1, SEG_LVL_ALT_Q, qi_delta - 2);
447
      vp9_set_segdata(seg, 1, SEG_LVL_ALT_LF, -2);
Paul Wilkins's avatar
Paul Wilkins committed
448

449 450
      vp9_enable_segfeature(seg, 1, SEG_LVL_ALT_Q);
      vp9_enable_segfeature(seg, 1, SEG_LVL_ALT_LF);
451

John Koleszar's avatar
John Koleszar committed
452
      // Where relevant assume segment data is delta data
453
      seg->abs_delta = SEGMENT_DELTADATA;
454
    }
455
  } else if (seg->enabled) {
456 457
    // All other frames if segmentation has been enabled

John Koleszar's avatar
John Koleszar committed
458
    // First normal frame in a valid gf or alt ref group
459
    if (rc->frames_since_golden == 0) {
Paul Wilkins's avatar
Paul Wilkins committed
460
      // Set up segment features for normal frames in an arf group
461
      if (rc->source_alt_ref_active) {
462 463 464
        seg->update_map = 0;
        seg->update_data = 1;
        seg->abs_delta = SEGMENT_DELTADATA;
Paul Wilkins's avatar
Paul Wilkins committed
465

466 467
        qi_delta = vp9_compute_qdelta(rc, rc->avg_q, rc->avg_q * 1.125,
                                      cm->bit_depth);
468
        vp9_set_segdata(seg, 1, SEG_LVL_ALT_Q, qi_delta + 2);
469
        vp9_enable_segfeature(seg, 1, SEG_LVL_ALT_Q);
Paul Wilkins's avatar
Paul Wilkins committed
470

471 472
        vp9_set_segdata(seg, 1, SEG_LVL_ALT_LF, -2);
        vp9_enable_segfeature(seg, 1, SEG_LVL_ALT_LF);
Paul Wilkins's avatar
Paul Wilkins committed
473

John Koleszar's avatar
John Koleszar committed
474 475
        // Segment coding disabled for compred testing
        if (high_q || (cpi->static_mb_pct == 100)) {
476 477 478
          vp9_set_segdata(seg, 1, SEG_LVL_REF_FRAME, ALTREF_FRAME);
          vp9_enable_segfeature(seg, 1, SEG_LVL_REF_FRAME);
          vp9_enable_segfeature(seg, 1, SEG_LVL_SKIP);
John Koleszar's avatar
John Koleszar committed
479
        }
480 481 482 483
      } else {
        // Disable segmentation and clear down features if alt ref
        // is not active for this group

484
        vp9_disable_segmentation(seg);
Paul Wilkins's avatar
Paul Wilkins committed
485

486
        vpx_memset(cpi->segmentation_map, 0, cm->mi_rows * cm->mi_cols);
Paul Wilkins's avatar
Paul Wilkins committed
487

488 489
        seg->update_map = 0;
        seg->update_data = 0;
Paul Wilkins's avatar
Paul Wilkins committed
490

491
        vp9_clearall_segfeatures(seg);
John Koleszar's avatar
John Koleszar committed
492
      }
493
    } else if (rc->is_src_frame_alt_ref) {
494 495 496
      // Special case where we are coding over the top of a previous
      // alt ref frame.
      // Segment coding disabled for compred testing
John Koleszar's avatar
John Koleszar committed
497

Paul Wilkins's avatar
Paul Wilkins committed
498
      // Enable ref frame features for segment 0 as well
499 500
      vp9_enable_segfeature(seg, 0, SEG_LVL_REF_FRAME);
      vp9_enable_segfeature(seg, 1, SEG_LVL_REF_FRAME);
John Koleszar's avatar
John Koleszar committed
501

Paul Wilkins's avatar
Paul Wilkins committed
502
      // All mbs should use ALTREF_FRAME
503 504 505 506
      vp9_clear_segdata(seg, 0, SEG_LVL_REF_FRAME);
      vp9_set_segdata(seg, 0, SEG_LVL_REF_FRAME, ALTREF_FRAME);
      vp9_clear_segdata(seg, 1, SEG_LVL_REF_FRAME);
      vp9_set_segdata(seg, 1, SEG_LVL_REF_FRAME, ALTREF_FRAME);
John Koleszar's avatar
John Koleszar committed
507

Paul Wilkins's avatar
Paul Wilkins committed
508
      // Skip all MBs if high Q (0,0 mv and skip coeffs)
John Koleszar's avatar
John Koleszar committed
509
      if (high_q) {
510 511
        vp9_enable_segfeature(seg, 0, SEG_LVL_SKIP);
        vp9_enable_segfeature(seg, 1, SEG_LVL_SKIP);
John Koleszar's avatar
John Koleszar committed
512
      }
Adrian Grange's avatar
Adrian Grange committed
513
      // Enable data update
514
      seg->update_data = 1;
515 516 517
    } else {
      // All other frames.

John Koleszar's avatar
John Koleszar committed
518
      // No updates.. leave things as they are.
519 520
      seg->update_map = 0;
      seg->update_data = 0;
John Koleszar's avatar
John Koleszar committed
521 522
    }
  }
523 524
}

525
static void update_reference_segmentation_map(VP9_COMP *cpi) {
Ronald S. Bultje's avatar
Ronald S. Bultje committed
526
  VP9_COMMON *const cm = &cpi->common;
hkuang's avatar
hkuang committed
527
  MODE_INFO *mi_8x8_ptr = cm->mi;
Dmitry Kovalev's avatar
Dmitry Kovalev committed
528
  uint8_t *cache_ptr = cm->last_frame_seg_map;
Ronald S. Bultje's avatar
Ronald S. Bultje committed
529 530
  int row, col;

531
  for (row = 0; row < cm->mi_rows; row++) {
hkuang's avatar
hkuang committed
532
    MODE_INFO *mi_8x8 = mi_8x8_ptr;
Dmitry Kovalev's avatar
Dmitry Kovalev committed
533
    uint8_t *cache = cache_ptr;
534
    for (col = 0; col < cm->mi_cols; col++, mi_8x8++, cache++)
hkuang's avatar
hkuang committed
535
      cache[0] = mi_8x8[0].src_mi->mbmi.segment_id;
536
    mi_8x8_ptr += cm->mi_stride;
537
    cache_ptr += cm->mi_cols;
Ronald S. Bultje's avatar
Ronald S. Bultje committed
538 539 540
  }
}

541
static void alloc_raw_frame_buffers(VP9_COMP *cpi) {
542
  VP9_COMMON *cm = &cpi->common;
543
  const VP9EncoderConfig *oxcf = &cpi->oxcf;
544

545 546 547
  if (!cpi->lookahead)
    cpi->lookahead = vp9_lookahead_init(oxcf->width, oxcf->height,
                                        cm->subsampling_x, cm->subsampling_y,
548 549 550
#if CONFIG_VP9_HIGHBITDEPTH
                                      cm->use_highbitdepth,
#endif
Dmitry Kovalev's avatar
Dmitry Kovalev committed
551
                                      oxcf->lag_in_frames);
John Koleszar's avatar
John Koleszar committed
552
  if (!cpi->lookahead)
553
    vpx_internal_error(&cm->error, VPX_CODEC_MEM_ERROR,
John Koleszar's avatar
John Koleszar committed
554
                       "Failed to allocate lag buffers");
John Koleszar's avatar
John Koleszar committed
555

556
  // TODO(agrange) Check if ARF is enabled and skip allocation if not.
557
  if (vp9_realloc_frame_buffer(&cpi->alt_ref_buffer,
Dmitry Kovalev's avatar
Dmitry Kovalev committed
558
                               oxcf->width, oxcf->height,
559
                               cm->subsampling_x, cm->subsampling_y,
560 561 562
#if CONFIG_VP9_HIGHBITDEPTH
                               cm->use_highbitdepth,
#endif
563 564
                               VP9_ENC_BORDER_IN_PIXELS, cm->byte_alignment,
                               NULL, NULL, NULL))
565
    vpx_internal_error(&cm->error, VPX_CODEC_MEM_ERROR,
John Koleszar's avatar
John Koleszar committed
566
                       "Failed to allocate altref buffer");
John Koleszar's avatar
John Koleszar committed
567
}
568

569
static void alloc_util_frame_buffers(VP9_COMP *cpi) {
570
  VP9_COMMON *const cm = &cpi->common;
571 572 573
  if (vp9_realloc_frame_buffer(&cpi->last_frame_uf,
                               cm->width, cm->height,
                               cm->subsampling_x, cm->subsampling_y,
574 575 576
#if CONFIG_VP9_HIGHBITDEPTH
                               cm->use_highbitdepth,
#endif
577 578
                               VP9_ENC_BORDER_IN_PIXELS, cm->byte_alignment,
                               NULL, NULL, NULL))
579
    vpx_internal_error(&cm->error, VPX_CODEC_MEM_ERROR,
580
                       "Failed to allocate last frame buffer");
581

582 583 584
  if (vp9_realloc_frame_buffer(&cpi->scaled_source,
                               cm->width, cm->height,
                               cm->subsampling_x, cm->subsampling_y,
585 586 587
#if CONFIG_VP9_HIGHBITDEPTH
                               cm->use_highbitdepth,
#endif
588 589
                               VP9_ENC_BORDER_IN_PIXELS, cm->byte_alignment,
                               NULL, NULL, NULL))
590
    vpx_internal_error(&cm->error, VPX_CODEC_MEM_ERROR,
591
                       "Failed to allocate scaled source buffer");
592

593 594 595
  if (vp9_realloc_frame_buffer(&cpi->scaled_last_source,
                               cm->width, cm->height,
                               cm->subsampling_x, cm->subsampling_y,
596 597 598
#if CONFIG_VP9_HIGHBITDEPTH
                               cm->use_highbitdepth,
#endif
599 600
                               VP9_ENC_BORDER_IN_PIXELS, cm->byte_alignment,
                               NULL, NULL, NULL))
601
    vpx_internal_error(&cm->error, VPX_CODEC_MEM_ERROR,
602 603
                       "Failed to allocate scaled last source buffer");
}
604

605 606
void vp9_alloc_compressor_data(VP9_COMP *cpi) {
  VP9_COMMON *cm = &cpi->common;
607

608 609
  vp9_alloc_context_buffers(cm, cm->width, cm->height);

610
  vpx_free(cpi->tile_tok[0][0]);
611 612 613

  {
    unsigned int tokens = get_token_alloc(cm->mb_rows, cm->mb_cols);
614 615
    CHECK_MEM_ERROR(cm, cpi->tile_tok[0][0],
        vpx_calloc(tokens, sizeof(*cpi->tile_tok[0][0])));
616
  }
617

618
  vp9_setup_pc_tree(&cpi->common, &cpi->td);
619 620 621 622
}

static void update_frame_size(VP9_COMP *cpi) {
  VP9_COMMON *const cm = &cpi->common;
623
  MACROBLOCKD *const xd = &cpi->td.mb.e_mbd;
Adrian Grange's avatar
Adrian Grange committed
624 625 626

  vp9_set_mb_mi(cm, cm->width, cm->height);
  vp9_init_context_buffers(cm);
627
  init_macroblockd(cm, xd);
628

629
  if (is_two_pass_svc(cpi)) {
630 631 632
    if (vp9_realloc_frame_buffer(&cpi->alt_ref_buffer,
                                 cm->width, cm->height,
                                 cm->subsampling_x, cm->subsampling_y,
633 634 635
#if CONFIG_VP9_HIGHBITDEPTH
                                 cm->use_highbitdepth,
#endif
636 637
                                 VP9_ENC_BORDER_IN_PIXELS, cm->byte_alignment,
                                 NULL, NULL, NULL))
638 639 640
      vpx_internal_error(&cm->error, VPX_CODEC_MEM_ERROR,
                         "Failed to reallocate alt_ref_buffer");
  }
641 642
}

James Zern's avatar
James Zern committed
643
void vp9_new_framerate(VP9_COMP *cpi, double framerate) {
644
  cpi->framerate = framerate < 0.1 ? 30 : framerate;
645
  vp9_rc_update_framerate(cpi);
John Koleszar's avatar
John Koleszar committed
646 647
}

Ronald S. Bultje's avatar
Ronald S. Bultje committed
648 649
static void set_tile_limits(VP9_COMP *cpi) {
  VP9_COMMON *const cm = &cpi->common;
650

Dmitry Kovalev's avatar
Dmitry Kovalev committed
651 652
  int min_log2_tile_cols, max_log2_tile_cols;
  vp9_get_tile_n_bits(cm->mi_cols, &min_log2_tile_cols, &max_log2_tile_cols);
653

Dmitry Kovalev's avatar
Dmitry Kovalev committed
654 655 656
  cm->log2_tile_cols = clamp(cpi->oxcf.tile_columns,
                             min_log2_tile_cols, max_log2_tile_cols);
  cm->log2_tile_rows = cpi->oxcf.tile_rows;
657
}
658

659 660 661 662 663 664
static void init_buffer_indices(VP9_COMP *cpi) {
  cpi->lst_fb_idx = 0;
  cpi->gld_fb_idx = 1;
  cpi->alt_fb_idx = 2;
}

665
static void init_config(struct VP9_COMP *cpi, VP9EncoderConfig *oxcf) {
666
  VP9_COMMON *const cm = &cpi->common;
John Koleszar's avatar
John Koleszar committed
667

John Koleszar's avatar
John Koleszar committed
668
  cpi->oxcf = *oxcf;
669
  cpi->framerate = oxcf->init_framerate;
John Koleszar's avatar
John Koleszar committed
670

671 672
  cm->profile = oxcf->profile;
  cm->bit_depth = oxcf->bit_depth;
673 674 675
#if CONFIG_VP9_HIGHBITDEPTH
  cm->use_highbitdepth = oxcf->use_highbitdepth;
#endif
676
  cm->color_space = oxcf->color_space;
John Koleszar's avatar
John Koleszar committed
677

678 679
  cm->width = oxcf->width;
  cm->height = oxcf->height;
680
  vp9_alloc_compressor_data(cpi);
681

682 683 684
  // Single thread case: use counts in common.
  cpi->td.counts = &cm->counts;

685 686 687 688 689
  // Spatial scalability.
  cpi->svc.number_spatial_layers = oxcf->ss_number_layers;
  // Temporal scalability.
  cpi->svc.number_temporal_layers = oxcf->ts_number_layers;

690
  if ((cpi->svc.number_temporal_layers > 1 && cpi->oxcf.rc_mode == VPX_CBR) ||
691 692
      ((cpi->svc.number_temporal_layers > 1 ||
        cpi->svc.number_spatial_layers > 1) &&
693
       cpi->oxcf.pass != 1)) {
694
    vp9_init_layer_context(cpi);
695 696
  }

John Koleszar's avatar
John Koleszar committed
697
  // change includes all joint functionality
Dmitry Kovalev's avatar
Dmitry Kovalev committed
698
  vp9_change_config(cpi, oxcf);
John Koleszar's avatar
John Koleszar committed
699

John Koleszar's avatar
John Koleszar committed
700
  cpi->static_mb_pct = 0;
701
  cpi->ref_frame_flags = 0;
702

703
  init_buffer_indices(cpi);
John Koleszar's avatar
John Koleszar committed
704 705
}

Dmitry Kovalev's avatar
Dmitry Kovalev committed
706 707 708 709 710 711 712 713 714 715 716 717 718 719
static void set_rc_buffer_sizes(RATE_CONTROL *rc,
                                const VP9EncoderConfig *oxcf) {
  const int64_t bandwidth = oxcf->target_bandwidth;
  const int64_t starting = oxcf->starting_buffer_level_ms;
  const int64_t optimal = oxcf->optimal_buffer_level_ms;
  const int64_t maximum = oxcf->maximum_buffer_size_ms;

  rc->starting_buffer_level = starting * bandwidth / 1000;
  rc->optimal_buffer_level = (optimal == 0) ? bandwidth / 8
                                            : optimal * bandwidth / 1000;
  rc->maximum_buffer_size = (maximum == 0) ? bandwidth / 8
                                           : maximum * bandwidth / 1000;
}

720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861
#if CONFIG_VP9_HIGHBITDEPTH
#define HIGHBD_BFP(BT, SDF, SDAF, VF, SVF, SVAF, SDX3F, SDX8F, SDX4DF) \
    cpi->fn_ptr[BT].sdf = SDF; \
    cpi->fn_ptr[BT].sdaf = SDAF; \
    cpi->fn_ptr[BT].vf = VF; \
    cpi->fn_ptr[BT].svf = SVF; \
    cpi->fn_ptr[BT].svaf = SVAF; \
    cpi->fn_ptr[BT].sdx3f = SDX3F; \
    cpi->fn_ptr[BT].sdx8f = SDX8F; \
    cpi->fn_ptr[BT].sdx4df = SDX4DF;

#define MAKE_BFP_SAD_WRAPPER(fnname) \
static unsigned int fnname##_bits8(const uint8_t *src_ptr, \
                                   int source_stride, \
                                   const uint8_t *ref_ptr, \
                                   int ref_stride) {  \
  return fnname(src_ptr, source_stride, ref_ptr, ref_stride); \
} \
static unsigned int fnname##_bits10(const uint8_t *src_ptr, \
                                    int source_stride, \
                                    const uint8_t *ref_ptr, \
                                    int ref_stride) {  \
  return fnname(src_ptr, source_stride, ref_ptr, ref_stride) >> 2; \
} \
static unsigned int fnname##_bits12(const uint8_t *src_ptr, \
                                    int source_stride, \
                                    const uint8_t *ref_ptr, \
                                    int ref_stride) {  \
  return fnname(src_ptr, source_stride, ref_ptr, ref_stride) >> 4; \
}

#define MAKE_BFP_SADAVG_WRAPPER(fnname) static unsigned int \
fnname##_bits8(const uint8_t *src_ptr, \
               int source_stride, \
               const uint8_t *ref_ptr, \
               int ref_stride, \
               const uint8_t *second_pred) {  \
  return fnname(src_ptr, source_stride, ref_ptr, ref_stride, second_pred); \
} \
static unsigned int fnname##_bits10(const uint8_t *src_ptr, \
                                    int source_stride, \
                                    const uint8_t *ref_ptr, \
                                    int ref_stride, \
                                    const uint8_t *second_pred) {  \
  return fnname(src_ptr, source_stride, ref_ptr, ref_stride, \
                second_pred) >> 2; \
} \
static unsigned int fnname##_bits12(const uint8_t *src_ptr, \
                                    int source_stride, \
                                    const uint8_t *ref_ptr, \
                                    int ref_stride, \
                                    const uint8_t *second_pred) {  \
  return fnname(src_ptr, source_stride, ref_ptr, ref_stride, \
                second_pred) >> 4; \
}

#define MAKE_BFP_SAD3_WRAPPER(fnname) \
static void fnname##_bits8(const uint8_t *src_ptr, \
                           int source_stride, \
                           const uint8_t *ref_ptr, \
                           int  ref_stride, \
                           unsigned int *sad_array) {  \
  fnname(src_ptr, source_stride, ref_ptr, ref_stride, sad_array); \
} \
static void fnname##_bits10(const uint8_t *src_ptr, \
                            int source_stride, \
                            const uint8_t *ref_ptr, \
                            int  ref_stride, \
                            unsigned int *sad_array) {  \
  int i; \
  fnname(src_ptr, source_stride, ref_ptr, ref_stride, sad_array); \
  for (i = 0; i < 3; i++) \
    sad_array[i] >>= 2; \
} \
static void fnname##_bits12(const uint8_t *src_ptr, \
                            int source_stride, \
                            const uint8_t *ref_ptr, \
                            int  ref_stride, \
                            unsigned int *sad_array) {  \
  int i; \
  fnname(src_ptr, source_stride, ref_ptr, ref_stride, sad_array); \
  for (i = 0; i < 3; i++) \
    sad_array[i] >>= 4; \
}

#define MAKE_BFP_SAD8_WRAPPER(fnname) \
static void fnname##_bits8(const uint8_t *src_ptr, \
                           int source_stride, \
                           const uint8_t *ref_ptr, \
                           int  ref_stride, \
                           unsigned int *sad_array) {  \
  fnname(src_ptr, source_stride, ref_ptr, ref_stride, sad_array); \
} \
static void fnname##_bits10(const uint8_t *src_ptr, \
                            int source_stride, \
                            const uint8_t *ref_ptr, \
                            int  ref_stride, \
                            unsigned int *sad_array) {  \
  int i; \
  fnname(src_ptr, source_stride, ref_ptr, ref_stride, sad_array); \
  for (i = 0; i < 8; i++) \
    sad_array[i] >>= 2; \
} \
static void fnname##_bits12(const uint8_t *src_ptr, \
                            int source_stride, \
                            const uint8_t *ref_ptr, \
                            int  ref_stride, \
                            unsigned int *sad_array) {  \
  int i; \
  fnname(src_ptr, source_stride, ref_ptr, ref_stride, sad_array); \
  for (i = 0; i < 8; i++) \
    sad_array[i] >>= 4; \
}
#define MAKE_BFP_SAD4D_WRAPPER(fnname) \
static void fnname##_bits8(const uint8_t *src_ptr, \
                           int source_stride, \
                           const uint8_t* const ref_ptr[], \
                           int  ref_stride, \
                           unsigned int *sad_array) {  \
  fnname(src_ptr, source_stride, ref_ptr, ref_stride, sad_array); \
} \
static void fnname##_bits10(const uint8_t *src_ptr, \
                            int source_stride, \
                            const uint8_t* const ref_ptr[], \
                            int  ref_stride, \
                            unsigned int *sad_array) {  \
  int i; \
  fnname(src_ptr, source_stride, ref_ptr, ref_stride, sad_array); \
  for (i = 0; i < 4; i++) \
  sad_array[i] >>= 2; \
} \
static void fnname##_bits12(const uint8_t *src_ptr, \
                            int source_stride, \
                            const uint8_t* const ref_ptr[], \
                            int  ref_stride, \
                            unsigned int *sad_array) {  \
  int i; \
  fnname(src_ptr, source_stride, ref_ptr, ref_stride, sad_array); \
  for (i = 0; i < 4; i++) \
  sad_array[i] >>= 4; \
}

862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918
MAKE_BFP_SAD_WRAPPER(vp9_highbd_sad32x16)
MAKE_BFP_SADAVG_WRAPPER(vp9_highbd_sad32x16_avg)
MAKE_BFP_SAD4D_WRAPPER(vp9_highbd_sad32x16x4d)
MAKE_BFP_SAD_WRAPPER(vp9_highbd_sad16x32)
MAKE_BFP_SADAVG_WRAPPER(vp9_highbd_sad16x32_avg)
MAKE_BFP_SAD4D_WRAPPER(vp9_highbd_sad16x32x4d)
MAKE_BFP_SAD_WRAPPER(vp9_highbd_sad64x32)
MAKE_BFP_SADAVG_WRAPPER(vp9_highbd_sad64x32_avg)
MAKE_BFP_SAD4D_WRAPPER(vp9_highbd_sad64x32x4d)
MAKE_BFP_SAD_WRAPPER(vp9_highbd_sad32x64)
MAKE_BFP_SADAVG_WRAPPER(vp9_highbd_sad32x64_avg)
MAKE_BFP_SAD4D_WRAPPER(vp9_highbd_sad32x64x4d)
MAKE_BFP_SAD_WRAPPER(vp9_highbd_sad32x32)
MAKE_BFP_SADAVG_WRAPPER(vp9_highbd_sad32x32_avg)
MAKE_BFP_SAD3_WRAPPER(vp9_highbd_sad32x32x3)
MAKE_BFP_SAD8_WRAPPER(vp9_highbd_sad32x32x8)
MAKE_BFP_SAD4D_WRAPPER(vp9_highbd_sad32x32x4d)
MAKE_BFP_SAD_WRAPPER(vp9_highbd_sad64x64)
MAKE_BFP_SADAVG_WRAPPER(vp9_highbd_sad64x64_avg)
MAKE_BFP_SAD3_WRAPPER(vp9_highbd_sad64x64x3)
MAKE_BFP_SAD8_WRAPPER(vp9_highbd_sad64x64x8)
MAKE_BFP_SAD4D_WRAPPER(vp9_highbd_sad64x64x4d)
MAKE_BFP_SAD_WRAPPER(vp9_highbd_sad16x16)
MAKE_BFP_SADAVG_WRAPPER(vp9_highbd_sad16x16_avg)
MAKE_BFP_SAD3_WRAPPER(vp9_highbd_sad16x16x3)
MAKE_BFP_SAD8_WRAPPER(vp9_highbd_sad16x16x8)
MAKE_BFP_SAD4D_WRAPPER(vp9_highbd_sad16x16x4d)
MAKE_BFP_SAD_WRAPPER(vp9_highbd_sad16x8)
MAKE_BFP_SADAVG_WRAPPER(vp9_highbd_sad16x8_avg)
MAKE_BFP_SAD3_WRAPPER(vp9_highbd_sad16x8x3)
MAKE_BFP_SAD8_WRAPPER(vp9_highbd_sad16x8x8)
MAKE_BFP_SAD4D_WRAPPER(vp9_highbd_sad16x8x4d)
MAKE_BFP_SAD_WRAPPER(vp9_highbd_sad8x16)
MAKE_BFP_SADAVG_WRAPPER(vp9_highbd_sad8x16_avg)
MAKE_BFP_SAD3_WRAPPER(vp9_highbd_sad8x16x3)
MAKE_BFP_SAD8_WRAPPER(vp9_highbd_sad8x16x8)
MAKE_BFP_SAD4D_WRAPPER(vp9_highbd_sad8x16x4d)
MAKE_BFP_SAD_WRAPPER(vp9_highbd_sad8x8)
MAKE_BFP_SADAVG_WRAPPER(vp9_highbd_sad8x8_avg)
MAKE_BFP_SAD3_WRAPPER(vp9_highbd_sad8x8x3)
MAKE_BFP_SAD8_WRAPPER(vp9_highbd_sad8x8x8)
MAKE_BFP_SAD4D_WRAPPER(vp9_highbd_sad8x8x4d)
MAKE_BFP_SAD_WRAPPER(vp9_highbd_sad8x4)
MAKE_BFP_SADAVG_WRAPPER(vp9_highbd_sad8x4_avg)
MAKE_BFP_SAD8_WRAPPER(vp9_highbd_sad8x4x8)
MAKE_BFP_SAD4D_WRAPPER(vp9_highbd_sad8x4x4d)
MAKE_BFP_SAD_WRAPPER(vp9_highbd_sad4x8)
MAKE_BFP_SADAVG_WRAPPER(vp9_highbd_sad4x8_avg)
MAKE_BFP_SAD8_WRAPPER(vp9_highbd_sad4x8x8)
MAKE_BFP_SAD4D_WRAPPER(vp9_highbd_sad4x8x4d)
MAKE_BFP_SAD_WRAPPER(vp9_highbd_sad4x4)
MAKE_BFP_SADAVG_WRAPPER(vp9_highbd_sad4x4_avg)
MAKE_BFP_SAD3_WRAPPER(vp9_highbd_sad4x4x3)
MAKE_BFP_SAD8_WRAPPER(vp9_highbd_sad4x4x8)
MAKE_BFP_SAD4D_WRAPPER(vp9_highbd_sad4x4x4d)

static void  highbd_set_var_fns(VP9_COMP *const cpi) {
919 920 921 922 923
  VP9_COMMON *const cm = &cpi->common;
  if (cm->use_highbitdepth) {
    switch (cm->bit_depth) {
      case VPX_BITS_8:
        HIGHBD_BFP(BLOCK_32X16,
924 925 926 927 928
                   vp9_highbd_sad32x16_bits8,
                   vp9_highbd_sad32x16_avg_bits8,
                   vp9_highbd_variance32x16,
                   vp9_highbd_sub_pixel_variance32x16,
                   vp9_highbd_sub_pixel_avg_variance32x16,
929 930
                   NULL,
                   NULL,
931
                   vp9_highbd_sad32x16x4d_bits8)
932 933

        HIGHBD_BFP(BLOCK_16X32,
934 935 936 937 938
                   vp9_highbd_sad16x32_bits8,
                   vp9_highbd_sad16x32_avg_bits8,
                   vp9_highbd_variance16x32,
                   vp9_highbd_sub_pixel_variance16x32,
                   vp9_highbd_sub_pixel_avg_variance16x32,
939 940
                   NULL,
                   NULL,
941
                   vp9_highbd_sad16x32x4d_bits8)
942 943

        HIGHBD_BFP(BLOCK_64X32,
944 945 946 947 948
                   vp9_highbd_sad64x32_bits8,
                   vp9_highbd_sad64x32_avg_bits8,
                   vp9_highbd_variance64x32,
                   vp9_highbd_sub_pixel_variance64x32,
                   vp9_highbd_sub_pixel_avg_variance64x32,
949 950
                   NULL,
                   NULL,
951
                   vp9_highbd_sad64x32x4d_bits8)
952 953

        HIGHBD_BFP(BLOCK_32X64,
954 955 956