vp9_reconintra.c 35.9 KB
Newer Older
John Koleszar's avatar
John Koleszar committed
1
/*
2
 *  Copyright (c) 2010 The WebM project authors. All Rights Reserved.
John Koleszar's avatar
John Koleszar committed
3
 *
4
 *  Use of this source code is governed by a BSD-style license
5 6
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
7
 *  in the file PATENTS.  All contributing project authors may
8
 *  be found in the AUTHORS file in the root of the source tree.
John Koleszar's avatar
John Koleszar committed
9 10
 */

11
#include "./vpx_config.h"
12
#include "./vp9_rtcd.h"
13 14

#include "vpx_mem/vpx_mem.h"
15
#include "vpx_ports/mem.h"
16
#include "vpx_ports/vpx_once.h"
17

18
#include "vp9/common/vp9_reconintra.h"
19
#include "vp9/common/vp9_onyxc_int.h"
John Koleszar's avatar
John Koleszar committed
20

21
const TX_TYPE intra_mode_to_tx_type_lookup[INTRA_MODES] = {
22 23 24 25 26 27 28 29 30 31
  DCT_DCT,    // DC
  ADST_DCT,   // V
  DCT_ADST,   // H
  DCT_DCT,    // D45
  ADST_ADST,  // D135
  ADST_DCT,   // D117
  DCT_ADST,   // D153
  DCT_ADST,   // D207
  ADST_DCT,   // D63
  ADST_ADST,  // TM
32 33
};

34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52
enum {
  NEED_LEFT = 1 << 1,
  NEED_ABOVE = 1 << 2,
  NEED_ABOVERIGHT = 1 << 3,
};

static const uint8_t extend_modes[INTRA_MODES] = {
  NEED_ABOVE | NEED_LEFT,       // DC
  NEED_ABOVE,                   // V
  NEED_LEFT,                    // H
  NEED_ABOVERIGHT,              // D45
  NEED_LEFT | NEED_ABOVE,       // D135
  NEED_LEFT | NEED_ABOVE,       // D117
  NEED_LEFT | NEED_ABOVE,       // D153
  NEED_LEFT,                    // D207
  NEED_ABOVERIGHT,              // D63
  NEED_LEFT | NEED_ABOVE,       // TM
};

53 54 55
// This serves as a wrapper function, so that all the prediction functions
// can be unified and accessed as a pointer array. Note that the boundary
// above and left are not necessarily used all the time.
56
#define intra_pred_sized(type, size) \
57 58 59 60 61 62 63
  void vp9_##type##_predictor_##size##x##size##_c(uint8_t *dst, \
                                                  ptrdiff_t stride, \
                                                  const uint8_t *above, \
                                                  const uint8_t *left) { \
    type##_predictor(dst, stride, size, above, left); \
  }

64
#if CONFIG_VP9_HIGHBITDEPTH
65 66
#define intra_pred_highbd_sized(type, size) \
  void vp9_highbd_##type##_predictor_##size##x##size##_c( \
67 68
      uint16_t *dst, ptrdiff_t stride, const uint16_t *above, \
      const uint16_t *left, int bd) { \
69
    highbd_##type##_predictor(dst, stride, size, above, left, bd); \
70 71 72 73 74 75 76
  }

#define intra_pred_allsizes(type) \
  intra_pred_sized(type, 4) \
  intra_pred_sized(type, 8) \
  intra_pred_sized(type, 16) \
  intra_pred_sized(type, 32) \
77 78 79 80
  intra_pred_highbd_sized(type, 4) \
  intra_pred_highbd_sized(type, 8) \
  intra_pred_highbd_sized(type, 16) \
  intra_pred_highbd_sized(type, 32)
81

82 83 84 85 86 87 88 89 90
#define intra_pred_no_4x4(type) \
  intra_pred_sized(type, 8) \
  intra_pred_sized(type, 16) \
  intra_pred_sized(type, 32) \
  intra_pred_highbd_sized(type, 4) \
  intra_pred_highbd_sized(type, 8) \
  intra_pred_highbd_sized(type, 16) \
  intra_pred_highbd_sized(type, 32)

91 92
#else

93 94 95 96 97
#define intra_pred_allsizes(type) \
  intra_pred_sized(type, 4) \
  intra_pred_sized(type, 8) \
  intra_pred_sized(type, 16) \
  intra_pred_sized(type, 32)
98 99 100 101 102

#define intra_pred_no_4x4(type) \
  intra_pred_sized(type, 8) \
  intra_pred_sized(type, 16) \
  intra_pred_sized(type, 32)
103 104 105
#endif  // CONFIG_VP9_HIGHBITDEPTH

#if CONFIG_VP9_HIGHBITDEPTH
106 107 108
static INLINE void highbd_d207_predictor(uint16_t *dst, ptrdiff_t stride,
                                         int bs, const uint16_t *above,
                                         const uint16_t *left, int bd) {
Deb Mukherjee's avatar
Deb Mukherjee committed
109
  int r, c;
110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139
  (void) above;
  (void) bd;

  // First column.
  for (r = 0; r < bs - 1; ++r) {
    dst[r * stride] = ROUND_POWER_OF_TWO(left[r] + left[r + 1], 1);
  }
  dst[(bs - 1) * stride] = left[bs - 1];
  dst++;

  // Second column.
  for (r = 0; r < bs - 2; ++r) {
    dst[r * stride] = ROUND_POWER_OF_TWO(left[r] + left[r + 1] * 2 +
                                         left[r + 2], 2);
  }
  dst[(bs - 2) * stride] = ROUND_POWER_OF_TWO(left[bs - 2] +
                                              left[bs - 1] * 3, 2);
  dst[(bs - 1) * stride] = left[bs - 1];
  dst++;

  // Rest of last row.
  for (c = 0; c < bs - 2; ++c)
    dst[(bs - 1) * stride + c] = left[bs - 1];

  for (r = bs - 2; r >= 0; --r) {
    for (c = 0; c < bs - 2; ++c)
      dst[r * stride + c] = dst[(r + 1) * stride + c - 2];
  }
}

140 141 142
static INLINE void highbd_d63_predictor(uint16_t *dst, ptrdiff_t stride,
                                        int bs, const uint16_t *above,
                                        const uint16_t *left, int bd) {
Deb Mukherjee's avatar
Deb Mukherjee committed
143
  int r, c;
144 145 146 147 148 149 150 151 152 153 154 155 156 157
  (void) left;
  (void) bd;
  for (r = 0; r < bs; ++r) {
    for (c = 0; c < bs; ++c) {
      dst[c] = r & 1 ? ROUND_POWER_OF_TWO(above[r/2 + c] +
                                          above[r/2 + c + 1] * 2 +
                                          above[r/2 + c + 2], 2)
                     : ROUND_POWER_OF_TWO(above[r/2 + c] +
                                          above[r/2 + c + 1], 1);
    }
    dst += stride;
  }
}

158 159 160
static INLINE void highbd_d45_predictor(uint16_t *dst, ptrdiff_t stride, int bs,
                                        const uint16_t *above,
                                        const uint16_t *left, int bd) {
Deb Mukherjee's avatar
Deb Mukherjee committed
161
  int r, c;
162 163 164 165 166 167 168 169 170 171 172 173 174
  (void) left;
  (void) bd;
  for (r = 0; r < bs; ++r) {
    for (c = 0; c < bs; ++c) {
      dst[c] = r + c + 2 < bs * 2 ?  ROUND_POWER_OF_TWO(above[r + c] +
                                                        above[r + c + 1] * 2 +
                                                        above[r + c + 2], 2)
                                  : above[bs * 2 - 1];
    }
    dst += stride;
  }
}

175 176 177
static INLINE void highbd_d117_predictor(uint16_t *dst, ptrdiff_t stride,
                                         int bs, const uint16_t *above,
                                         const uint16_t *left, int bd) {
178
  int r, c;
Deb Mukherjee's avatar
Deb Mukherjee committed
179
  (void) bd;
180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205

  // first row
  for (c = 0; c < bs; c++)
    dst[c] = ROUND_POWER_OF_TWO(above[c - 1] + above[c], 1);
  dst += stride;

  // second row
  dst[0] = ROUND_POWER_OF_TWO(left[0] + above[-1] * 2 + above[0], 2);
  for (c = 1; c < bs; c++)
    dst[c] = ROUND_POWER_OF_TWO(above[c - 2] + above[c - 1] * 2 + above[c], 2);
  dst += stride;

  // the rest of first col
  dst[0] = ROUND_POWER_OF_TWO(above[-1] + left[0] * 2 + left[1], 2);
  for (r = 3; r < bs; ++r)
    dst[(r - 2) * stride] = ROUND_POWER_OF_TWO(left[r - 3] + left[r - 2] * 2 +
                                               left[r - 1], 2);

  // the rest of the block
  for (r = 2; r < bs; ++r) {
    for (c = 1; c < bs; c++)
      dst[c] = dst[-2 * stride + c - 1];
    dst += stride;
  }
}

206 207 208
static INLINE void highbd_d135_predictor(uint16_t *dst, ptrdiff_t stride,
                                         int bs, const uint16_t *above,
                                         const uint16_t *left, int bd) {
209
  int r, c;
Deb Mukherjee's avatar
Deb Mukherjee committed
210
  (void) bd;
211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227
  dst[0] = ROUND_POWER_OF_TWO(left[0] + above[-1] * 2 + above[0], 2);
  for (c = 1; c < bs; c++)
    dst[c] = ROUND_POWER_OF_TWO(above[c - 2] + above[c - 1] * 2 + above[c], 2);

  dst[stride] = ROUND_POWER_OF_TWO(above[-1] + left[0] * 2 + left[1], 2);
  for (r = 2; r < bs; ++r)
    dst[r * stride] = ROUND_POWER_OF_TWO(left[r - 2] + left[r - 1] * 2 +
                                         left[r], 2);

  dst += stride;
  for (r = 1; r < bs; ++r) {
    for (c = 1; c < bs; c++)
      dst[c] = dst[-stride + c - 1];
    dst += stride;
  }
}

228 229 230
static INLINE void highbd_d153_predictor(uint16_t *dst, ptrdiff_t stride,
                                         int bs, const uint16_t *above,
                                         const uint16_t *left, int bd) {
231
  int r, c;
Deb Mukherjee's avatar
Deb Mukherjee committed
232
  (void) bd;
233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255
  dst[0] = ROUND_POWER_OF_TWO(above[-1] + left[0], 1);
  for (r = 1; r < bs; r++)
    dst[r * stride] = ROUND_POWER_OF_TWO(left[r - 1] + left[r], 1);
  dst++;

  dst[0] = ROUND_POWER_OF_TWO(left[0] + above[-1] * 2 + above[0], 2);
  dst[stride] = ROUND_POWER_OF_TWO(above[-1] + left[0] * 2 + left[1], 2);
  for (r = 2; r < bs; r++)
    dst[r * stride] = ROUND_POWER_OF_TWO(left[r - 2] + left[r - 1] * 2 +
                                         left[r], 2);
  dst++;

  for (c = 0; c < bs - 2; c++)
    dst[c] = ROUND_POWER_OF_TWO(above[c - 1] + above[c] * 2 + above[c + 1], 2);
  dst += stride;

  for (r = 1; r < bs; ++r) {
    for (c = 0; c < bs - 2; c++)
      dst[c] = dst[-stride + c - 2];
    dst += stride;
  }
}

256 257 258
static INLINE void highbd_v_predictor(uint16_t *dst, ptrdiff_t stride,
                                      int bs, const uint16_t *above,
                                      const uint16_t *left, int bd) {
Deb Mukherjee's avatar
Deb Mukherjee committed
259
  int r;
260 261 262
  (void) left;
  (void) bd;
  for (r = 0; r < bs; r++) {
James Zern's avatar
James Zern committed
263
    memcpy(dst, above, bs * sizeof(uint16_t));
264 265 266 267
    dst += stride;
  }
}

268 269 270
static INLINE void highbd_h_predictor(uint16_t *dst, ptrdiff_t stride,
                                      int bs, const uint16_t *above,
                                      const uint16_t *left, int bd) {
Deb Mukherjee's avatar
Deb Mukherjee committed
271
  int r;
272 273 274 275 276 277 278 279
  (void) above;
  (void) bd;
  for (r = 0; r < bs; r++) {
    vpx_memset16(dst, left[r], bs);
    dst += stride;
  }
}

280 281 282
static INLINE void highbd_tm_predictor(uint16_t *dst, ptrdiff_t stride,
                                       int bs, const uint16_t *above,
                                       const uint16_t *left, int bd) {
283 284
  int r, c;
  int ytop_left = above[-1];
Deb Mukherjee's avatar
Deb Mukherjee committed
285
  (void) bd;
286 287 288

  for (r = 0; r < bs; r++) {
    for (c = 0; c < bs; c++)
289
      dst[c] = clip_pixel_highbd(left[r] + above[c] - ytop_left, bd);
290 291 292 293
    dst += stride;
  }
}

294 295 296
static INLINE void highbd_dc_128_predictor(uint16_t *dst, ptrdiff_t stride,
                                           int bs, const uint16_t *above,
                                           const uint16_t *left, int bd) {
Deb Mukherjee's avatar
Deb Mukherjee committed
297
  int r;
298 299 300 301 302 303 304 305 306
  (void) above;
  (void) left;

  for (r = 0; r < bs; r++) {
    vpx_memset16(dst, 128 << (bd - 8), bs);
    dst += stride;
  }
}

307 308 309
static INLINE void highbd_dc_left_predictor(uint16_t *dst, ptrdiff_t stride,
                                            int bs, const uint16_t *above,
                                            const uint16_t *left, int bd) {
Deb Mukherjee's avatar
Deb Mukherjee committed
310
  int i, r, expected_dc, sum = 0;
311 312 313 314 315 316 317 318 319 320 321 322 323
  (void) above;
  (void) bd;

  for (i = 0; i < bs; i++)
    sum += left[i];
  expected_dc = (sum + (bs >> 1)) / bs;

  for (r = 0; r < bs; r++) {
    vpx_memset16(dst, expected_dc, bs);
    dst += stride;
  }
}

324 325 326
static INLINE void highbd_dc_top_predictor(uint16_t *dst, ptrdiff_t stride,
                                           int bs, const uint16_t *above,
                                           const uint16_t *left, int bd) {
Deb Mukherjee's avatar
Deb Mukherjee committed
327
  int i, r, expected_dc, sum = 0;
328 329 330 331 332 333 334 335 336 337 338 339 340
  (void) left;
  (void) bd;

  for (i = 0; i < bs; i++)
    sum += above[i];
  expected_dc = (sum + (bs >> 1)) / bs;

  for (r = 0; r < bs; r++) {
    vpx_memset16(dst, expected_dc, bs);
    dst += stride;
  }
}

341 342 343
static INLINE void highbd_dc_predictor(uint16_t *dst, ptrdiff_t stride,
                                       int bs, const uint16_t *above,
                                       const uint16_t *left, int bd) {
344 345
  int i, r, expected_dc, sum = 0;
  const int count = 2 * bs;
Deb Mukherjee's avatar
Deb Mukherjee committed
346
  (void) bd;
347 348 349 350 351 352 353 354 355 356 357 358 359 360

  for (i = 0; i < bs; i++) {
    sum += above[i];
    sum += left[i];
  }

  expected_dc = (sum + (count >> 1)) / count;

  for (r = 0; r < bs; r++) {
    vpx_memset16(dst, expected_dc, bs);
    dst += stride;
  }
}
#endif  // CONFIG_VP9_HIGHBITDEPTH
361

362 363
#define DST(x, y) dst[(x) + (y) * stride]
#define AVG3(a, b, c) (((a) + 2 * (b) + (c) + 2) >> 2)
364
#define AVG2(a, b) (((a) + (b) + 1) >> 1)
365

366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382
void vp9_d207_predictor_4x4_c(uint8_t *dst, ptrdiff_t stride,
                              const uint8_t *above, const uint8_t *left) {
  const int I = left[0];
  const int J = left[1];
  const int K = left[2];
  const int L = left[3];
  (void)above;
  DST(0, 0) =             AVG2(I, J);
  DST(2, 0) = DST(0, 1) = AVG2(J, K);
  DST(2, 1) = DST(0, 2) = AVG2(K, L);
  DST(1, 0) =             AVG3(I, J, K);
  DST(3, 0) = DST(1, 1) = AVG3(J, K, L);
  DST(3, 1) = DST(1, 2) = AVG3(K, L, L);
  DST(3, 2) = DST(2, 2) =
      DST(0, 3) = DST(1, 3) = DST(2, 3) = DST(3, 3) = L;
}

383 384
static INLINE void d207_predictor(uint8_t *dst, ptrdiff_t stride, int bs,
                                  const uint8_t *above, const uint8_t *left) {
385
  int r, c;
386
  (void) above;
387
  // first column
Dmitry Kovalev's avatar
Dmitry Kovalev committed
388
  for (r = 0; r < bs - 1; ++r)
389 390 391
    dst[r * stride] = ROUND_POWER_OF_TWO(left[r] + left[r + 1], 1);
  dst[(bs - 1) * stride] = left[bs - 1];
  dst++;
Dmitry Kovalev's avatar
Dmitry Kovalev committed
392

393
  // second column
Dmitry Kovalev's avatar
Dmitry Kovalev committed
394
  for (r = 0; r < bs - 2; ++r)
395 396 397 398 399 400
    dst[r * stride] = ROUND_POWER_OF_TWO(left[r] + left[r + 1] * 2 +
                                         left[r + 2], 2);
  dst[(bs - 2) * stride] = ROUND_POWER_OF_TWO(left[bs - 2] +
                                              left[bs - 1] * 3, 2);
  dst[(bs - 1) * stride] = left[bs - 1];
  dst++;
401

402
  // rest of last row
Dmitry Kovalev's avatar
Dmitry Kovalev committed
403
  for (c = 0; c < bs - 2; ++c)
404
    dst[(bs - 1) * stride + c] = left[bs - 1];
405

Dmitry Kovalev's avatar
Dmitry Kovalev committed
406 407
  for (r = bs - 2; r >= 0; --r)
    for (c = 0; c < bs - 2; ++c)
408
      dst[r * stride + c] = dst[(r + 1) * stride + c - 2];
409
}
410
intra_pred_no_4x4(d207)
411

412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435
void vp9_d63_predictor_4x4_c(uint8_t *dst, ptrdiff_t stride,
                             const uint8_t *above, const uint8_t *left) {
  const int A = above[0];
  const int B = above[1];
  const int C = above[2];
  const int D = above[3];
  const int E = above[4];
  const int F = above[5];
  const int G = above[6];
  const int H = above[7];
  (void)left;
  DST(0, 0) =             AVG2(A, B);
  DST(1, 0) = DST(0, 2) = AVG2(B, C);
  DST(2, 0) = DST(1, 2) = AVG2(C, D);
  DST(3, 0) = DST(2, 2) = AVG2(D, E);

  DST(0, 1) =             AVG3(A, B, C);
  DST(1, 1) = DST(0, 3) = AVG3(B, C, D);
  DST(2, 1) = DST(1, 3) = AVG3(C, D, E);
  DST(3, 1) = DST(2, 3) = AVG3(D, E, F);
              DST(3, 2) = AVG3(E, F, G);
              DST(3, 3) = AVG3(F, G, H);
}

436 437
static INLINE void d63_predictor(uint8_t *dst, ptrdiff_t stride, int bs,
                                 const uint8_t *above, const uint8_t *left) {
438
  int r, c;
439
  (void) left;
440
  for (r = 0; r < bs; ++r) {
Dmitry Kovalev's avatar
Dmitry Kovalev committed
441
    for (c = 0; c < bs; ++c)
442 443 444 445 446 447
      dst[c] = r & 1 ? ROUND_POWER_OF_TWO(above[r/2 + c] +
                                          above[r/2 + c + 1] * 2 +
                                          above[r/2 + c + 2], 2)
                     : ROUND_POWER_OF_TWO(above[r/2 + c] +
                                          above[r/2 + c + 1], 1);
    dst += stride;
John Koleszar's avatar
John Koleszar committed
448
  }
449
}
450
intra_pred_no_4x4(d63)
451

452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472
void vp9_d45_predictor_4x4_c(uint8_t *dst, ptrdiff_t stride,
                             const uint8_t *above, const uint8_t *left) {
  const int A = above[0];
  const int B = above[1];
  const int C = above[2];
  const int D = above[3];
  const int E = above[4];
  const int F = above[5];
  const int G = above[6];
  const int H = above[7];
  (void)stride;
  (void)left;
  DST(0, 0)                                     = AVG3(A, B, C);
  DST(1, 0) = DST(0, 1)                         = AVG3(B, C, D);
  DST(2, 0) = DST(1, 1) = DST(0, 2)             = AVG3(C, D, E);
  DST(3, 0) = DST(2, 1) = DST(1, 2) = DST(0, 3) = AVG3(D, E, F);
              DST(3, 1) = DST(2, 2) = DST(1, 3) = AVG3(E, F, G);
                          DST(3, 2) = DST(2, 3) = AVG3(F, G, H);
                                      DST(3, 3) = AVG3(G, H, H);
}

473 474
static INLINE void d45_predictor(uint8_t *dst, ptrdiff_t stride, int bs,
                                 const uint8_t *above, const uint8_t *left) {
475 476 477 478 479 480 481 482 483 484 485
  const uint8_t above_right = above[bs - 1];
  int x, size;
  uint8_t avg[31];  // TODO(jzern): this could be block size specific
  (void)left;

  for (x = 0; x < bs - 1; ++x) {
    avg[x] = AVG3(above[x], above[x + 1], above[x + 2]);
  }
  for (x = 0, size = bs - 1; x < bs; ++x, --size) {
    memcpy(dst, avg + x, size);
    memset(dst + size, above_right, x + 1);
486
    dst += stride;
John Koleszar's avatar
John Koleszar committed
487
  }
488
}
489
intra_pred_no_4x4(d45)
490

491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513
void vp9_d117_predictor_4x4_c(uint8_t *dst, ptrdiff_t stride,
                              const uint8_t *above, const uint8_t *left) {
  const int I = left[0];
  const int J = left[1];
  const int K = left[2];
  const int X = above[-1];
  const int A = above[0];
  const int B = above[1];
  const int C = above[2];
  const int D = above[3];
  DST(0, 0) = DST(1, 2) = AVG2(X, A);
  DST(1, 0) = DST(2, 2) = AVG2(A, B);
  DST(2, 0) = DST(3, 2) = AVG2(B, C);
  DST(3, 0)             = AVG2(C, D);

  DST(0, 3) =             AVG3(K, J, I);
  DST(0, 2) =             AVG3(J, I, X);
  DST(0, 1) = DST(1, 3) = AVG3(I, X, A);
  DST(1, 1) = DST(2, 3) = AVG3(X, A, B);
  DST(2, 1) = DST(3, 3) = AVG3(A, B, C);
  DST(3, 1) =             AVG3(B, C, D);
}

514 515
static INLINE void d117_predictor(uint8_t *dst, ptrdiff_t stride, int bs,
                                  const uint8_t *above, const uint8_t *left) {
John Koleszar's avatar
John Koleszar committed
516
  int r, c;
517

518
  // first row
519
  for (c = 0; c < bs; c++)
520 521
    dst[c] = ROUND_POWER_OF_TWO(above[c - 1] + above[c], 1);
  dst += stride;
522 523

  // second row
524
  dst[0] = ROUND_POWER_OF_TWO(left[0] + above[-1] * 2 + above[0], 2);
525
  for (c = 1; c < bs; c++)
526 527
    dst[c] = ROUND_POWER_OF_TWO(above[c - 2] + above[c - 1] * 2 + above[c], 2);
  dst += stride;
528 529

  // the rest of first col
530
  dst[0] = ROUND_POWER_OF_TWO(above[-1] + left[0] * 2 + left[1], 2);
531
  for (r = 3; r < bs; ++r)
532 533 534
    dst[(r - 2) * stride] = ROUND_POWER_OF_TWO(left[r - 3] + left[r - 2] * 2 +
                                               left[r - 1], 2);

535
  // the rest of the block
536 537
  for (r = 2; r < bs; ++r) {
    for (c = 1; c < bs; c++)
538 539
      dst[c] = dst[-2 * stride + c - 1];
    dst += stride;
John Koleszar's avatar
John Koleszar committed
540
  }
541
}
542
intra_pred_no_4x4(d117)
543

544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564
void vp9_d135_predictor_4x4(uint8_t *dst, ptrdiff_t stride,
                            const uint8_t *above, const uint8_t *left) {
  const int I = left[0];
  const int J = left[1];
  const int K = left[2];
  const int L = left[3];
  const int X = above[-1];
  const int A = above[0];
  const int B = above[1];
  const int C = above[2];
  const int D = above[3];
  (void)stride;
  DST(0, 3)                                     = AVG3(J, K, L);
  DST(1, 3) = DST(0, 2)                         = AVG3(I, J, K);
  DST(2, 3) = DST(1, 2) = DST(0, 1)             = AVG3(X, I, J);
  DST(3, 3) = DST(2, 2) = DST(1, 1) = DST(0, 0) = AVG3(A, X, I);
              DST(3, 2) = DST(2, 1) = DST(1, 0) = AVG3(B, A, X);
                          DST(3, 1) = DST(2, 0) = AVG3(C, B, A);
                                      DST(3, 0) = AVG3(D, C, B);
}

565 566
static INLINE void d135_predictor(uint8_t *dst, ptrdiff_t stride, int bs,
                                  const uint8_t *above, const uint8_t *left) {
John Koleszar's avatar
John Koleszar committed
567
  int r, c;
568
  dst[0] = ROUND_POWER_OF_TWO(left[0] + above[-1] * 2 + above[0], 2);
569
  for (c = 1; c < bs; c++)
570
    dst[c] = ROUND_POWER_OF_TWO(above[c - 2] + above[c - 1] * 2 + above[c], 2);
571

572
  dst[stride] = ROUND_POWER_OF_TWO(above[-1] + left[0] * 2 + left[1], 2);
573
  for (r = 2; r < bs; ++r)
574 575
    dst[r * stride] = ROUND_POWER_OF_TWO(left[r - 2] + left[r - 1] * 2 +
                                         left[r], 2);
John Koleszar's avatar
John Koleszar committed
576

577
  dst += stride;
578 579
  for (r = 1; r < bs; ++r) {
    for (c = 1; c < bs; c++)
580 581
      dst[c] = dst[-stride + c - 1];
    dst += stride;
John Koleszar's avatar
John Koleszar committed
582
  }
583
}
584
intra_pred_no_4x4(d135)
585

586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609
void vp9_d153_predictor_4x4_c(uint8_t *dst, ptrdiff_t stride,
                              const uint8_t *above, const uint8_t *left) {
  const int I = left[0];
  const int J = left[1];
  const int K = left[2];
  const int L = left[3];
  const int X = above[-1];
  const int A = above[0];
  const int B = above[1];
  const int C = above[2];

  DST(0, 0) = DST(2, 1) = AVG2(I, X);
  DST(0, 1) = DST(2, 2) = AVG2(J, I);
  DST(0, 2) = DST(2, 3) = AVG2(K, J);
  DST(0, 3)             = AVG2(L, K);

  DST(3, 0)             = AVG3(A, B, C);
  DST(2, 0)             = AVG3(X, A, B);
  DST(1, 0) = DST(3, 1) = AVG3(I, X, A);
  DST(1, 1) = DST(3, 2) = AVG3(J, I, X);
  DST(1, 2) = DST(3, 3) = AVG3(K, J, I);
  DST(1, 3)             = AVG3(L, K, J);
}

610 611
static INLINE void d153_predictor(uint8_t *dst, ptrdiff_t stride, int bs,
                                  const uint8_t *above, const uint8_t *left) {
John Koleszar's avatar
John Koleszar committed
612
  int r, c;
613
  dst[0] = ROUND_POWER_OF_TWO(above[-1] + left[0], 1);
614
  for (r = 1; r < bs; r++)
615 616 617 618 619
    dst[r * stride] = ROUND_POWER_OF_TWO(left[r - 1] + left[r], 1);
  dst++;

  dst[0] = ROUND_POWER_OF_TWO(left[0] + above[-1] * 2 + above[0], 2);
  dst[stride] = ROUND_POWER_OF_TWO(above[-1] + left[0] * 2 + left[1], 2);
620
  for (r = 2; r < bs; r++)
621 622 623
    dst[r * stride] = ROUND_POWER_OF_TWO(left[r - 2] + left[r - 1] * 2 +
                                         left[r], 2);
  dst++;
John Koleszar's avatar
John Koleszar committed
624

625
  for (c = 0; c < bs - 2; c++)
626 627 628
    dst[c] = ROUND_POWER_OF_TWO(above[c - 1] + above[c] * 2 + above[c + 1], 2);
  dst += stride;

629 630
  for (r = 1; r < bs; ++r) {
    for (c = 0; c < bs - 2; c++)
631 632
      dst[c] = dst[-stride + c - 2];
    dst += stride;
John Koleszar's avatar
John Koleszar committed
633
  }
634
}
635
intra_pred_no_4x4(d153)
636

637 638
static INLINE void v_predictor(uint8_t *dst, ptrdiff_t stride, int bs,
                               const uint8_t *above, const uint8_t *left) {
639
  int r;
640
  (void) left;
641 642

  for (r = 0; r < bs; r++) {
James Zern's avatar
James Zern committed
643
    memcpy(dst, above, bs);
644
    dst += stride;
645 646
  }
}
647
intra_pred_allsizes(v)
648

649 650
static INLINE void h_predictor(uint8_t *dst, ptrdiff_t stride, int bs,
                               const uint8_t *above, const uint8_t *left) {
651
  int r;
652
  (void) above;
653 654

  for (r = 0; r < bs; r++) {
James Zern's avatar
James Zern committed
655
    memset(dst, left[r], bs);
656
    dst += stride;
657 658
  }
}
659
intra_pred_allsizes(h)
660

661 662
static INLINE void tm_predictor(uint8_t *dst, ptrdiff_t stride, int bs,
                                const uint8_t *above, const uint8_t *left) {
663
  int r, c;
664
  int ytop_left = above[-1];
665 666 667

  for (r = 0; r < bs; r++) {
    for (c = 0; c < bs; c++)
668 669
      dst[c] = clip_pixel(left[r] + above[c] - ytop_left);
    dst += stride;
670 671
  }
}
672
intra_pred_allsizes(tm)
673

674 675
static INLINE void dc_128_predictor(uint8_t *dst, ptrdiff_t stride, int bs,
                                    const uint8_t *above, const uint8_t *left) {
676
  int r;
677 678
  (void) above;
  (void) left;
679 680

  for (r = 0; r < bs; r++) {
James Zern's avatar
James Zern committed
681
    memset(dst, 128, bs);
682
    dst += stride;
683 684
  }
}
685
intra_pred_allsizes(dc_128)
686

687 688 689 690
static INLINE void dc_left_predictor(uint8_t *dst, ptrdiff_t stride, int bs,
                                     const uint8_t *above,
                                     const uint8_t *left) {
  int i, r, expected_dc, sum = 0;
691
  (void) above;
692 693

  for (i = 0; i < bs; i++)
694 695
    sum += left[i];
  expected_dc = (sum + (bs >> 1)) / bs;
696 697

  for (r = 0; r < bs; r++) {
James Zern's avatar
James Zern committed
698
    memset(dst, expected_dc, bs);
699
    dst += stride;
700 701
  }
}
702
intra_pred_allsizes(dc_left)
703

704 705 706
static INLINE void dc_top_predictor(uint8_t *dst, ptrdiff_t stride, int bs,
                                    const uint8_t *above, const uint8_t *left) {
  int i, r, expected_dc, sum = 0;
707
  (void) left;
708 709

  for (i = 0; i < bs; i++)
710 711
    sum += above[i];
  expected_dc = (sum + (bs >> 1)) / bs;
712 713

  for (r = 0; r < bs; r++) {
James Zern's avatar
James Zern committed
714
    memset(dst, expected_dc, bs);
715
    dst += stride;
716 717
  }
}
718
intra_pred_allsizes(dc_top)
719

720 721 722
static INLINE void dc_predictor(uint8_t *dst, ptrdiff_t stride, int bs,
                                const uint8_t *above, const uint8_t *left) {
  int i, r, expected_dc, sum = 0;
723 724
  const int count = 2 * bs;

725 726 727 728 729 730
  for (i = 0; i < bs; i++) {
    sum += above[i];
    sum += left[i];
  }

  expected_dc = (sum + (count >> 1)) / count;
731 732

  for (r = 0; r < bs; r++) {
James Zern's avatar
James Zern committed
733
    memset(dst, expected_dc, bs);
734
    dst += stride;
735 736
  }
}
737 738 739
intra_pred_allsizes(dc)
#undef intra_pred_allsizes

740 741
typedef void (*intra_pred_fn)(uint8_t *dst, ptrdiff_t stride,
                              const uint8_t *above, const uint8_t *left);
742

743 744 745
static intra_pred_fn pred[INTRA_MODES][TX_SIZES];
static intra_pred_fn dc_pred[2][2][TX_SIZES];

746 747 748 749 750 751 752 753
#if CONFIG_VP9_HIGHBITDEPTH
typedef void (*intra_high_pred_fn)(uint16_t *dst, ptrdiff_t stride,
                                   const uint16_t *above, const uint16_t *left,
                                   int bd);
static intra_high_pred_fn pred_high[INTRA_MODES][4];
static intra_high_pred_fn dc_pred_high[2][2][4];
#endif  // CONFIG_VP9_HIGHBITDEPTH

754
static void vp9_init_intra_predictors_internal(void) {
755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775
#define INIT_ALL_SIZES(p, type) \
  p[TX_4X4] = vp9_##type##_predictor_4x4; \
  p[TX_8X8] = vp9_##type##_predictor_8x8; \
  p[TX_16X16] = vp9_##type##_predictor_16x16; \
  p[TX_32X32] = vp9_##type##_predictor_32x32

  INIT_ALL_SIZES(pred[V_PRED], v);
  INIT_ALL_SIZES(pred[H_PRED], h);
  INIT_ALL_SIZES(pred[D207_PRED], d207);
  INIT_ALL_SIZES(pred[D45_PRED], d45);
  INIT_ALL_SIZES(pred[D63_PRED], d63);
  INIT_ALL_SIZES(pred[D117_PRED], d117);
  INIT_ALL_SIZES(pred[D135_PRED], d135);
  INIT_ALL_SIZES(pred[D153_PRED], d153);
  INIT_ALL_SIZES(pred[TM_PRED], tm);

  INIT_ALL_SIZES(dc_pred[0][0], dc_128);
  INIT_ALL_SIZES(dc_pred[0][1], dc_top);
  INIT_ALL_SIZES(dc_pred[1][0], dc_left);
  INIT_ALL_SIZES(dc_pred[1][1], dc);

776
#if CONFIG_VP9_HIGHBITDEPTH
777 778 779 780 781 782 783 784 785 786 787 788 789 790
  INIT_ALL_SIZES(pred_high[V_PRED], highbd_v);
  INIT_ALL_SIZES(pred_high[H_PRED], highbd_h);
  INIT_ALL_SIZES(pred_high[D207_PRED], highbd_d207);
  INIT_ALL_SIZES(pred_high[D45_PRED], highbd_d45);
  INIT_ALL_SIZES(pred_high[D63_PRED], highbd_d63);
  INIT_ALL_SIZES(pred_high[D117_PRED], highbd_d117);
  INIT_ALL_SIZES(pred_high[D135_PRED], highbd_d135);
  INIT_ALL_SIZES(pred_high[D153_PRED], highbd_d153);
  INIT_ALL_SIZES(pred_high[TM_PRED], highbd_tm);

  INIT_ALL_SIZES(dc_pred_high[0][0], highbd_dc_128);
  INIT_ALL_SIZES(dc_pred_high[0][1], highbd_dc_top);
  INIT_ALL_SIZES(dc_pred_high[1][0], highbd_dc_left);
  INIT_ALL_SIZES(dc_pred_high[1][1], highbd_dc);
791 792 793
#endif  // CONFIG_VP9_HIGHBITDEPTH

#undef intra_pred_allsizes
794
}
795

796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811
#if CONFIG_VP9_HIGHBITDEPTH
static void build_intra_predictors_high(const MACROBLOCKD *xd,
                                        const uint8_t *ref8,
                                        int ref_stride,
                                        uint8_t *dst8,
                                        int dst_stride,
                                        PREDICTION_MODE mode,
                                        TX_SIZE tx_size,
                                        int up_available,
                                        int left_available,
                                        int right_available,
                                        int x, int y,
                                        int plane, int bd) {
  int i;
  uint16_t *dst = CONVERT_TO_SHORTPTR(dst8);
  uint16_t *ref = CONVERT_TO_SHORTPTR(ref8);
812
  DECLARE_ALIGNED(16, uint16_t, left_col[32]);
813
  DECLARE_ALIGNED(16, uint16_t, above_data[64 + 16]);
814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872
  uint16_t *above_row = above_data + 16;
  const uint16_t *const_above_row = above_row;
  const int bs = 4 << tx_size;
  int frame_width, frame_height;
  int x0, y0;
  const struct macroblockd_plane *const pd = &xd->plane[plane];
  //  int base=128;
  int base = 128 << (bd - 8);
  // 127 127 127 .. 127 127 127 127 127 127
  // 129  A   B  ..  Y   Z
  // 129  C   D  ..  W   X
  // 129  E   F  ..  U   V
  // 129  G   H  ..  S   T   T   T   T   T

  // Get current frame pointer, width and height.
  if (plane == 0) {
    frame_width = xd->cur_buf->y_width;
    frame_height = xd->cur_buf->y_height;
  } else {
    frame_width = xd->cur_buf->uv_width;
    frame_height = xd->cur_buf->uv_height;
  }

  // Get block position in current frame.
  x0 = (-xd->mb_to_left_edge >> (3 + pd->subsampling_x)) + x;
  y0 = (-xd->mb_to_top_edge >> (3 + pd->subsampling_y)) + y;

  // left
  if (left_available) {
    if (xd->mb_to_bottom_edge < 0) {
      /* slower path if the block needs border extension */
      if (y0 + bs <= frame_height) {
        for (i = 0; i < bs; ++i)
          left_col[i] = ref[i * ref_stride - 1];
      } else {
        const int extend_bottom = frame_height - y0;
        for (i = 0; i < extend_bottom; ++i)
          left_col[i] = ref[i * ref_stride - 1];
        for (; i < bs; ++i)
          left_col[i] = ref[(extend_bottom - 1) * ref_stride - 1];
      }
    } else {
      /* faster path if the block does not need extension */
      for (i = 0; i < bs; ++i)
        left_col[i] = ref[i * ref_stride - 1];
    }
  } else {
    // TODO(Peter): this value should probably change for high bitdepth
    vpx_memset16(left_col, base + 1, bs);
  }

  // TODO(hkuang) do not extend 2*bs pixels for all modes.
  // above
  if (up_available) {
    const uint16_t *above_ref = ref - ref_stride;
    if (xd->mb_to_right_edge < 0) {
      /* slower path if the block needs border extension */
      if (x0 + 2 * bs <= frame_width) {
        if (right_available && bs == 4) {
James Zern's avatar
James Zern committed
873
          memcpy(above_row, above_ref, 2 * bs * sizeof(uint16_t));
874
        } else {
James Zern's avatar
James Zern committed
875
          memcpy(above_row, above_ref, bs * sizeof(uint16_t));
876 877 878 879 880
          vpx_memset16(above_row + bs, above_row[bs - 1], bs);
        }
      } else if (x0 + bs <= frame_width) {
        const int r = frame_width - x0;
        if (right_available && bs == 4) {
James Zern's avatar
James Zern committed
881
          memcpy(above_row, above_ref, r * sizeof(uint16_t));
882 883 884
          vpx_memset16(above_row + r, above_row[r - 1],
                       x0 + 2 * bs - frame_width);
        } else {
James Zern's avatar
James Zern committed
885
          memcpy(above_row, above_ref, bs * sizeof(uint16_t));
886 887 888 889
          vpx_memset16(above_row + bs, above_row[bs - 1], bs);
        }
      } else if (x0 <= frame_width) {
        const int r = frame_width - x0;
James Zern's avatar
James Zern committed
890
        memcpy(above_row, above_ref, r * sizeof(uint16_t));
hkuang's avatar
hkuang committed
891
        vpx_memset16(above_row + r, above_row[r - 1],
892 893 894 895 896 897 898 899 900
                       x0 + 2 * bs - frame_width);
      }
      // TODO(Peter) this value should probably change for high bitdepth
      above_row[-1] = left_available ? above_ref[-1] : (base+1);
    } else {
      /* faster path if the block does not need extension */
      if (bs == 4 && right_available && left_available) {
        const_above_row = above_ref;
      } else {
James Zern's avatar
James Zern committed
901
        memcpy(above_row, above_ref, bs * sizeof(uint16_t));
902
        if (bs == 4 && right_available)
James Zern's avatar
James Zern committed
903
          memcpy(above_row + bs, above_ref + bs, bs * sizeof(uint16_t));
904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927
        else
          vpx_memset16(above_row + bs, above_row[bs - 1], bs);
        // TODO(Peter): this value should probably change for high bitdepth
        above_row[-1] = left_available ? above_ref[-1] : (base+1);
      }
    }
  } else {
    vpx_memset16(above_row, base - 1, bs * 2);
    // TODO(Peter): this value should probably change for high bitdepth
    above_row[-1] = base - 1;
  }

  // predict
  if (mode == DC_PRED) {
    dc_pred_high[left_available][up_available][tx_size](dst, dst_stride,
                                                        const_above_row,
                                                        left_col, xd->bd);
  } else {
    pred_high[mode][tx_size](dst, dst_stride, const_above_row, left_col,
                             xd->bd);
  }
}
#endif  // CONFIG_VP9_HIGHBITDEPTH

928 929
static void build_intra_predictors(const MACROBLOCKD *xd, const uint8_t *ref,
                                   int ref_stride, uint8_t *dst, int dst_stride,
930
                                   PREDICTION_MODE mode, TX_SIZE tx_size,
931
                                   int up_available, int left_available,
932 933
                                   int right_available, int x, int y,
                                   int plane) {
934
  int i;
935
  DECLARE_ALIGNED(16, uint8_t, left_col[32]);
936
  DECLARE_ALIGNED(16, uint8_t, above_data[64 + 16]);
937 938 939
  uint8_t *above_row = above_data + 16;
  const uint8_t *const_above_row = above_row;
  const int bs = 4 << tx_size;
940 941 942
  int frame_width, frame_height;
  int x0, y0;
  const struct macroblockd_plane *const pd = &xd->plane[plane];
Dmitry Kovalev's avatar
Dmitry Kovalev committed
943 944 945 946 947 948 949

  // 127 127 127 .. 127 127 127 127 127 127
  // 129  A   B  ..  Y   Z
  // 129  C   D  ..  W   X
  // 129  E   F  ..  U   V
  // 129  G   H  ..  S   T   T   T   T   T
  // ..
John Koleszar's avatar
John Koleszar committed
950

951 952 953 954 955 956 957 958 959 960 961 962 963
  // Get current frame pointer, width and height.
  if (plane == 0) {
    frame_width = xd->cur_buf->y_width;
    frame_height = xd->cur_buf->y_height;
  } else {
    frame_width = xd->cur_buf->uv_width;
    frame_height = xd->cur_buf->uv_height;
  }

  // Get block position in current frame.
  x0 = (-xd->mb_to_left_edge >> (3 + pd->subsampling_x)) + x;
  y0 = (-xd->mb_to_top_edge >> (3 + pd->subsampling_y)) + y;

964 965 966 967 968 969 970 971 972 973 974 975 976 977 978
  // NEED_LEFT
  if (extend_modes[mode] & NEED_LEFT) {
    if (left_available) {
      if (xd->mb_to_bottom_edge < 0) {
        /* slower path if the block needs border extension */
        if (y0 + bs <= frame_height) {
          for (i = 0; i < bs; ++i)
            left_col[i] = ref[i * ref_stride - 1];
        } else {
          const int extend_bottom = frame_height - y0;
          for (i = 0; i < extend_bottom; ++i)
            left_col[i] = ref[i * ref_stride - 1];
          for (; i < bs; ++i)
            left_col[i] = ref[(extend_bottom - 1) * ref_stride - 1];
        }
979
      } else {
980 981
        /* faster path if the block does not need extension */
        for (i = 0; i < bs; ++i)
982 983 984
          left_col[i] = ref[i * ref_stride - 1];
      }
    } else {
James Zern's avatar
James Zern committed
985
      memset(left_col, 129, bs);
986
    }
987 988
  }