Commit 87bf1a14 authored by Yaowu Xu's avatar Yaowu Xu

Fix ubsan warnings: vp9/encoder/vp9_mcomp.c

This commit fixes a number of ubsan warnings in HBD build.

BUG=webm:1219

Change-Id: I05f0fd0ef50e93db4ba34205005c54af1ed32acc
parent 40477f1f
......@@ -59,8 +59,8 @@ static unsigned int do_16x16_motion_iteration(VP9_COMP *cpi,
// Try sub-pixel MC
// if (bestsme > error_thresh && bestsme < INT_MAX)
{
int distortion;
unsigned int sse;
uint32_t distortion;
uint32_t sse;
cpi->find_fractional_mv_step(
x, dst_mv, ref_mv, cpi->common.allow_high_precision_mv, x->errorperbit,
&v_fn_ptr, 0, mv_sf->subpel_iters_per_step,
......
This diff is collapsed.
......@@ -74,7 +74,7 @@ unsigned int vp9_int_pro_motion_estimation(const struct VP9_COMP *cpi,
BLOCK_SIZE bsize,
int mi_row, int mi_col);
typedef int (fractional_mv_step_fp) (
typedef uint32_t (fractional_mv_step_fp) (
const MACROBLOCK *x,
MV *bestmv, const MV *ref_mv,
int allow_hp,
......@@ -84,7 +84,7 @@ typedef int (fractional_mv_step_fp) (
int iters_per_step,
int *cost_list,
int *mvjcost, int *mvcost[2],
int *distortion, unsigned int *sse1,
uint32_t *distortion, uint32_t *sse1,
const uint8_t *second_pred,
int w, int h);
......
......@@ -157,7 +157,7 @@ static int combined_motion_search(VP9_COMP *cpi, MACROBLOCK *x,
const int ref = mi->ref_frame[0];
const MV ref_mv = x->mbmi_ext->ref_mvs[ref][0].as_mv;
MV center_mv;
int dis;
uint32_t dis;
int rate_mode;
const int tmp_col_min = x->mv_col_min;
const int tmp_col_max = x->mv_col_max;
......@@ -1564,7 +1564,8 @@ void vp9_pick_inter_mode(VP9_COMP *cpi, MACROBLOCK *x,
!cpi->use_svc &&
cpi->oxcf.rc_mode == VPX_CBR) {
int tmp_sad;
int dis, cost_list[5];
uint32_t dis;
int cost_list[5];
if (bsize < BLOCK_16X16)
continue;
......@@ -2175,7 +2176,7 @@ void vp9_pick_inter_mode_sub8x8(VP9_COMP *cpi, MACROBLOCK *x,
const int tmp_col_max = x->mv_col_max;
const int tmp_row_min = x->mv_row_min;
const int tmp_row_max = x->mv_row_max;
int dummy_dist;
uint32_t dummy_dist;
if (i == 0) {
mvp_full.row = b_mv[NEARESTMV].as_mv.row >> 3;
......
......@@ -1710,8 +1710,8 @@ static void joint_motion_search(VP9_COMP *cpi, MACROBLOCK *x,
x->mv_row_max = tmp_row_max;
if (bestsme < INT_MAX) {
int dis; /* TODO: use dis in distortion calculation later. */
unsigned int sse;
uint32_t dis; /* TODO: use dis in distortion calculation later. */
uint32_t sse;
bestsme = cpi->find_fractional_mv_step(
x, &tmp_mv,
&ref_mv[id].as_mv,
......@@ -1911,7 +1911,7 @@ static int64_t rd_pick_best_sub8x8_mode(VP9_COMP *cpi, MACROBLOCK *x,
INT_MAX, 1);
if (bestsme < INT_MAX) {
int distortion;
uint32_t distortion;
cpi->find_fractional_mv_step(
x,
new_mv,
......@@ -2341,7 +2341,7 @@ static void single_motion_search(VP9_COMP *cpi, MACROBLOCK *x,
x->mv_row_max = tmp_row_max;
if (bestsme < INT_MAX) {
int dis; /* TODO: use dis in distortion calculation later. */
uint32_t dis; /* TODO: use dis in distortion calculation later. */
cpi->find_fractional_mv_step(x, &tmp_mv->as_mv, &ref_mv,
cm->allow_high_precision_mv,
x->errorperbit,
......
......@@ -264,8 +264,8 @@ static int temporal_filter_find_matching_mb_c(VP9_COMP *cpi,
int step_param;
int sadpb = x->sadperbit16;
int bestsme = INT_MAX;
int distortion;
unsigned int sse;
uint32_t distortion;
uint32_t sse;
int cost_list[5];
MV best_ref_mv1 = {0, 0};
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment