diff --git a/vp9/common/vp9_blockd.h b/vp9/common/vp9_blockd.h
index 17e847462ad983172fe6a47a825247febe84827d..27ef22fffd3fe956d37919da701c210ea2c7d20c 100644
--- a/vp9/common/vp9_blockd.h
+++ b/vp9/common/vp9_blockd.h
@@ -45,6 +45,7 @@ void vpx_log(const char *format, ...);
 #define SEGMENT_DELTADATA   0
 #define SEGMENT_ABSDATA     1
 #define MAX_MV_REFS 9
+#define MAX_MV_REF_CANDIDATES 4
 
 typedef struct {
   int r, c;
@@ -238,7 +239,7 @@ typedef struct {
   MV_REFERENCE_FRAME ref_frame, second_ref_frame;
   TX_SIZE txfm_size;
   int_mv mv[2]; // for each reference frame used
-  int_mv ref_mvs[MAX_REF_FRAMES][MAX_MV_REFS];
+  int_mv ref_mvs[MAX_REF_FRAMES][MAX_MV_REF_CANDIDATES];
   int_mv best_mv, best_second_mv;
 #if CONFIG_NEW_MVREF
   int best_index, best_second_index;
diff --git a/vp9/common/vp9_findnearmv.c b/vp9/common/vp9_findnearmv.c
index 02c28fbdee0298ad1629492670539311f4c395e0..4fd468973b51f34b8fc5f18333b15b56bcb400e6 100644
--- a/vp9/common/vp9_findnearmv.c
+++ b/vp9/common/vp9_findnearmv.c
@@ -139,8 +139,8 @@ void vp9_find_best_ref_mvs(MACROBLOCKD *xd,
 #if CONFIG_SUBPELREFMV
   unsigned int sse;
 #endif
-  unsigned int ref_scores[MAX_MV_REFS] = {0};
-  int_mv sorted_mvs[MAX_MV_REFS];
+  unsigned int ref_scores[MAX_MV_REF_CANDIDATES] = {0};
+  int_mv sorted_mvs[MAX_MV_REF_CANDIDATES];
   int zero_seen = FALSE;
 
   // Default all to 0,0 if nothing else available
@@ -159,9 +159,8 @@ void vp9_find_best_ref_mvs(MACROBLOCKD *xd,
   left_ref  = ref_y_buffer - 3;
 #endif
 
-  //for(i = 0; i < MAX_MV_REFS; ++i) {
-  // Limit search to the predicted best 4
-  for(i = 0; i < 4; ++i) {
+  // Limit search to the predicted best few candidates
+  for(i = 0; i < MAX_MV_REF_CANDIDATES; ++i) {
     int_mv this_mv;
     int offset = 0;
     int row_offset, col_offset;
@@ -268,7 +267,7 @@ void vp9_find_best_ref_mvs(MACROBLOCKD *xd,
   }
 
   // Make sure all the candidates are properly clamped etc
-  for (i = 0; i < 4; ++i) {
+  for (i = 0; i < MAX_MV_REF_CANDIDATES; ++i) {
     lower_mv_precision(&sorted_mvs[i], xd->allow_high_precision_mv);
     clamp_mv2(&sorted_mvs[i], xd);
   }
diff --git a/vp9/common/vp9_mvref_common.c b/vp9/common/vp9_mvref_common.c
index bf60630e38aba33e3d5f1307e09557591647e5d3..bbed0554f9cd1ae4aa5cffea5c217254c882029b 100644
--- a/vp9/common/vp9_mvref_common.c
+++ b/vp9/common/vp9_mvref_common.c
@@ -170,14 +170,20 @@ static void addmv_and_shuffle(
   int weight
 ) {
 
-  int i = *index;
+  int i;
+  int insert_point;
   int duplicate_found = FALSE;
 
-  // Check for duplicates. If there is one increment its score.
-  // Duplicate defined as being the same full pel vector with rounding.
+  // Check for duplicates. If there is one increase its score.
+  // We only compare vs the current top candidates.
+  insert_point = (*index < (MAX_MV_REF_CANDIDATES - 1))
+                 ? *index : (MAX_MV_REF_CANDIDATES - 1);
+
+  i = insert_point;
+  if (*index > i)
+    i++;
   while (i > 0) {
     i--;
-
     if (candidate_mv.as_int == mv_list[i].as_int) {
       duplicate_found = TRUE;
       mv_scores[i] += weight;
@@ -185,11 +191,13 @@ static void addmv_and_shuffle(
     }
   }
 
-  // If no duplicate was found add the new vector and give it a weight
-  if (!duplicate_found) {
-    mv_list[*index].as_int = candidate_mv.as_int;
-    mv_scores[*index] = weight;
-    i = *index;
+  // If no duplicate and the new candidate is good enough then add it.
+  if (!duplicate_found ) {
+    if (weight > mv_scores[insert_point]) {
+      mv_list[insert_point].as_int = candidate_mv.as_int;
+      mv_scores[insert_point] = weight;
+      i = insert_point;
+    }
     (*index)++;
   }
 
@@ -224,12 +232,12 @@ void vp9_find_mv_refs(
   int i;
   MODE_INFO *candidate_mi;
   MB_MODE_INFO * mbmi = &xd->mode_info_context->mbmi;
-  int_mv candidate_mvs[MAX_MV_REFS];
+  int_mv candidate_mvs[MAX_MV_REF_CANDIDATES];
   int_mv c_refmv;
-  MV_REFERENCE_FRAME c_ref_frame;
   int_mv c2_refmv;
+  MV_REFERENCE_FRAME c_ref_frame;
   MV_REFERENCE_FRAME c2_ref_frame;
-  int candidate_scores[MAX_MV_REFS];
+  int candidate_scores[MAX_MV_REF_CANDIDATES];
   int index = 0;
   int split_count = 0;
   int ref_weight = 0;
@@ -238,8 +246,8 @@ void vp9_find_mv_refs(
   int *ref_distance_weight;
 
   // Blank the reference vector lists and other local structures.
-  vpx_memset(mv_ref_list, 0, sizeof(int_mv) * MAX_MV_REFS);
-  vpx_memset(candidate_mvs, 0, sizeof(int_mv) * MAX_MV_REFS);
+  vpx_memset(mv_ref_list, 0, sizeof(int_mv) * MAX_MV_REF_CANDIDATES);
+  vpx_memset(candidate_mvs, 0, sizeof(int_mv) * MAX_MV_REF_CANDIDATES);
   vpx_memset(candidate_scores, 0, sizeof(candidate_scores));
 
 #if CONFIG_SUPERBLOCKS
@@ -349,11 +357,6 @@ void vp9_find_mv_refs(
     }
   }
 
-  // Make sure we are able to add 0,0
-  if (index > (MAX_MV_REFS - 1)) {
-    index = (MAX_MV_REFS - 1);
-  }
-
   // Define inter mode coding context.
   // 0,0 was best
   if (candidate_mvs[0].as_int == 0) {
@@ -383,14 +386,12 @@ void vp9_find_mv_refs(
   }
 
   // 0,0 is always a valid reference.
-  for (i = 0; i < index; ++i) {
+  for (i = 0; i < MAX_MV_REF_CANDIDATES; ++i) {
     if (candidate_mvs[i].as_int == 0)
       break;
   }
-  if (i == index) {
-    c_refmv.as_int = 0;
-    addmv_and_shuffle(candidate_mvs, candidate_scores,
-                      &index, c_refmv, candidate_scores[3]+1 );
+  if (i == MAX_MV_REF_CANDIDATES) {
+    candidate_mvs[MAX_MV_REF_CANDIDATES-1].as_int = 0;
   }
 
   // Copy over the candidate list.
diff --git a/vp9/encoder/vp9_block.h b/vp9/encoder/vp9_block.h
index d5bf89c7d615db45ffd758df266bd18042ca921e..4669d2ed6945dbccff7d75213c748f30c699cf37 100644
--- a/vp9/encoder/vp9_block.h
+++ b/vp9/encoder/vp9_block.h
@@ -70,7 +70,7 @@ typedef struct {
   PARTITION_INFO partition_info;
   int_mv best_ref_mv;
   int_mv second_best_ref_mv;
-  int_mv ref_mvs[MAX_REF_FRAMES][MAX_MV_REFS];
+  int_mv ref_mvs[MAX_REF_FRAMES][MAX_MV_REF_CANDIDATES];
   int rate;
   int distortion;
   int64_t intra_error;
diff --git a/vp9/encoder/vp9_encodeframe.c b/vp9/encoder/vp9_encodeframe.c
index d2008a775bb8bca8454e4275d8a151bd59255bdb..6ab23cae0b2f43480dea341bd2e1ea362e5b1703 100644
--- a/vp9/encoder/vp9_encodeframe.c
+++ b/vp9/encoder/vp9_encodeframe.c
@@ -395,8 +395,7 @@ static unsigned int pick_best_mv_ref(MACROBLOCK *x,
          vp9_mv_bit_cost(&target_mv, &mv_ref_list[0], x->nmvjointcost,
                          x->mvcost, 96, xd->allow_high_precision_mv);
 
-  // Use 4 for now : for (i = 1; i < MAX_MV_REFS; ++i ) {
-  for (i = 1; i < 4; ++i) {
+  for (i = 1; i < MAX_MV_REF_CANDIDATES; ++i) {
     // If we see a 0,0 reference vector for a second time we have reached
     // the end of the list of valid candidate vectors.
     if (!mv_ref_list[i].as_int) {
diff --git a/vp9/encoder/vp9_onyx_if.c b/vp9/encoder/vp9_onyx_if.c
index 8cabed9c9704deae37db7f8047979b89730003a4..448b56a08cf950dcbb3d71491bc1c9c1d738510f 100644
--- a/vp9/encoder/vp9_onyx_if.c
+++ b/vp9/encoder/vp9_onyx_if.c
@@ -3821,7 +3821,7 @@ static void encode_frame_to_data_rate
   {
     FILE *f = fopen("mv_ref_dist.stt", "a");
     unsigned int i;
-    for (i = 0; i < MAX_MV_REFS; ++i) {
+    for (i = 0; i < MAX_MV_REF_CANDIDATES; ++i) {
       fprintf(f, "%10d", cpi->best_ref_index_counts[0][i]);
     }
     fprintf(f, "\n" );
diff --git a/vp9/encoder/vp9_onyx_int.h b/vp9/encoder/vp9_onyx_int.h
index 8180d16de3a7bca86aa43f2b685486df7b784992..511e62f1c85776012cef5da467438502977f9465 100644
--- a/vp9/encoder/vp9_onyx_int.h
+++ b/vp9/encoder/vp9_onyx_int.h
@@ -790,7 +790,7 @@ typedef struct VP9_COMP {
   unsigned int switchable_interp_count[VP9_SWITCHABLE_FILTERS + 1]
                                       [VP9_SWITCHABLE_FILTERS];
 #if CONFIG_NEW_MVREF
-  unsigned int best_ref_index_counts[MAX_REF_FRAMES][MAX_MV_REFS];
+  unsigned int best_ref_index_counts[MAX_REF_FRAMES][MAX_MV_REF_CANDIDATES];
 #endif
 
 } VP9_COMP;