diff --git a/vp9/encoder/vp9_encodeframe.c b/vp9/encoder/vp9_encodeframe.c
index e4acc7419e267119e310e443a28b19cfa70b6eac..dcb972eddc16ab69c273041e18a6ee0c9229dec6 100644
--- a/vp9/encoder/vp9_encodeframe.c
+++ b/vp9/encoder/vp9_encodeframe.c
@@ -1999,13 +1999,6 @@ static void encode_sb_row(VP9_COMP *cpi, int mi_row, TOKENEXTRA **tp,
     int dummy_rate;
     int64_t dummy_dist;
 
-    // Initialize a mask of modes that we will not consider;
-    // cpi->unused_mode_skip_mask = 0x0000000AAE17F800 (test no golden)
-    if (cpi->common.frame_type == KEY_FRAME)
-      cpi->unused_mode_skip_mask = 0;
-    else
-      cpi->unused_mode_skip_mask = 0xFFFFFFFFFFFFFE00;
-
     if (cpi->sf.reference_masking)
       rd_pick_reference_frame(cpi, mi_row, mi_col);
 
diff --git a/vp9/encoder/vp9_onyx_if.c b/vp9/encoder/vp9_onyx_if.c
index 8b0af311fb5874b061e593cb75f50ad9cef6fe56..94ac89d48df4c7b42c0924b6ca80bd69e5a3df6e 100644
--- a/vp9/encoder/vp9_onyx_if.c
+++ b/vp9/encoder/vp9_onyx_if.c
@@ -743,9 +743,8 @@ void vp9_set_speed_features(VP9_COMP *cpi) {
   sf->use_fast_lpf_pick = 0;
   sf->use_fast_coef_updates = 0;
   sf->using_small_partition_info = 0;
-  // Skip any mode not chosen at size < X for all sizes > X
-  // Hence BLOCK_64X64 (skip is off)
-  sf->unused_mode_skip_lvl = BLOCK_64X64;
+  sf->mode_skip_start = MAX_MODES;  // Mode index at which mode skip mask set
+
 
 #if CONFIG_MULTIPLE_ARF
   // Switch segmentation off.
@@ -782,7 +781,6 @@ void vp9_set_speed_features(VP9_COMP *cpi) {
                                    cpi->common.show_frame == 0);
         sf->disable_splitmv =
             (MIN(cpi->common.width, cpi->common.height) >= 720)? 1 : 0;
-        sf->unused_mode_skip_lvl = BLOCK_32X32;
         sf->mode_search_skip_flags = FLAG_SKIP_INTRA_DIRMISMATCH |
                                      FLAG_SKIP_INTRA_BESTINTER |
                                      FLAG_SKIP_COMP_BESTINTRA |
@@ -804,6 +802,7 @@ void vp9_set_speed_features(VP9_COMP *cpi) {
         sf->intra_y_mode_mask = INTRA_DC_TM_H_V;
         sf->intra_uv_mode_mask = INTRA_DC_TM_H_V;
         sf->use_fast_coef_updates = 1;
+        sf->mode_skip_start = 9;
       }
       if (speed == 2) {
         sf->adjust_thresholds_by_speed = 1;
@@ -813,7 +812,6 @@ void vp9_set_speed_features(VP9_COMP *cpi) {
         sf->use_lastframe_partitioning = 1;
         sf->adjust_partitioning_from_last_frame = 1;
         sf->last_partitioning_redo_frequency = 3;
-        sf->unused_mode_skip_lvl = BLOCK_32X32;
         sf->tx_size_search_method = ((cpi->common.frame_type == KEY_FRAME ||
                                       cpi->common.intra_only ||
                                       cpi->common.show_frame == 0) ?
@@ -843,6 +841,7 @@ void vp9_set_speed_features(VP9_COMP *cpi) {
         sf->disable_split_var_thresh = 32;
         sf->disable_filter_search_var_thresh = 32;
         sf->use_fast_coef_updates = 2;
+        sf->mode_skip_start = 9;
       }
       if (speed == 3) {
         sf->comp_inter_joint_search_thresh = BLOCK_SIZES;
@@ -870,6 +869,7 @@ void vp9_set_speed_features(VP9_COMP *cpi) {
         sf->intra_y_mode_mask = INTRA_DC_ONLY;
         sf->intra_uv_mode_mask = INTRA_DC_ONLY;
         sf->use_fast_coef_updates = 2;
+        sf->mode_skip_start = 9;
       }
       if (speed == 4) {
         sf->comp_inter_joint_search_thresh = BLOCK_SIZES;
@@ -901,20 +901,8 @@ void vp9_set_speed_features(VP9_COMP *cpi) {
         sf->intra_y_mode_mask = INTRA_DC_ONLY;
         sf->intra_uv_mode_mask = INTRA_DC_ONLY;
         sf->use_fast_coef_updates = 2;
+        sf->mode_skip_start = 9;
       }
-      /*
-      if (speed == 2) {
-        sf->first_step = 0;
-        sf->comp_inter_joint_search_thresh = BLOCK_8X8;
-        sf->max_partition_size = BLOCK_16X16;
-      }
-      if (speed == 3) {
-        sf->first_step = 0;
-        sf->comp_inter_joint_search_thresh = BLOCK_B8X8;
-        sf->min_partition_size = BLOCK_8X8;
-      }
-      */
-
       break;
 
   }; /* switch */
diff --git a/vp9/encoder/vp9_onyx_int.h b/vp9/encoder/vp9_onyx_int.h
index 2dbd7a01af567a16cbfbce2abe391f9497aeee6c..334a23c05641261e26dbc5a8c737d1268577528b 100644
--- a/vp9/encoder/vp9_onyx_int.h
+++ b/vp9/encoder/vp9_onyx_int.h
@@ -147,18 +147,19 @@ typedef struct {
 // const MODE_DEFINITION vp9_mode_order[MAX_MODES] used in the rd code.
 typedef enum {
   THR_NEARESTMV,
-  THR_DC,
-
   THR_NEARESTA,
   THR_NEARESTG,
-  THR_NEWMV,
-  THR_COMP_NEARESTLA,
-  THR_NEARMV,
-  THR_COMP_NEARESTGA,
 
-  THR_NEWG,
+  THR_DC,
+
+  THR_NEWMV,
   THR_NEWA,
+  THR_NEWG,
+
+  THR_NEARMV,
   THR_NEARA,
+  THR_COMP_NEARESTLA,
+  THR_COMP_NEARESTGA,
 
   THR_TM,
 
@@ -273,7 +274,7 @@ typedef struct {
   int use_one_partition_size_always;
   int less_rectangular_check;
   int use_square_partition_only;
-  int unused_mode_skip_lvl;
+  int mode_skip_start;
   int reference_masking;
   BLOCK_SIZE always_this_block_size;
   int auto_min_max_partition_size;
@@ -384,7 +385,7 @@ typedef struct VP9_COMP {
   unsigned int mode_check_freq[MAX_MODES];
   unsigned int mode_test_hit_counts[MAX_MODES];
   unsigned int mode_chosen_counts[MAX_MODES];
-  int64_t unused_mode_skip_mask;
+  int64_t mode_skip_mask;
   int ref_frame_mask;
   int set_ref_frame_mask;
 
diff --git a/vp9/encoder/vp9_rdopt.c b/vp9/encoder/vp9_rdopt.c
index 41e43fe50e7b7ddeea82710d47f1ee7c1048eac2..53a064c393bd89c044346c412f9ed0f3e8d4c2ae 100644
--- a/vp9/encoder/vp9_rdopt.c
+++ b/vp9/encoder/vp9_rdopt.c
@@ -51,20 +51,25 @@ DECLARE_ALIGNED(16, extern const uint8_t,
 #define I4X4_PRED 0x8000
 #define SPLITMV 0x10000
 
+#define LAST_FRAME_MODE_MASK    0xFFDADCD60
+#define GOLDEN_FRAME_MODE_MASK  0xFFB5A3BB0
+#define ALT_REF_MODE_MASK       0xFF8C648D0
+
 const MODE_DEFINITION vp9_mode_order[MAX_MODES] = {
   {NEARESTMV, LAST_FRAME,   NONE},
-  {DC_PRED,   INTRA_FRAME,  NONE},
-
   {NEARESTMV, ALTREF_FRAME, NONE},
   {NEARESTMV, GOLDEN_FRAME, NONE},
-  {NEWMV,     LAST_FRAME,   NONE},
-  {NEARESTMV, LAST_FRAME,   ALTREF_FRAME},
-  {NEARMV,    LAST_FRAME,   NONE},
-  {NEARESTMV, GOLDEN_FRAME, ALTREF_FRAME},
 
-  {NEWMV,     GOLDEN_FRAME, NONE},
+  {DC_PRED,   INTRA_FRAME,  NONE},
+
+  {NEWMV,     LAST_FRAME,   NONE},
   {NEWMV,     ALTREF_FRAME, NONE},
+  {NEWMV,     GOLDEN_FRAME, NONE},
+
+  {NEARMV,    LAST_FRAME,   NONE},
   {NEARMV,    ALTREF_FRAME, NONE},
+  {NEARESTMV, LAST_FRAME,   ALTREF_FRAME},
+  {NEARESTMV, GOLDEN_FRAME, ALTREF_FRAME},
 
   {TM_PRED,   INTRA_FRAME,  NONE},
 
@@ -3187,10 +3192,31 @@ int64_t vp9_rd_pick_inter_mode_sb(VP9_COMP *cpi, MACROBLOCK *x,
     ref_frame = vp9_mode_order[mode_index].ref_frame;
     second_ref_frame = vp9_mode_order[mode_index].second_ref_frame;
 
-    // Skip modes that have been masked off but always consider first mode.
-    if (mode_index && (bsize > cpi->sf.unused_mode_skip_lvl) &&
-         (cpi->unused_mode_skip_mask & (1 << mode_index)) )
-      continue;
+    // Look at the reference frame of the best mode so far and set the
+    // skip mask to look at a subset of the remaining modes.
+    if (mode_index > cpi->sf.mode_skip_start) {
+      if (mode_index == (cpi->sf.mode_skip_start + 1)) {
+        switch (vp9_mode_order[best_mode_index].ref_frame) {
+          case INTRA_FRAME:
+            cpi->mode_skip_mask = 0;
+            break;
+          case LAST_FRAME:
+            cpi->mode_skip_mask = LAST_FRAME_MODE_MASK;
+            break;
+          case GOLDEN_FRAME:
+            cpi->mode_skip_mask = GOLDEN_FRAME_MODE_MASK;
+            break;
+          case ALTREF_FRAME:
+            cpi->mode_skip_mask = ALT_REF_MODE_MASK;
+            break;
+          case NONE:
+          case MAX_REF_FRAMES:
+            assert(!"Invalid Reference frame");
+        }
+      }
+      if (cpi->mode_skip_mask & (1 << mode_index))
+        continue;
+    }
 
     // Skip if the current reference frame has been masked off
     if (cpi->sf.reference_masking && !cpi->set_ref_frame_mask &&
@@ -3859,13 +3885,6 @@ int64_t vp9_rd_pick_inter_mode_sb(VP9_COMP *cpi, MACROBLOCK *x,
     }
   }
 
-  // If indicated then mark the index of the chosen mode to be inspected at
-  // other block sizes.
-  if (bsize <= cpi->sf.unused_mode_skip_lvl) {
-    cpi->unused_mode_skip_mask = cpi->unused_mode_skip_mask &
-                                 (~((int64_t)1 << best_mode_index));
-  }
-
   // If we are using reference masking and the set mask flag is set then
   // create the reference frame mask.
   if (cpi->sf.reference_masking && cpi->set_ref_frame_mask)