diff --git a/vp9/encoder/vp9_onyx_int.h b/vp9/encoder/vp9_onyx_int.h
index 0a6aab902d88aa1597d291051ba06c8582a9be09..d2f42dd3e4b177fbbf8f339b541b65897b89534d 100644
--- a/vp9/encoder/vp9_onyx_int.h
+++ b/vp9/encoder/vp9_onyx_int.h
@@ -756,8 +756,10 @@ static int get_token_alloc(int mb_rows, int mb_cols) {
 
 static void set_ref_ptrs(VP9_COMMON *cm, MACROBLOCKD *xd,
                          MV_REFERENCE_FRAME ref0, MV_REFERENCE_FRAME ref1) {
-  xd->block_refs[0] = &cm->frame_refs[ref0 - LAST_FRAME];
-  xd->block_refs[1] = &cm->frame_refs[ref1 - LAST_FRAME];
+  xd->block_refs[0] = &cm->frame_refs[ref0 >= LAST_FRAME ? ref0 - LAST_FRAME
+                                                         : 0];
+  xd->block_refs[1] = &cm->frame_refs[ref1 >= LAST_FRAME ? ref1 - LAST_FRAME
+                                                         : 0];
 }
 
 #ifdef __cplusplus