diff --git a/vp9/common/vp9_reconinter.c b/vp9/common/vp9_reconinter.c
index 9a1f5fc07895875021be95875b1567272b206fae..3c2b2cdf68b596c7edae9c91f4af9561755ebc6c 100644
--- a/vp9/common/vp9_reconinter.c
+++ b/vp9/common/vp9_reconinter.c
@@ -210,7 +210,7 @@ void vp9_setup_scale_factors(VP9_COMMON *cm, int i) {
     vp9_zero(*sf);
   } else {
     YV12_BUFFER_CONFIG *const fb = &cm->yv12_fb[ref];
-    vp9_setup_scale_factors_for_frame(sf,
+    vp9_setup_scale_factors_for_frame(cm, sf,
                                       fb->y_crop_width, fb->y_crop_height,
                                       cm->width, cm->height);
 
diff --git a/vp9/common/vp9_scale.c b/vp9/common/vp9_scale.c
index 3f28d2811ec609f0658df7b296d3abcd074bc910..12acac1c930d7877bde747a21e31d8a218cff85a 100644
--- a/vp9/common/vp9_scale.c
+++ b/vp9/common/vp9_scale.c
@@ -10,6 +10,7 @@
 
 #include "./vp9_rtcd.h"
 #include "vp9/common/vp9_filter.h"
+#include "vp9/common/vp9_onyxc_int.h"
 #include "vp9/common/vp9_scale.h"
 
 static INLINE int scaled_x(int val, const struct scale_factors *scale) {
@@ -61,9 +62,23 @@ static int get_fixed_point_scale_factor(int other_size, int this_size) {
   return (other_size << VP9_REF_SCALE_SHIFT) / this_size;
 }
 
-void vp9_setup_scale_factors_for_frame(struct scale_factors *scale,
+static int check_scale_factors(int other_w, int other_h,
+                               int this_w, int this_h) {
+  return 2 * this_w >= other_w &&
+         2 * this_h >= other_h &&
+         this_w <= 16 * other_w &&
+         this_h <= 16 * other_h;
+}
+
+void vp9_setup_scale_factors_for_frame(struct VP9Common *cm,
+                                       struct scale_factors *scale,
                                        int other_w, int other_h,
                                        int this_w, int this_h) {
+  if (!check_scale_factors(other_w, other_h, this_w, this_h))
+    vpx_internal_error(&cm->error, VPX_CODEC_UNSUP_BITSTREAM,
+                       "Invalid scale factors");
+
+
   scale->x_scale_fp = get_fixed_point_scale_factor(other_w, this_w);
   scale->x_offset_q4 = 0;  // calculated per block
   scale->x_step_q4 = scaled_x(16, scale);
diff --git a/vp9/common/vp9_scale.h b/vp9/common/vp9_scale.h
index 0414dde5e60a085656b246846ebfe187293a414e..21a46dd2f0facc4054cad5262ceb9dee6f74e332 100644
--- a/vp9/common/vp9_scale.h
+++ b/vp9/common/vp9_scale.h
@@ -14,6 +14,8 @@
 #include "vp9/common/vp9_mv.h"
 #include "vp9/common/vp9_convolve.h"
 
+struct VP9Common;
+
 #define VP9_REF_SCALE_SHIFT 14
 #define VP9_REF_NO_SCALE (1 << VP9_REF_SCALE_SHIFT)
 
@@ -33,7 +35,8 @@ struct scale_factors {
   convolve_fn_t predict[2][2][2];  // horiz, vert, avg
 };
 
-void vp9_setup_scale_factors_for_frame(struct scale_factors *scale,
+void vp9_setup_scale_factors_for_frame(struct VP9Common *cm,
+                                       struct scale_factors *scale,
                                        int other_w, int other_h,
                                        int this_w, int this_h);
 
diff --git a/vp9/encoder/vp9_temporal_filter.c b/vp9/encoder/vp9_temporal_filter.c
index 3052e8f70f07281cf58afa383344eadb881a3fad..74fb07ee3316faf5784a700ee3ab688d07ab1a62 100644
--- a/vp9/encoder/vp9_temporal_filter.c
+++ b/vp9/encoder/vp9_temporal_filter.c
@@ -437,7 +437,7 @@ void vp9_temporal_filter_prepare(VP9_COMP *cpi, int distance) {
 #endif
 
   // Setup scaling factors. Scaling on each of the arnr frames is not supported
-  vp9_setup_scale_factors_for_frame(&cpi->mb.e_mbd.scale_factor[0],
+  vp9_setup_scale_factors_for_frame(cm, &cpi->mb.e_mbd.scale_factor[0],
       cm->yv12_fb[cm->new_fb_idx].y_crop_width,
       cm->yv12_fb[cm->new_fb_idx].y_crop_height,
       cm->width, cm->height);