diff --git a/vp9/encoder/vp9_bitstream.c b/vp9/encoder/vp9_bitstream.c
index 88e25ebf26000e40468530550ff14ccc04b7e72a..547ad52f65003f8c62c163ff42c29a3ce43c3967 100644
--- a/vp9/encoder/vp9_bitstream.c
+++ b/vp9/encoder/vp9_bitstream.c
@@ -1256,23 +1256,23 @@ static size_t write_compressed_header(VP9_COMP *cpi, uint8_t *data) {
         if (use_hybrid_pred)
           for (i = 0; i < COMP_INTER_CONTEXTS; i++)
             vp9_cond_prob_diff_update(&header_bc, &fc->comp_inter_prob[i],
-                                      cpi->comp_inter_count[i]);
+                                      cm->counts.comp_inter[i]);
       }
     }
 
     if (cm->reference_mode != COMPOUND_REFERENCE) {
       for (i = 0; i < REF_CONTEXTS; i++) {
         vp9_cond_prob_diff_update(&header_bc, &fc->single_ref_prob[i][0],
-                                  cpi->single_ref_count[i][0]);
+                                  cm->counts.single_ref[i][0]);
         vp9_cond_prob_diff_update(&header_bc, &fc->single_ref_prob[i][1],
-                                  cpi->single_ref_count[i][1]);
+                                  cm->counts.single_ref[i][1]);
       }
     }
 
     if (cm->reference_mode != SINGLE_REFERENCE)
       for (i = 0; i < REF_CONTEXTS; i++)
         vp9_cond_prob_diff_update(&header_bc, &fc->comp_ref_prob[i],
-                                  cpi->comp_ref_count[i]);
+                                  cm->counts.comp_ref[i]);
 
     for (i = 0; i < BLOCK_SIZE_GROUPS; ++i)
       prob_diff_update(vp9_intra_mode_tree, cm->fc.y_mode_prob[i],
diff --git a/vp9/encoder/vp9_encodeframe.c b/vp9/encoder/vp9_encodeframe.c
index 7af9a1f04b90ae24b33c9cfeaac1bcdfe0bb18b5..d49e3c1d1bbdcd50fc594d130a7c8a006124831c 100644
--- a/vp9/encoder/vp9_encodeframe.c
+++ b/vp9/encoder/vp9_encodeframe.c
@@ -762,17 +762,17 @@ static void update_stats(VP9_COMP *cpi) {
     // the reference frame counts used to work out probabilities.
     if (is_inter_block(mbmi) && !seg_ref_active) {
       if (cm->reference_mode == REFERENCE_MODE_SELECT)
-        cpi->comp_inter_count[vp9_get_reference_mode_context(cm, xd)]
+        cm->counts.comp_inter[vp9_get_reference_mode_context(cm, xd)]
                              [has_second_ref(mbmi)]++;
 
       if (has_second_ref(mbmi)) {
-        cpi->comp_ref_count[vp9_get_pred_context_comp_ref_p(cm, xd)]
+        cm->counts.comp_ref[vp9_get_pred_context_comp_ref_p(cm, xd)]
                            [mbmi->ref_frame[0] == GOLDEN_FRAME]++;
       } else {
-        cpi->single_ref_count[vp9_get_pred_context_single_ref_p1(xd)][0]
+        cm->counts.single_ref[vp9_get_pred_context_single_ref_p1(xd)][0]
                              [mbmi->ref_frame[0] != LAST_FRAME]++;
         if (mbmi->ref_frame[0] != LAST_FRAME)
-          cpi->single_ref_count[vp9_get_pred_context_single_ref_p2(xd)][1]
+          cm->counts.single_ref[vp9_get_pred_context_single_ref_p2(xd)][1]
                                [mbmi->ref_frame[0] != GOLDEN_FRAME]++;
       }
     }
@@ -2011,9 +2011,9 @@ static void init_encode_frame_mb_context(VP9_COMP *cpi) {
   vp9_zero(cm->counts.inter_mode);
   vp9_zero(cm->counts.partition);
   vp9_zero(cpi->intra_inter_count);
-  vp9_zero(cpi->comp_inter_count);
-  vp9_zero(cpi->single_ref_count);
-  vp9_zero(cpi->comp_ref_count);
+  vp9_zero(cm->counts.comp_inter);
+  vp9_zero(cm->counts.single_ref);
+  vp9_zero(cm->counts.comp_ref);
   vp9_zero(cm->counts.tx);
   vp9_zero(cm->counts.mbskip);
 
@@ -2431,16 +2431,16 @@ void vp9_encode_frame(VP9_COMP *cpi) {
       int comp_count_zero = 0;
 
       for (i = 0; i < COMP_INTER_CONTEXTS; i++) {
-        single_count_zero += cpi->comp_inter_count[i][0];
-        comp_count_zero += cpi->comp_inter_count[i][1];
+        single_count_zero += cm->counts.comp_inter[i][0];
+        comp_count_zero += cm->counts.comp_inter[i][1];
       }
 
       if (comp_count_zero == 0) {
         cpi->common.reference_mode = SINGLE_REFERENCE;
-        vp9_zero(cpi->comp_inter_count);
+        vp9_zero(cm->counts.comp_inter);
       } else if (single_count_zero == 0) {
         cpi->common.reference_mode = COMPOUND_REFERENCE;
-        vp9_zero(cpi->comp_inter_count);
+        vp9_zero(cm->counts.comp_inter);
       }
     }
 
diff --git a/vp9/encoder/vp9_onyx_if.c b/vp9/encoder/vp9_onyx_if.c
index e4ebdbafaf60eb2a9ae3e4e7e1f1fc59749b8867..7fbce13db4f24af3b9365982eee3685e672cbcd7 100644
--- a/vp9/encoder/vp9_onyx_if.c
+++ b/vp9/encoder/vp9_onyx_if.c
@@ -3089,9 +3089,6 @@ static void encode_frame_to_data_rate(VP9_COMP *cpi,
     vp9_copy(counts->y_mode, cpi->y_mode_count);
     vp9_copy(counts->uv_mode, cpi->y_uv_mode_count);
     vp9_copy(counts->intra_inter, cpi->intra_inter_count);
-    vp9_copy(counts->comp_inter, cpi->comp_inter_count);
-    vp9_copy(counts->single_ref, cpi->single_ref_count);
-    vp9_copy(counts->comp_ref, cpi->comp_ref_count);
     if (!cpi->common.error_resilient_mode &&
         !cpi->common.frame_parallel_decoding_mode) {
       vp9_adapt_mode_probs(&cpi->common);
diff --git a/vp9/encoder/vp9_onyx_int.h b/vp9/encoder/vp9_onyx_int.h
index 41f402bae2510de1e588929a5011e2524a129b75..3679c008a2e9411c08f4e68eb6d9c9175f3e9395 100644
--- a/vp9/encoder/vp9_onyx_int.h
+++ b/vp9/encoder/vp9_onyx_int.h
@@ -432,9 +432,6 @@ typedef struct VP9_COMP {
   int64_t rd_comp_pred_diff[REFERENCE_MODES];
   int64_t rd_prediction_type_threshes[4][REFERENCE_MODES];
   unsigned int intra_inter_count[INTRA_INTER_CONTEXTS][2];
-  unsigned int comp_inter_count[COMP_INTER_CONTEXTS][2];
-  unsigned int single_ref_count[REF_CONTEXTS][2][2];
-  unsigned int comp_ref_count[REF_CONTEXTS][2];
 
   int64_t rd_tx_select_diff[TX_MODES];
   // FIXME(rbultje) can this overflow?