From c3f5b2931b8c41a00378ba4eebde88bde7c1d477 Mon Sep 17 00:00:00 2001
From: "Ronald S. Bultje" <rbultje@google.com>
Date: Fri, 2 Mar 2012 11:18:24 -0800
Subject: [PATCH] Use per-MB compound intra prediction.

This gives a modest gain on derf overall, although at low bitrates the
cost is still too high, so this can be improved further.

Patch 2. Re-base and fix 80 column issues

Change-Id: Ida2f9fa3fe75370669f6a27b37108dc602231c63
---
 vp8/decoder/decodemv.c  | 12 ++++++------
 vp8/encoder/bitstream.c | 28 ++++++++++++----------------
 vp8/encoder/rdopt.c     | 41 +++++++++++++++++++++++++----------------
 3 files changed, 43 insertions(+), 38 deletions(-)

diff --git a/vp8/decoder/decodemv.c b/vp8/decoder/decodemv.c
index a70d1113b5..0a5436a72d 100644
--- a/vp8/decoder/decodemv.c
+++ b/vp8/decoder/decodemv.c
@@ -134,13 +134,13 @@ static void vp8_kfread_modes(VP8D_COMP *pbi,
     if ((m->mbmi.mode = y_mode) == B_PRED)
     {
         int i = 0;
+#if CONFIG_COMP_INTRA_PRED
+        int use_comp_pred = vp8_read(bc, 128);
+#endif
         do
         {
             const B_PREDICTION_MODE A = above_block_mode(m, i, mis);
             const B_PREDICTION_MODE L = left_block_mode(m, i);
-#if CONFIG_COMP_INTRA_PRED
-            int use_comp_pred = vp8_read(bc, 128);
-#endif
 
             m->bmi[i].as_mode.first =
                 (B_PREDICTION_MODE) vp8_read_bmode(
@@ -933,11 +933,11 @@ static void read_mb_modes_mv(VP8D_COMP *pbi, MODE_INFO *mi, MB_MODE_INFO *mbmi,
         if (mbmi->mode == B_PRED)
         {
             int j = 0;
-            do
-            {
 #if CONFIG_COMP_INTRA_PRED
-                int use_comp_pred = vp8_read(bc, 128);
+            int use_comp_pred = vp8_read(bc, 128);
 #endif
+            do
+            {
                 mi->bmi[j].as_mode.first = (B_PREDICTION_MODE)vp8_read_bmode(bc, pbi->common.fc.bmode_prob);
 #if CONFIG_COMP_INTRA_PRED
                 if (use_comp_pred)
diff --git a/vp8/encoder/bitstream.c b/vp8/encoder/bitstream.c
index 8e22926e14..58e34cdc32 100644
--- a/vp8/encoder/bitstream.c
+++ b/vp8/encoder/bitstream.c
@@ -755,16 +755,16 @@ static void pack_inter_mode_mvs(VP8_COMP *const cpi)
 
                     if (mode == B_PRED)
                     {
-                        int j = 0;
+                        int j = 0, uses_second = m->bmi[0].as_mode.second != (B_PREDICTION_MODE);
 
+                        vp8_write(w, uses_second, 128);
                         do {
 #if CONFIG_COMP_INTRA_PRED
                             int mode2 = m->bmi[j].as_mode.second;
-                            vp8_encode_bool(w, mode2 != (B_PREDICTION_MODE) (B_DC_PRED - 1), 128);
 #endif
                             write_bmode(w, m->bmi[j].as_mode.first, pc->fc.bmode_prob);
 #if CONFIG_COMP_INTRA_PRED
-                            if (mode2 != (B_PREDICTION_MODE) (B_DC_PRED - 1))
+                            if (uses_second)
                             {
                                 write_bmode(w, mode2, pc->fc.bmode_prob);
                             }
@@ -1124,16 +1124,16 @@ static void pack_inter_mode_mvs(VP8_COMP *const cpi)
 
                 if (mode == B_PRED)
                 {
-                    int j = 0;
+                    int j = 0, uses_second = m->bmi[0].as_mode.second != (B_PREDICTION_MODE) (B_DC_PRED - 1);
 
+                    vp8_write(w, uses_second, 128);
                     do {
 #if CONFIG_COMP_INTRA_PRED
                         B_PREDICTION_MODE mode2 = m->bmi[j].as_mode.second;
-                        vp8_write(w, mode2 != (B_PREDICTION_MODE) (B_DC_PRED - 1), 128);
 #endif
                         write_bmode(w, m->bmi[j].as_mode.first, pc->fc.bmode_prob);
 #if CONFIG_COMP_INTRA_PRED
-                        if (mode2 != (B_PREDICTION_MODE) (B_DC_PRED - 1))
+                        if (uses_second)
                         {
                             write_bmode(w, mode2, pc->fc.bmode_prob);
                         }
@@ -1396,8 +1396,9 @@ static void write_kfmodes(VP8_COMP *cpi)
 #endif
                 if (ym == B_PRED)
                 {
-                    int i = 0;
+                    int i = 0, uses_second = m->bmi[0].as_mode.second != (B_PREDICTION_MODE);
 
+                    vp8_write(bc, uses_second, 128);
                     do
                     {
                         const B_PREDICTION_MODE A = above_block_mode(m, i, mis);
@@ -1411,12 +1412,9 @@ static void write_kfmodes(VP8_COMP *cpi)
                         ++intra_mode_stats [A] [L] [bm];
 #endif
 
-#if CONFIG_COMP_INTRA_PRED
-                        vp8_write(bc, bm2 != (B_PREDICTION_MODE) (B_DC_PRED - 1), 128);
-#endif
                         write_bmode(bc, bm, c->kf_bmode_prob [A] [L]);
 #if CONFIG_COMP_INTRA_PRED
-                        if (bm2 != (B_PREDICTION_MODE) (B_DC_PRED - 1))
+                        if (uses_second)
                         {
                             write_bmode(bc, bm2, c->kf_bmode_prob [A] [L]);
                         }
@@ -1516,8 +1514,9 @@ static void write_kfmodes(VP8_COMP *cpi)
             if (ym == B_PRED)
             {
                 const int mis = c->mode_info_stride;
-                int i = 0;
+                int i = 0, uses_second = m->bmi[0].as_mode.second != (B_PREDICTION_MODE) (B_DC_PRED - 1);
 
+                vp8_write(bc, uses_second, 128);
                 do
                 {
                     const B_PREDICTION_MODE A = above_block_mode(m, i, mis);
@@ -1531,12 +1530,9 @@ static void write_kfmodes(VP8_COMP *cpi)
                     ++intra_mode_stats [A] [L] [bm];
 #endif
 
-#if CONFIG_COMP_INTRA_PRED
-                    vp8_write(bc, bm2 != (B_PREDICTION_MODE) (B_DC_PRED - 1), 128);
-#endif
                     write_bmode(bc, bm, c->kf_bmode_prob [A] [L]);
 #if CONFIG_COMP_INTRA_PRED
-                    if (bm2 != (B_PREDICTION_MODE) (B_DC_PRED - 1))
+                    if (uses_second)
                     {
                         write_bmode(bc, bm2, c->kf_bmode_prob [A] [L]);
                     }
diff --git a/vp8/encoder/rdopt.c b/vp8/encoder/rdopt.c
index fd64182f20..e239281ffa 100644
--- a/vp8/encoder/rdopt.c
+++ b/vp8/encoder/rdopt.c
@@ -837,7 +837,8 @@ static int rd_pick_intra4x4block(
 
     int *bestrate,
     int *bestratey,
-    int *bestdistortion)
+    int *bestdistortion,
+    int allow_comp)
 {
     B_PREDICTION_MODE mode;
 #if CONFIG_COMP_INTRA_PRED
@@ -860,7 +861,7 @@ static int rd_pick_intra4x4block(
     for (mode = B_DC_PRED; mode <= B_HU_PRED; mode++)
     {
 #if CONFIG_COMP_INTRA_PRED
-        for (mode2 = B_DC_PRED - 1; mode2 != B_HU_PRED + 1; mode2++)
+        for (mode2 = (allow_comp ? 0 : (B_DC_PRED - 1)); mode2 != (allow_comp ? (mode + 1) : 0); mode2++)
         {
 #endif
         int this_rd;
@@ -880,13 +881,11 @@ static int rd_pick_intra4x4block(
         RECON_INVOKE(&cpi->rtcd.common->recon, intra4x4_predict)
                      (b, mode, b->predictor);
 #if CONFIG_COMP_INTRA_PRED
-                rate += vp8_cost_bit(128, 0);
             }
             else
             {
                 RECON_INVOKE(&cpi->rtcd.common->recon, comp_intra4x4_predict)
                     (b, mode, mode2, b->predictor);
-                rate += vp8_cost_bit(128, 1);
                 rate += bmode_costs[mode2];
             }
 #endif
@@ -934,7 +933,8 @@ static int rd_pick_intra4x4block(
 }
 
 static int rd_pick_intra4x4mby_modes(VP8_COMP *cpi, MACROBLOCK *mb, int *Rate,
-                                     int *rate_y, int *Distortion, int best_rd)
+                                     int *rate_y, int *Distortion, int best_rd,
+                                     int allow_comp)
 {
     MACROBLOCKD *const xd = &mb->e_mbd;
     int i;
@@ -983,7 +983,7 @@ static int rd_pick_intra4x4mby_modes(VP8_COMP *cpi, MACROBLOCK *mb, int *Rate,
             &best_second_mode,
 #endif
             bmode_costs, ta + vp8_block2above[i],
-            tl + vp8_block2left[i], &r, &ry, &d);
+            tl + vp8_block2left[i], &r, &ry, &d, allow_comp);
 
         cost += r;
         distortion += d;
@@ -1001,7 +1001,7 @@ static int rd_pick_intra4x4mby_modes(VP8_COMP *cpi, MACROBLOCK *mb, int *Rate,
     if(total_rd >= (int64_t)best_rd)
         return INT_MAX;
 
-    *Rate = cost;
+    *Rate = cost + vp8_cost_bit(128, allow_comp);
     *rate_y += tot_rate_y;
     *Distortion = distortion;
 
@@ -2736,7 +2736,7 @@ void vp8_rd_pick_inter_mode(VP8_COMP *cpi, MACROBLOCK *x, int recon_yoffset, int
             int tmp_rd;
 
             // Note the rate value returned here includes the cost of coding the BPRED mode : x->mbmode_cost[x->e_mbd.frame_type][BPRED];
-            tmp_rd = rd_pick_intra4x4mby_modes(cpi, x, &rate, &rate_y, &distortion, best_yrd);
+            tmp_rd = rd_pick_intra4x4mby_modes(cpi, x, &rate, &rate_y, &distortion, best_yrd, 0);
             rate2 += rate;
             distortion2 += distortion;
 
@@ -3478,7 +3478,6 @@ void vp8_rd_pick_inter_mode(VP8_COMP *cpi, MACROBLOCK *x, int recon_yoffset, int
 
     if (best_mbmode.mode == I8X8_PRED)
     {
-        //printf("inter\n");
         set_i8x8_block_modes(x, mode8x8);
     }
 
@@ -3512,9 +3511,9 @@ void vp8_rd_pick_inter_mode(VP8_COMP *cpi, MACROBLOCK *x, int recon_yoffset, int
 void vp8_rd_pick_intra_mode(VP8_COMP *cpi, MACROBLOCK *x, int *rate_)
 {
     MACROBLOCKD *xd = &x->e_mbd;
-    int error4x4, error16x16;
-    int rate4x4, rate16x16 = 0, rateuv;
-    int dist4x4, dist16x16, distuv;
+    int error4x4, error16x16, error4x4d;
+    int rate4x4, rate16x16 = 0, rateuv, rate4x4d;
+    int dist4x4, dist16x16, distuv, dist4x4d;
     int rate;
     int rate4x4_tokenonly = 0;
     int rate16x16_tokenonly = 0;
@@ -3550,14 +3549,21 @@ void vp8_rd_pick_intra_mode(VP8_COMP *cpi, MACROBLOCK *x, int *rate_)
 
     error4x4 = rd_pick_intra4x4mby_modes(cpi, x,
                                          &rate4x4, &rate4x4_tokenonly,
-                                         &dist4x4, error16x16);
+                                         &dist4x4, error16x16, 0);
+    error4x4d = rd_pick_intra4x4mby_modes(cpi, x,
+                                         &rate4x4d, &rate4x4_tokenonly,
+                                         &dist4x4d, error16x16, 1);
 
     if(error8x8> error16x16)
     {
         if (error4x4 < error16x16)
         {
+            rate += (error4x4d < error4x4) ? rate4x4d : rate4x4;
+            if (error4x4d >= error4x4) // FIXME save original modes etc.
+                error4x4 = rd_pick_intra4x4mby_modes(cpi, x, &rate4x4,
+                                                     &rate4x4_tokenonly,
+                                                     &dist4x4, error16x16, 0);
             x->e_mbd.mode_info_context->mbmi.mode = B_PRED;
-            rate += rate4x4;
         }
         else
         {
@@ -3570,14 +3576,17 @@ void vp8_rd_pick_intra_mode(VP8_COMP *cpi, MACROBLOCK *x, int *rate_)
     {
         if (error4x4 < error8x8)
         {
+            rate += (error4x4d < error4x4) ? rate4x4d : rate4x4;
+            if (error4x4d >= error4x4) // FIXME save original modes etc.
+                error4x4 = rd_pick_intra4x4mby_modes(cpi, x, &rate4x4,
+                                                     &rate4x4_tokenonly,
+                                                     &dist4x4, error16x16, 0);
             x->e_mbd.mode_info_context->mbmi.mode = B_PRED;
-            rate += rate4x4;
         }
         else
         {
 
             x->e_mbd.mode_info_context->mbmi.mode = I8X8_PRED;
-            //printf("intra\n");
             set_i8x8_block_modes(x, mode8x8);
             rate += rate8x8;
         }
-- 
GitLab