vpnext: pick loop filter segment by segment

Picks a per segment loopfilter.  Adapts the algorithm to search for
a loopfilter value for each separate segment.  Further todo fix the
bias

Improvements .06 % ov psnr, .11% ssim
http://www.corp.google.com/~jimbankoski/no_crawl/segmentedpicklpf.html

Change-Id: Ic6a571c16fcd6ec0139f4de1f8061f87c6515a10

vp8/common/loopfilter.c

@@ -580,6 +580,138 @@ void vp8_loop_filter_frame_yonly
 
 }
 
+// TODO: Multiple copies of loop filtering code should be pruned and
+// cut down.   This just adds yet another so that I can do an if
+// on segment.
+void vp8_loop_filter_frame_segment(VP8_COMMON *cm, MACROBLOCKD *xd,
+                                   int default_filt_lvl, int segment)
+{
+    YV12_BUFFER_CONFIG *post = cm->frame_to_show;
+
+    unsigned char *y_ptr;
+    int mb_row;
+    int mb_col;
+
+    loop_filter_info_n *lfi_n = &cm->lf_info;
+    loop_filter_info lfi;
+
+    int filter_level;
+    FRAME_TYPE frame_type = cm->frame_type;
+
+    /* Point at base of Mb MODE_INFO list */
+    const MODE_INFO *mode_info_context = cm->mi;
+
+#if 0
+    if(default_filt_lvl == 0) /* no filter applied */
+    return;
+#endif
+
+    /* Initialize the loop filter for this frame. */
+    vp8_loop_filter_frame_init(cm, xd, default_filt_lvl);
+
+    /* Set up the buffer pointers */
+    y_ptr = post->y_buffer;
+
+    /* vp8_filter each macro block */
+    for (mb_row = 0; mb_row < cm->mb_rows; mb_row++)
+    {
+        for (mb_col = 0; mb_col < cm->mb_cols; mb_col++)
+        {
+            int skip_lf = (mode_info_context->mbmi.mode != B_PRED
+                           && mode_info_context->mbmi.mode != I8X8_PRED
+                           && mode_info_context->mbmi.mode != SPLITMV
+                           && mode_info_context->mbmi.mb_skip_coeff);
+
+            const int mode_index = lfi_n->mode_lf_lut[mode_info_context->mbmi
+                    .mode];
+            const int seg = mode_info_context->mbmi.segment_id;
+            const int ref_frame = mode_info_context->mbmi.ref_frame;
+
+            filter_level = lfi_n->lvl[seg][ref_frame][mode_index];
+
+            // check if this mb has filtering applied
+            //    and then whether it is the right segment or
+            //    if not whether the passed in segment is 0 and this
+            //    segment has no alt lf
+
+            // TODO: Make this work for when segment 0 has the alt lv enabled
+            if (filter_level
+                && (seg == segment
+                    || (!segfeature_active(xd, seg, SEG_LVL_ALT_LF)
+                        && segment == 0)))
+            {
+                if (cm->filter_type == NORMAL_LOOPFILTER)
+                {
+                    const int hev_index =
+                            lfi_n->hev_thr_lut[frame_type][filter_level];
+                    lfi.mblim = lfi_n->mblim[filter_level];
+                    lfi.blim = lfi_n->blim[filter_level];
+                    lfi.lim = lfi_n->lim[filter_level];
+                    lfi.hev_thr = lfi_n->hev_thr[hev_index];
+
+                    if (mb_col > 0)
+#if CONFIG_NEWLPF
+                        vp8_loop_filter_mbv_c(y_ptr, 0, 0, post->y_stride, 0,
+                                              &lfi);
+#else
+                    LF_INVOKE(&cm->rtcd.loopfilter, normal_mb_v)
+                    (y_ptr, 0, 0, post->y_stride, 0, &lfi);
+#endif
+
+                    if (!skip_lf)
+                        LF_INVOKE(&cm->rtcd.loopfilter, normal_b_v)(
+                                y_ptr, 0, 0, post->y_stride, 0, &lfi);
+
+                    /* don't apply across umv border */
+                    if (mb_row > 0)
+#if CONFIG_NEWLPF
+                        vp8_loop_filter_mbh_c(y_ptr, 0, 0, post->y_stride, 0,
+                                              &lfi);
+#else
+                    LF_INVOKE(&cm->rtcd.loopfilter, normal_mb_h)
+                    (y_ptr, 0, 0, post->y_stride, 0, &lfi);
+#endif
+                    if (!skip_lf)
+                        LF_INVOKE(&cm->rtcd.loopfilter, normal_b_h)(
+                                y_ptr, 0, 0, post->y_stride, 0, &lfi);
+                }
+                else
+                {
+                    if (mb_col > 0)
+                        LF_INVOKE(&cm->rtcd.loopfilter, simple_mb_v)(
+                                y_ptr, post->y_stride,
+                                lfi_n->mblim[filter_level]);
+
+                    if (!skip_lf)
+                        LF_INVOKE(&cm->rtcd.loopfilter, simple_b_v)(
+                                y_ptr, post->y_stride,
+                                lfi_n->blim[filter_level]);
+
+                    /* don't apply across umv border */
+                    if (mb_row > 0)
+                        LF_INVOKE(&cm->rtcd.loopfilter, simple_mb_h)(
+                                y_ptr, post->y_stride,
+                                lfi_n->mblim[filter_level]);
+
+                    if (!skip_lf)
+                        LF_INVOKE(&cm->rtcd.loopfilter, simple_b_h)(
+                                y_ptr, post->y_stride,
+                                lfi_n->blim[filter_level]);
+                }
+            }
+
+            y_ptr += 16;
+            mode_info_context++; /* step to next MB */
+
+        }
+
+        y_ptr += post->y_stride * 16 - post->y_width;
+        mode_info_context++; /* Skip border mb */
+    }
+
+}
+
+
 void vp8_loop_filter_partial_frame
 (
     VP8_COMMON *cm,

vp8/encoder/picklpf.c

@@ -36,6 +36,15 @@ extern void
 (*vp8_yv12_copy_partial_frame_ptr)(YV12_BUFFER_CONFIG *src_ybc,
                                    YV12_BUFFER_CONFIG *dst_ybc,
                                    int Fraction);
+
+extern void vp8_loop_filter_frame_segment
+(
+    VP8_COMMON *cm,
+    MACROBLOCKD *xd,
+    int default_filt_lvl,
+    int segment
+);
+
 void
 vp8_yv12_copy_partial_frame(YV12_BUFFER_CONFIG *src_ybc, YV12_BUFFER_CONFIG *dst_ybc, int Fraction)
 {
@@ -63,7 +72,6 @@ vp8_yv12_copy_partial_frame(YV12_BUFFER_CONFIG *src_ybc, YV12_BUFFER_CONFIG *dst
 
     vpx_memcpy(dst_y, src_y, ystride *(linestocopy + 16));
 }
-
 static int vp8_calc_partial_ssl_err(YV12_BUFFER_CONFIG *source, YV12_BUFFER_CONFIG *dest, int Fraction, const vp8_variance_rtcd_vtable_t *rtcd)
 {
     int i, j;
@@ -261,7 +269,8 @@ void vp8cx_set_alt_lf_level(VP8_COMP *cpi, int filt_val)
 {
 }
 
-void vp8cx_pick_filter_level(YV12_BUFFER_CONFIG *sd, VP8_COMP *cpi)
+
+void vp8cx_pick_filter_level_sg(YV12_BUFFER_CONFIG *sd, VP8_COMP *cpi, int segment)
 {
     VP8_COMMON *cm = &cpi->common;
 
@@ -315,7 +324,7 @@ void vp8cx_pick_filter_level(YV12_BUFFER_CONFIG *sd, VP8_COMP *cpi)
 
     // Get baseline error score
     vp8cx_set_alt_lf_level(cpi, filt_mid);
-    vp8_loop_filter_frame_yonly(cm, &cpi->mb.e_mbd, filt_mid);
+    vp8_loop_filter_frame_segment(cm, &cpi->mb.e_mbd, filt_mid,segment);
 
     best_err = vp8_calc_ss_err(sd, cm->frame_to_show, IF_RTCD(&cpi->rtcd.variance));
     filt_best = filt_mid;
@@ -353,7 +362,7 @@ void vp8cx_pick_filter_level(YV12_BUFFER_CONFIG *sd, VP8_COMP *cpi)
         {
             // Get Low filter error score
             vp8cx_set_alt_lf_level(cpi, filt_low);
-            vp8_loop_filter_frame_yonly(cm, &cpi->mb.e_mbd, filt_low);
+            vp8_loop_filter_frame_segment(cm, &cpi->mb.e_mbd, filt_low, segment);
 
             filt_err = vp8_calc_ss_err(sd, cm->frame_to_show, IF_RTCD(&cpi->rtcd.variance));
 
@@ -390,7 +399,7 @@ void vp8cx_pick_filter_level(YV12_BUFFER_CONFIG *sd, VP8_COMP *cpi)
         if ((filt_direction >= 0) && (filt_high != filt_mid))
         {
             vp8cx_set_alt_lf_level(cpi, filt_high);
-            vp8_loop_filter_frame_yonly(cm, &cpi->mb.e_mbd, filt_high);
+            vp8_loop_filter_frame_segment(cm, &cpi->mb.e_mbd, filt_high, segment);
 
             filt_err = vp8_calc_ss_err(sd, cm->frame_to_show, IF_RTCD(&cpi->rtcd.variance));
 
@@ -435,3 +444,44 @@ void vp8cx_pick_filter_level(YV12_BUFFER_CONFIG *sd, VP8_COMP *cpi)
 
     cm->filter_level = filt_best;
 }
+
+void vp8cx_pick_filter_level(YV12_BUFFER_CONFIG *sd, VP8_COMP *cpi)
+{
+    VP8_COMMON *oci = &cpi->common;
+    MODE_INFO *mi = oci->mi;
+    int filt_lev[2];
+    int i, j;
+    MACROBLOCKD * const xd = &cpi->mb.e_mbd;
+    int max_seg;
+    int mb_index = 0;
+
+    // pick the loop filter for each segment after segment 0
+    for (i = 1; i < MAX_MB_SEGMENTS; i++)
+    {
+        // if the segment loop filter is active
+        if (segfeature_active(xd, i, SEG_LVL_ALT_LF))
+        {
+            set_segdata(xd, i, SEG_LVL_ALT_LF, 0);
+            vp8cx_pick_filter_level_sg(sd, cpi, i);
+            filt_lev[i] = oci->filter_level;
+        }
+    }
+
+    // do the 0 segment ( this filter also picks the filter value for all
+    // the not enabled features )
+
+    // TODO : Fix the code if segment 0 is the one with seg_lvl_alt_lf on
+    // right now assumes segment 0 gets base loop filter and the rest are
+    // deltas off of segment 0.
+    set_segdata(xd, 0, SEG_LVL_ALT_LF, 0);
+    vp8cx_pick_filter_level_sg(sd, cpi, 0);
+    filt_lev[0] = oci->filter_level;
+
+    // convert the best filter level for the mbs of the segment to
+    // a delta from 0
+    for (i = 1; i < MAX_MB_SEGMENTS; i++)
+        if (segfeature_active(xd, i, SEG_LVL_ALT_LF))
+        {
+            set_segdata(xd, i, SEG_LVL_ALT_LF, filt_lev[i] - filt_lev[0]);
+        }
+}