fix superframe index marker masks

The superframe index marker byte carries data in the lower 5 bits. Only the
upper 3 should be used as part of the mask to detect it. By masking with
0xf0, the previous code was incorrect for frames over 65k bytes.

Change-Id: I6248889f5af227457f359a56b2348ef6db87a3b4

test/superframe_test.cc

@@ -54,7 +54,7 @@ class SuperframeTest : public ::libvpx_test::EncoderTest,
     const int frames = (marker & 0x7) + 1;
     const int mag = ((marker >> 3) & 3) + 1;
     const unsigned int index_sz = 2 + mag  * frames;
-    if ((marker & 0xf0) == 0xc0 &&
+    if ((marker & 0xe0) == 0xc0 &&
         pkt->data.frame.sz >= index_sz &&
         buffer[pkt->data.frame.sz - index_sz] == marker) {
       // frame is a superframe. strip off the index.

vp9/vp9_dx_iface.c

@@ -435,7 +435,7 @@ static void parse_superframe_index(const uint8_t *data,
   marker = data[data_sz - 1];
   *count = 0;
 
-  if ((marker & 0xf0) == 0xc0) {
+  if ((marker & 0xe0) == 0xc0) {
     const int frames = (marker & 0x7) + 1;
     const int mag = ((marker >> 3) & 3) + 1;
     const int index_sz = 2 + mag  * frames;
@@ -473,7 +473,7 @@ static vpx_codec_err_t vp9_decode(vpx_codec_alg_priv_t  *ctx,
 
   do {
     // Skip over the superframe index, if present
-    if (data_sz && (*data_start & 0xf0) == 0xc0) {
+    if (data_sz && (*data_start & 0xe0) == 0xc0) {
       const uint8_t marker = *data_start;
       const int frames = (marker & 0x7) + 1;
       const int mag = ((marker >> 3) & 3) + 1;