From bd9cd9a1859aa464b3092f2023b3a4040166572d Mon Sep 17 00:00:00 2001
From: John Koleszar <jkoleszar@google.com>
Date: Tue, 12 Mar 2013 19:03:05 -0700
Subject: [PATCH] fix superframe index marker masks

The superframe index marker byte carries data in the lower 5 bits. Only the
upper 3 should be used as part of the mask to detect it. By masking with
0xf0, the previous code was incorrect for frames over 65k bytes.

Change-Id: I6248889f5af227457f359a56b2348ef6db87a3b4
---
 test/superframe_test.cc | 2 +-
 vp9/vp9_dx_iface.c      | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/test/superframe_test.cc b/test/superframe_test.cc
index 39190ea78a..52faddb43e 100644
--- a/test/superframe_test.cc
+++ b/test/superframe_test.cc
@@ -54,7 +54,7 @@ class SuperframeTest : public ::libvpx_test::EncoderTest,
     const int frames = (marker & 0x7) + 1;
     const int mag = ((marker >> 3) & 3) + 1;
     const unsigned int index_sz = 2 + mag  * frames;
-    if ((marker & 0xf0) == 0xc0 &&
+    if ((marker & 0xe0) == 0xc0 &&
         pkt->data.frame.sz >= index_sz &&
         buffer[pkt->data.frame.sz - index_sz] == marker) {
       // frame is a superframe. strip off the index.
diff --git a/vp9/vp9_dx_iface.c b/vp9/vp9_dx_iface.c
index 1bda1703f1..eabdb85564 100644
--- a/vp9/vp9_dx_iface.c
+++ b/vp9/vp9_dx_iface.c
@@ -435,7 +435,7 @@ static void parse_superframe_index(const uint8_t *data,
   marker = data[data_sz - 1];
   *count = 0;
 
-  if ((marker & 0xf0) == 0xc0) {
+  if ((marker & 0xe0) == 0xc0) {
     const int frames = (marker & 0x7) + 1;
     const int mag = ((marker >> 3) & 3) + 1;
     const int index_sz = 2 + mag  * frames;
@@ -473,7 +473,7 @@ static vpx_codec_err_t vp9_decode(vpx_codec_alg_priv_t  *ctx,
 
   do {
     // Skip over the superframe index, if present
-    if (data_sz && (*data_start & 0xf0) == 0xc0) {
+    if (data_sz && (*data_start & 0xe0) == 0xc0) {
       const uint8_t marker = *data_start;
       const int frames = (marker & 0x7) + 1;
       const int mag = ((marker >> 3) & 3) + 1;
-- 
GitLab