Commit 6a9ed8d2 authored by Jingning Han's avatar Jingning Han

Fix forward transform bit range limits

Change-Id: I13c0ecff8c58a0571d9de4bc5fbbebe72533ccdb
parent 8198be11
......@@ -204,7 +204,7 @@ static void fdct4(const tran_low_t *input, tran_low_t *output) {
tran_low_t step[4];
// stage 0
range_check(input, 4, 11);
range_check(input, 4, 14);
// stage 1
output[0] = input[0] + input[3];
......@@ -212,7 +212,7 @@ static void fdct4(const tran_low_t *input, tran_low_t *output) {
output[2] = input[1] - input[2];
output[3] = input[0] - input[3];
range_check(output, 4, 12);
range_check(output, 4, 15);
// stage 2
temp = output[0] * cospi_16_64 + output[1] * cospi_16_64;
......@@ -224,7 +224,7 @@ static void fdct4(const tran_low_t *input, tran_low_t *output) {
temp = output[3] * cospi_24_64 + output[2] * -cospi_8_64;
step[3] = (tran_low_t)fdct_round_shift(temp);
range_check(step, 4, 13);
range_check(step, 4, 16);
// stage 3
output[0] = step[0];
......@@ -232,7 +232,7 @@ static void fdct4(const tran_low_t *input, tran_low_t *output) {
output[2] = step[1];
output[3] = step[3];
range_check(output, 4, 13);
range_check(output, 4, 16);
}
static void fdct8(const tran_low_t *input, tran_low_t *output) {
......@@ -240,7 +240,7 @@ static void fdct8(const tran_low_t *input, tran_low_t *output) {
tran_low_t step[8];
// stage 0
range_check(input, 8, 12);
range_check(input, 8, 13);
// stage 1
output[0] = input[0] + input[7];
......@@ -252,7 +252,7 @@ static void fdct8(const tran_low_t *input, tran_low_t *output) {
output[6] = input[1] - input[6];
output[7] = input[0] - input[7];
range_check(output, 8, 13);
range_check(output, 8, 14);
// stage 2
step[0] = output[0] + output[3];
......@@ -266,7 +266,7 @@ static void fdct8(const tran_low_t *input, tran_low_t *output) {
step[6] = (tran_low_t)fdct_round_shift(temp);
step[7] = output[7];
range_check(step, 8, 14);
range_check(step, 8, 15);
// stage 3
temp = step[0] * cospi_16_64 + step[1] * cospi_16_64;
......@@ -282,7 +282,7 @@ static void fdct8(const tran_low_t *input, tran_low_t *output) {
output[6] = step[7] - step[6];
output[7] = step[7] + step[6];
range_check(output, 8, 14);
range_check(output, 8, 16);
// stage 4
step[0] = output[0];
......@@ -298,7 +298,7 @@ static void fdct8(const tran_low_t *input, tran_low_t *output) {
temp = output[7] * cospi_28_64 + output[4] * -cospi_4_64;
step[7] = (tran_low_t)fdct_round_shift(temp);
range_check(step, 8, 14);
range_check(step, 8, 16);
// stage 5
output[0] = step[0];
......@@ -310,7 +310,7 @@ static void fdct8(const tran_low_t *input, tran_low_t *output) {
output[6] = step[3];
output[7] = step[7];
range_check(output, 8, 14);
range_check(output, 8, 16);
}
static void fdct16(const tran_low_t *input, tran_low_t *output) {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment