Commit cc472311 authored by Jerome Jiang's avatar Jerome Jiang

vp9: Reduce stack usage of choose_partioning.

Change type of sum_square_error from int64_t to uint32_t.
Change type of sum_error from int64_t to int32_t.

This reduces the stack usage from ~131K to ~87K.

BUG=b/68362457

Change-Id: I147d7c7b226bceb4f0817bb86848e1fa9d9ac149
parent 401e6d48
...@@ -287,8 +287,12 @@ static void set_block_size(VP9_COMP *const cpi, MACROBLOCK *const x, ...@@ -287,8 +287,12 @@ static void set_block_size(VP9_COMP *const cpi, MACROBLOCK *const x,
} }
typedef struct { typedef struct {
int64_t sum_square_error; // This struct is used for computing variance in choose_partitioning(), where
int64_t sum_error; // the max number of samples within a superblock is 16x16 (with 4x4 avg). Even
// in high bitdepth, uint32_t is enough for sum_square_error (2^12 * 2^12 * 16
// * 16 = 2^32).
uint32_t sum_square_error;
int32_t sum_error;
int log2_count; int log2_count;
int variance; int variance;
} var; } var;
...@@ -381,7 +385,7 @@ static void tree_to_node(void *data, BLOCK_SIZE bsize, variance_node *node) { ...@@ -381,7 +385,7 @@ static void tree_to_node(void *data, BLOCK_SIZE bsize, variance_node *node) {
} }
// Set variance values given sum square error, sum error, count. // Set variance values given sum square error, sum error, count.
static void fill_variance(int64_t s2, int64_t s, int c, var *v) { static void fill_variance(uint32_t s2, int32_t s, int c, var *v) {
v->sum_square_error = s2; v->sum_square_error = s2;
v->sum_error = s; v->sum_error = s;
v->log2_count = c; v->log2_count = c;
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment