Merge "vp9 encoder: fix integer overflows" into main
diff --git a/test/realtime_test.cc b/test/realtime_test.cc
index ab2080a..107f2e2 100644
--- a/test/realtime_test.cc
+++ b/test/realtime_test.cc
@@ -85,9 +85,7 @@
TEST_P(RealtimeTest, IntegerOverflowLarge) {
if (IsVP9()) {
- GTEST_SKIP() << "TODO(https://crbug.com/webm/1750): Enable this test after "
- "undefined sanitizer warnings are fixed.";
- // TestIntegerOverflow(16384, 16384);
+ TestIntegerOverflow(16384, 16384);
} else {
GTEST_SKIP()
<< "TODO(https://crbug.com/webm/1748,https://crbug.com/webm/1751):"
diff --git a/vp9/encoder/vp9_bitstream.c b/vp9/encoder/vp9_bitstream.c
index d4ac4ff..99cc2ee 100644
--- a/vp9/encoder/vp9_bitstream.c
+++ b/vp9/encoder/vp9_bitstream.c
@@ -563,7 +563,7 @@
for (t = 0; t < entropy_nodes_update; ++t) {
vpx_prob newp = new_coef_probs[i][j][k][l][t];
const vpx_prob oldp = old_coef_probs[i][j][k][l][t];
- int s;
+ int64_t s;
int u = 0;
if (t == PIVOT_NODE)
s = vp9_prob_diff_update_savings_search_model(
@@ -600,7 +600,7 @@
vpx_prob newp = new_coef_probs[i][j][k][l][t];
vpx_prob *oldp = old_coef_probs[i][j][k][l] + t;
const vpx_prob upd = DIFF_UPDATE_PROB;
- int s;
+ int64_t s;
int u = 0;
if (t == PIVOT_NODE)
s = vp9_prob_diff_update_savings_search_model(
@@ -636,7 +636,7 @@
for (t = 0; t < entropy_nodes_update; ++t) {
vpx_prob newp = new_coef_probs[i][j][k][l][t];
vpx_prob *oldp = old_coef_probs[i][j][k][l] + t;
- int s;
+ int64_t s;
int u = 0;
if (t == PIVOT_NODE) {
diff --git a/vp9/encoder/vp9_cost.h b/vp9/encoder/vp9_cost.h
index 638d72a..ee0033f 100644
--- a/vp9/encoder/vp9_cost.h
+++ b/vp9/encoder/vp9_cost.h
@@ -29,9 +29,8 @@
#define vp9_cost_bit(prob, bit) vp9_cost_zero((bit) ? 256 - (prob) : (prob))
-static INLINE unsigned int cost_branch256(const unsigned int ct[2],
- vpx_prob p) {
- return ct[0] * vp9_cost_zero(p) + ct[1] * vp9_cost_one(p);
+static INLINE uint64_t cost_branch256(const unsigned int ct[2], vpx_prob p) {
+ return (uint64_t)ct[0] * vp9_cost_zero(p) + (uint64_t)ct[1] * vp9_cost_one(p);
}
static INLINE int treed_cost(vpx_tree tree, const vpx_prob *probs, int bits,
diff --git a/vp9/encoder/vp9_subexp.c b/vp9/encoder/vp9_subexp.c
index 19bbd53..661294b 100644
--- a/vp9/encoder/vp9_subexp.c
+++ b/vp9/encoder/vp9_subexp.c
@@ -114,19 +114,20 @@
encode_term_subexp(w, delp);
}
-int vp9_prob_diff_update_savings_search(const unsigned int *ct, vpx_prob oldp,
- vpx_prob *bestp, vpx_prob upd) {
- const int old_b = cost_branch256(ct, oldp);
- int bestsavings = 0;
+int64_t vp9_prob_diff_update_savings_search(const unsigned int *ct,
+ vpx_prob oldp, vpx_prob *bestp,
+ vpx_prob upd) {
+ const int64_t old_b = cost_branch256(ct, oldp);
+ int64_t bestsavings = 0;
vpx_prob newp, bestnewp = oldp;
const int step = *bestp > oldp ? -1 : 1;
const int upd_cost = vp9_cost_one(upd) - vp9_cost_zero(upd);
if (old_b > upd_cost + (MIN_DELP_BITS << VP9_PROB_COST_SHIFT)) {
for (newp = *bestp; newp != oldp; newp += step) {
- const int new_b = cost_branch256(ct, newp);
- const int update_b = prob_diff_update_cost(newp, oldp) + upd_cost;
- const int savings = old_b - new_b - update_b;
+ const int64_t new_b = cost_branch256(ct, newp);
+ const int64_t update_b = prob_diff_update_cost(newp, oldp) + upd_cost;
+ const int64_t savings = old_b - new_b - update_b;
if (savings > bestsavings) {
bestsavings = savings;
bestnewp = newp;
@@ -137,15 +138,15 @@
return bestsavings;
}
-int vp9_prob_diff_update_savings_search_model(const unsigned int *ct,
- const vpx_prob oldp,
- vpx_prob *bestp, vpx_prob upd,
- int stepsize) {
- int i, old_b, new_b, update_b, savings, bestsavings;
- int newp;
- const int step_sign = *bestp > oldp ? -1 : 1;
- const int step = stepsize * step_sign;
- const int upd_cost = vp9_cost_one(upd) - vp9_cost_zero(upd);
+int64_t vp9_prob_diff_update_savings_search_model(const unsigned int *ct,
+ const vpx_prob oldp,
+ vpx_prob *bestp, vpx_prob upd,
+ int stepsize) {
+ int64_t i, old_b, new_b, update_b, savings, bestsavings;
+ int64_t newp;
+ const int64_t step_sign = *bestp > oldp ? -1 : 1;
+ const int64_t step = stepsize * step_sign;
+ const int64_t upd_cost = vp9_cost_one(upd) - vp9_cost_zero(upd);
const vpx_prob *newplist, *oldplist;
vpx_prob bestnewp;
oldplist = vp9_pareto8_full[oldp - 1];
@@ -182,7 +183,7 @@
const unsigned int ct[2]) {
const vpx_prob upd = DIFF_UPDATE_PROB;
vpx_prob newp = get_binary_prob(ct[0], ct[1]);
- const int savings =
+ const int64_t savings =
vp9_prob_diff_update_savings_search(ct, *oldp, &newp, upd);
assert(newp >= 1);
if (savings > 0) {
diff --git a/vp9/encoder/vp9_subexp.h b/vp9/encoder/vp9_subexp.h
index f0d544b..2d016d2 100644
--- a/vp9/encoder/vp9_subexp.h
+++ b/vp9/encoder/vp9_subexp.h
@@ -25,13 +25,14 @@
void vp9_cond_prob_diff_update(struct vpx_writer *w, vpx_prob *oldp,
const unsigned int ct[2]);
-int vp9_prob_diff_update_savings_search(const unsigned int *ct, vpx_prob oldp,
- vpx_prob *bestp, vpx_prob upd);
+int64_t vp9_prob_diff_update_savings_search(const unsigned int *ct,
+ vpx_prob oldp, vpx_prob *bestp,
+ vpx_prob upd);
-int vp9_prob_diff_update_savings_search_model(const unsigned int *ct,
- const vpx_prob oldp,
- vpx_prob *bestp, vpx_prob upd,
- int stepsize);
+int64_t vp9_prob_diff_update_savings_search_model(const unsigned int *ct,
+ const vpx_prob oldp,
+ vpx_prob *bestp, vpx_prob upd,
+ int stepsize);
#ifdef __cplusplus
} // extern "C"
