Google Git Google Git
Sign in
chromium/external/github.com/WebAssembly/binaryen/refs/heads/make-shared-obj-funcs-only/./src/passes/OptimizeInstructions.cpp
blob: b862dd9e45cdec131105c3d2d5275e49ac90cf1c [file] [edit]
/*
* Copyright 2016 WebAssembly Community Group participants
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
//
// Optimize combinations of instructions
//
#include <algorithm>
#include <cmath>
#include <type_traits>
#include <ir/abstract.h>
#include <ir/bits.h>
#include <ir/boolean.h>
#include <ir/branch-hints.h>
#include <ir/cost.h>
#include <ir/drop.h>
#include <ir/effects.h>
#include <ir/eh-utils.h>
#include <ir/find_all.h>
#include <ir/gc-type-utils.h>
#include <ir/iteration.h>
#include <ir/literal-utils.h>
#include <ir/load-utils.h>
#include <ir/localize.h>
#include <ir/manipulation.h>
#include <ir/match.h>
#include <ir/ordering.h>
#include <ir/properties.h>
#include <ir/type-updating.h>
#include <ir/utils.h>
#include <pass.h>
#include <support/stdckdint.h>
#include <support/threads.h>
#include <wasm.h>
#include "call-utils.h"
#include "support/utilities.h"
#include "wasm-type.h"
// TODO: Use the new sign-extension opcodes where appropriate. This needs to be
// conditionalized on the availability of atomics.
namespace wasm {
static Index getBitsForType(Type type) {
if (!type.isNumber()) {
return -1;
}
return type.getByteSize() * 8;
}
static bool isSignedOp(BinaryOp op) {
switch (op) {
case LtSInt32:
case LeSInt32:
case GtSInt32:
case GeSInt32:
case LtSInt64:
case LeSInt64:
case GtSInt64:
case GeSInt64:
return true;
default:
return false;
}
}
// Useful information about locals
struct LocalInfo {
static const Index kUnknown = Index(-1);
Index maxBits = -1;
Index signExtBits = 0;
};
struct LocalScanner : PostWalker<LocalScanner> {
std::vector<LocalInfo>& localInfo;
const PassOptions& passOptions;
LocalScanner(std::vector<LocalInfo>& localInfo,
const PassOptions& passOptions)
: localInfo(localInfo), passOptions(passOptions) {}
void doWalkFunction(Function* func) {
// prepare
localInfo.resize(func->getNumLocals());
for (Index i = 0; i < func->getNumLocals(); i++) {
auto& info = localInfo[i];
if (func->isParam(i)) {
info.maxBits = getBitsForType(func->getLocalType(i)); // worst-case
info.signExtBits = LocalInfo::kUnknown; // we will never know anything
} else {
info.maxBits = info.signExtBits = 0; // we are open to learning
}
}
// walk
PostWalker<LocalScanner>::doWalkFunction(func);
// finalize
for (Index i = 0; i < func->getNumLocals(); i++) {
auto& info = localInfo[i];
if (info.signExtBits == LocalInfo::kUnknown) {
info.signExtBits = 0;
}
}
}
void visitLocalSet(LocalSet* curr) {
auto* func = getFunction();
if (func->isParam(curr->index)) {
return;
}
auto type = getFunction()->getLocalType(curr->index);
if (type != Type::i32 && type != Type::i64) {
return;
}
// an integer var, worth processing
auto* value =
Properties::getFallthrough(curr->value, passOptions, *getModule());
auto& info = localInfo[curr->index];
info.maxBits = std::max(info.maxBits, Bits::getMaxBits(value, this));
info.maxBits = std::min(info.maxBits, getBitsForType(type));
auto signExtBits = LocalInfo::kUnknown;
if (Properties::getSignExtValue(value)) {
signExtBits = Properties::getSignExtBits(value);
} else if (auto* load = value->dynCast<Load>()) {
if (LoadUtils::isSignRelevant(load) && load->signed_) {
signExtBits = load->bytes * 8;
}
}
if (info.signExtBits == 0) {
info.signExtBits = signExtBits; // first info we see
} else if (info.signExtBits != signExtBits) {
// contradictory information, give up
info.signExtBits = LocalInfo::kUnknown;
}
}
// define this for the templated getMaxBits method. we know nothing here yet
// about locals, so return the maxes
Index getMaxBitsForLocal(LocalGet* get) { return getBitsForType(get->type); }
};
namespace {
// perform some final optimizations
struct FinalOptimizer : public PostWalker<FinalOptimizer> {
const PassOptions& passOptions;
FinalOptimizer(const PassOptions& passOptions) : passOptions(passOptions) {}
void visitBinary(Binary* curr) {
if (auto* replacement = optimize(curr)) {
replaceCurrent(replacement);
}
}
Binary* optimize(Binary* curr) {
using namespace Abstract;
using namespace Match;
{
Const* c;
if (matches(curr, binary(Add, any(), ival(&c)))) {
// normalize x + (-C) ==> x - C
if (c->value.isNegative()) {
c->value = c->value.neg();
curr->op = Abstract::getBinary(c->type, Sub);
}
// Wasm binary encoding uses signed LEBs, which slightly favor negative
// numbers: -64 is more efficient than +64 etc., as well as other powers
// of two 7 bits etc. higher. we therefore prefer x - -64 over x + 64.
// in theory we could just prefer negative numbers over positive, but
// that can have bad effects on gzip compression (as it would mean more
// subtractions than the more common additions).
int64_t value = c->value.getInteger();
if (value == 0x40LL || value == 0x2000LL || value == 0x100000LL ||
value == 0x8000000LL || value == 0x400000000LL ||
value == 0x20000000000LL || value == 0x1000000000000LL ||
value == 0x80000000000000LL || value == 0x4000000000000000LL) {
c->value = c->value.neg();
if (curr->op == Abstract::getBinary(c->type, Add)) {
curr->op = Abstract::getBinary(c->type, Sub);
} else {
curr->op = Abstract::getBinary(c->type, Add);
}
}
return curr;
}
}
return nullptr;
}
};
} // anonymous namespace
// Create a custom matcher for checking side effects
template<class Opt> struct PureMatcherKind {};
template<class Opt>
struct Match::Internal::KindTypeRegistry<PureMatcherKind<Opt>> {
using matched_t = Expression*;
using data_t = Opt*;
};
template<class Opt> struct Match::Internal::MatchSelf<PureMatcherKind<Opt>> {
bool operator()(Expression* curr, Opt* opt) {
return !opt->effects(curr).hasSideEffects();
}
};
// Main pass class
struct OptimizeInstructions
: public WalkerPass<PostWalker<OptimizeInstructions>> {
bool isFunctionParallel() override { return true; }
std::unique_ptr<Pass> create() override {
return std::make_unique<OptimizeInstructions>();
}
bool fastMath;
// If set, we never fold/merge code together. This is important when fuzzing
// branch hints, as if we allow folding, then we may fold code identical in
// all ways but for branch hints, leading to an invalid branch hint executing
// later (imagine one arm had the right hint and the other the wrong one; we
// leave one of the two arbitrarily, so we might get unlucky).
bool neverFold;
// As neverFold, but for reordering code. If we move a branch hint around code
// that might trap, and the trap happens later, the branch hint might start to
// execute, and it could be wrong.
bool neverReorder;
// In rare cases we make a change to a type, and will do a refinalize.
bool refinalize = false;
void doWalkFunction(Function* func) {
fastMath = getPassOptions().fastMath;
neverFold = neverReorder =
hasArgument("optimize-instructions-never-fold-or-reorder");
// First, scan locals.
{
LocalScanner scanner(localInfo, getPassOptions());
scanner.setModule(getModule());
scanner.walkFunction(func);
}
// Main walk.
Super::doWalkFunction(func);
if (refinalize) {
ReFinalize().walkFunctionInModule(func, getModule());
}
// Final optimizations.
{
FinalOptimizer optimizer(getPassOptions());
optimizer.walkFunction(func);
}
// Some patterns create blocks that can interfere 'catch' and 'pop', nesting
// the 'pop' into a block making it invalid.
EHUtils::handleBlockNestedPops(func, *getModule());
}
// Set to true when one of the visitors makes a change (either replacing the
// node or modifying it).
bool changed;
// Used to avoid recursion in replaceCurrent, see below.
bool inReplaceCurrent = false;
void replaceCurrent(Expression* rep) {
if (rep->type != getCurrent()->type) {
// This operation will change the type, so refinalize.
refinalize = true;
}
WalkerPass<PostWalker<OptimizeInstructions>>::replaceCurrent(rep);
// We may be able to apply multiple patterns as one may open opportunities
// for others. NB: patterns must not have cycles
// To avoid recursion, this uses the following pattern: the initial call to
// this method comes from one of the visit*() methods. We then loop in here,
// and if we are called again we set |changed| instead of recursing, so that
// we can loop on that value.
if (inReplaceCurrent) {
// We are in the loop below so just note a change and return to there.
changed = true;
return;
}
// Loop on further changes.
inReplaceCurrent = true;
do {
changed = false;
visit(getCurrent());
} while (changed);
inReplaceCurrent = false;
}
EffectAnalyzer effects(Expression* expr) {
return EffectAnalyzer(getPassOptions(), *getModule(), expr);
}
Expression* getFallthrough(Expression* curr) {
return Properties::getFallthrough(curr, getPassOptions(), *getModule());
}
Type getFallthroughType(Expression* curr) {
return Properties::getFallthroughType(curr, getPassOptions(), *getModule());
}
decltype(auto) pure(Expression** binder) {
using namespace Match::Internal;
return Matcher<PureMatcherKind<OptimizeInstructions>>(binder, this);
}
bool canReorder(Expression* a, Expression* b) {
if (neverReorder) {
return false;
}
return EffectAnalyzer::canReorder(getPassOptions(), *getModule(), a, b);
}
// If an expression can only have zero bits, return a constant 0 (or null if
// we cannot optimize).
Expression* replaceZeroBitsWithZero(Expression* curr) {
// We should never be called with a constant.
assert(!curr->is<Const>());
if (!curr->type.isInteger() || Bits::getMaxBits(curr, this) != 0) {
return nullptr;
}
auto zero = Builder(*getModule()).makeConst(Literal::makeZero(curr->type));
return getDroppedChildrenAndAppend(curr, zero);
}
void visitBinary(Binary* curr) {
// If this contains dead code, don't bother trying to optimize it, the type
// might change (if might not be unreachable if just one arm is, for
// example). This optimization pass focuses on actually executing code.
if (curr->type == Type::unreachable) {
return;
}
if (shouldCanonicalize(curr)) {
canonicalize(curr);
}
{
// TODO: It is an ongoing project to port more transformations to the
// match API. Once most of the transformations have been ported, the
// `using namespace Match` can be hoisted to function scope and this extra
// block scope can be removed.
using namespace Match;
using namespace Abstract;
Builder builder(*getModule());
{
// try to get rid of (0 - ..), that is, a zero only used to negate an
// int. an add of a subtract can be flipped in order to remove it:
// (ival.add
// (ival.sub
// (ival.const 0)
// X
// )
// Y
// )
// =>
// (ival.sub
// Y
// X
// )
// Note that this reorders X and Y, so we need to be careful about that.
Expression *x, *y;
Binary* sub;
if (matches(
curr,
binary(Add, binary(&sub, Sub, ival(0), any(&x)), any(&y))) &&
canReorder(x, y)) {
sub->left = y;
sub->right = x;
return replaceCurrent(sub);
}
}
{
// The flip case is even easier, as no reordering occurs:
// (ival.add
// Y
// (ival.sub
// (ival.const 0)
// X
// )
// )
// =>
// (ival.sub
// Y
// X
// )
Expression* y;
Binary* sub;
if (matches(curr,
binary(Add, any(&y), binary(&sub, Sub, ival(0), any())))) {
sub->left = y;
return replaceCurrent(sub);
}
}
{
// try de-morgan's AND law,
// (eqz X) and (eqz Y) === eqz (X or Y)
// Note that the OR and XOR laws do not work here, as these
// are not booleans (we could check if they are, but a boolean
// would already optimize with the eqz anyhow, unless propagating).
// But for AND, the left is true iff X and Y are each all zero bits,
// and the right is true if the union of their bits is zero; same.
Unary* un;
Binary* bin;
Expression *x, *y;
if (matches(curr,
binary(&bin,
AndInt32,
unary(&un, EqZInt32, any(&x)),
unary(EqZInt32, any(&y))))) {
bin->op = OrInt32;
bin->left = x;
bin->right = y;
un->value = bin;
return replaceCurrent(un);
}
}
{
// x <<>> (C & (31 | 63)) ==> x <<>> C'
// x <<>> (y & (31 | 63)) ==> x <<>> y
// x <<>> (y & (32 | 64)) ==> x
// where '<<>>':
// '<<', '>>', '>>>'. 'rotl' or 'rotr'
BinaryOp op;
Const* c;
Expression *x, *y;
// x <<>> C
if (matches(curr, binary(&op, any(&x), ival(&c))) &&
Abstract::hasAnyShift(op)) {
// truncate RHS constant to effective size as:
// i32(x) <<>> const(C & 31))
// i64(x) <<>> const(C & 63))
c->value = c->value.and_(
Literal::makeFromInt32(c->type.getByteSize() * 8 - 1, c->type));
// x <<>> 0 ==> x
if (c->value.isZero()) {
return replaceCurrent(x);
}
}
if (matches(curr,
binary(&op, any(&x), binary(And, any(&y), ival(&c)))) &&
Abstract::hasAnyShift(op)) {
// i32(x) <<>> (y & 31) ==> x <<>> y
// i64(x) <<>> (y & 63) ==> x <<>> y
if ((c->type == Type::i32 && (c->value.geti32() & 31) == 31) ||
(c->type == Type::i64 && (c->value.geti64() & 63LL) == 63LL)) {
curr->cast<Binary>()->right = y;
return replaceCurrent(curr);
}
// i32(x) <<>> (y & C) ==> x, where (C & 31) == 0
// i64(x) <<>> (y & C) ==> x, where (C & 63) == 0
if (((c->type == Type::i32 && (c->value.geti32() & 31) == 0) ||
(c->type == Type::i64 && (c->value.geti64() & 63LL) == 0LL)) &&
!effects(y).hasSideEffects()) {
return replaceCurrent(x);
}
}
}
{
// -x + y ==> y - x
// where x, y are floating points
Expression *x, *y;
if (matches(curr, binary(Add, unary(Neg, any(&x)), any(&y))) &&
canReorder(x, y)) {
curr->op = Abstract::getBinary(curr->type, Sub);
curr->left = x;
std::swap(curr->left, curr->right);
return replaceCurrent(curr);
}
}
{
// x + (-y) ==> x - y
// x - (-y) ==> x + y
// where x, y are floating points
Expression* y;
if (matches(curr, binary(Add, any(), unary(Neg, any(&y)))) ||
matches(curr, binary(Sub, any(), unary(Neg, any(&y))))) {
curr->op = Abstract::getBinary(
curr->type,
curr->op == Abstract::getBinary(curr->type, Add) ? Sub : Add);
curr->right = y;
return replaceCurrent(curr);
}
}
{
// -x * -y ==> x * y
// where x, y are integers
Binary* bin;
Expression *x, *y;
if (matches(curr,
binary(&bin,
Mul,
binary(Sub, ival(0), any(&x)),
binary(Sub, ival(0), any(&y))))) {
bin->left = x;
bin->right = y;
return replaceCurrent(curr);
}
}
{
// -x * y ==> -(x * y)
// x * -y ==> -(x * y)
// where x, y are integers
Expression *x, *y;
if ((matches(curr,
binary(Mul, binary(Sub, ival(0), any(&x)), any(&y))) ||
matches(curr,
binary(Mul, any(&x), binary(Sub, ival(0), any(&y))))) &&
!x->is<Const>() && !y->is<Const>()) {
Builder builder(*getModule());
return replaceCurrent(
builder.makeBinary(Abstract::getBinary(curr->type, Sub),
builder.makeConst(Literal::makeZero(curr->type)),
builder.makeBinary(curr->op, x, y)));
}
}
{
if (getModule()->features.hasSignExt()) {
Const *c1, *c2;
Expression* x;
// i64(x) << 56 >> 56 ==> i64.extend8_s(x)
// i64(x) << 48 >> 48 ==> i64.extend16_s(x)
// i64(x) << 32 >> 32 ==> i64.extend32_s(x)
if (matches(curr,
binary(ShrSInt64,
binary(ShlInt64, any(&x), i64(&c1)),
i64(&c2))) &&
Bits::getEffectiveShifts(c1) == Bits::getEffectiveShifts(c2)) {
switch (64 - Bits::getEffectiveShifts(c1)) {
case 8:
return replaceCurrent(builder.makeUnary(ExtendS8Int64, x));
case 16:
return replaceCurrent(builder.makeUnary(ExtendS16Int64, x));
case 32:
return replaceCurrent(builder.makeUnary(ExtendS32Int64, x));
default:
break;
}
}
// i32(x) << 24 >> 24 ==> i32.extend8_s(x)
// i32(x) << 16 >> 16 ==> i32.extend16_s(x)
if (matches(curr,
binary(ShrSInt32,
binary(ShlInt32, any(&x), i32(&c1)),
i32(&c2))) &&
Bits::getEffectiveShifts(c1) == Bits::getEffectiveShifts(c2)) {
switch (32 - Bits::getEffectiveShifts(c1)) {
case 8:
return replaceCurrent(builder.makeUnary(ExtendS8Int32, x));
case 16:
return replaceCurrent(builder.makeUnary(ExtendS16Int32, x));
default:
break;
}
}
}
}
{
// unsigned(x) >= 0 => i32(1)
Const* c;
Expression* x;
if (matches(curr, binary(GeU, any(&x), ival(&c))) &&
c->value.isZero()) {
c->value = Literal::makeOne(Type::i32);
c->type = Type::i32;
return replaceCurrent(getDroppedChildrenAndAppend(curr, c));
}
// unsigned(x) < 0 => i32(0)
if (curr->op == Abstract::getBinary(curr->left->type, Abstract::LtU) &&
(c = getFallthrough(curr->right)->dynCast<Const>()) &&
c->value.isZero()) {
// We could reuse c here, if we checked it had no more uses
auto zero =
Builder(*getModule()).makeConst(Literal::makeZero(Type::i32));
return replaceCurrent(getDroppedChildrenAndAppend(curr, zero));
}
}
}
if (auto* ext = Properties::getAlmostSignExt(curr)) {
Index extraLeftShifts;
auto bits = Properties::getAlmostSignExtBits(curr, extraLeftShifts);
if (extraLeftShifts == 0) {
if (auto* load = getFallthrough(ext)->dynCast<Load>()) {
// pattern match a load of 8 bits and a sign extend using a shl of
// 24 then shr_s of 24 as well, etc.
if (LoadUtils::canBeSigned(load) &&
((load->bytes == 1 && bits == 8) ||
(load->bytes == 2 && bits == 16))) {
// if the value falls through, we can't alter the load, as it
// might be captured in a tee
if (load->signed_ == true || load == ext) {
load->signed_ = true;
return replaceCurrent(ext);
}
}
}
}
// We can in some cases remove part of a sign extend, that is,
// (x << A) >> B => x << (A - B)
// If the sign-extend input cannot have a sign bit, we don't need it.
if (Bits::getMaxBits(ext, this) + extraLeftShifts < bits) {
return replaceCurrent(removeAlmostSignExt(curr));
}
// We also don't need it if it already has an identical-sized sign
// extend applied to it. That is, if it is already a sign-extended
// value, then another sign extend will do nothing. We do need to be
// careful of the extra shifts, though.
if (isSignExted(ext, bits) && extraLeftShifts == 0) {
return replaceCurrent(removeAlmostSignExt(curr));
}
} else if (curr->op == EqInt32 || curr->op == NeInt32) {
if (auto* c = curr->right->dynCast<Const>()) {
if (auto* ext = Properties::getSignExtValue(curr->left)) {
// We are comparing a sign extend to a constant, which means we can
// use a cheaper zero-extend in some cases. That is,
// (x << S) >> S ==/!= C => x & T ==/!= C
// where S and T are the matching values for sign/zero extend of the
// same size. For example, for an effective 8-bit value:
// (x << 24) >> 24 ==/!= C => x & 255 ==/!= C
//
// The key thing to track here are the upper bits plus the sign bit;
// call those the "relevant bits". This is crucial because x is
// sign-extended, that is, its effective sign bit is spread to all
// the upper bits, which means that the relevant bits on the left
// side are either all 0, or all 1.
auto bits = Properties::getSignExtBits(curr->left);
uint32_t right = c->value.geti32();
uint32_t numRelevantBits = 32 - bits + 1;
uint32_t setRelevantBits =
Bits::popCount(right >> uint32_t(bits - 1));
// If all the relevant bits on C are zero
// then we can mask off the high bits instead of sign-extending x.
// This is valid because if x is negative, then the comparison was
// false before (negative vs positive), and will still be false
// as the sign bit will remain to cause a difference. And if x is
// positive then the upper bits would be zero anyhow.
if (setRelevantBits == 0) {
curr->left = makeZeroExt(ext, bits);
return replaceCurrent(curr);
} else if (setRelevantBits == numRelevantBits) {
// If all those bits are one, then we can do something similar if
// we also zero-extend on the right as well. This is valid
// because, as in the previous case, the sign bit differentiates
// the two sides when they are different, and if the sign bit is
// identical, then the upper bits don't matter, so masking them
// off both sides is fine.
curr->left = makeZeroExt(ext, bits);
c->value = c->value.and_(Literal(Bits::lowBitMask(bits)));
return replaceCurrent(curr);
} else {
// Otherwise, C's relevant bits are mixed, and then the two sides
// can never be equal, as the left side's bits cannot be mixed.
Builder builder(*getModule());
// The result is either always true, or always false.
c->value = Literal::makeFromInt32(curr->op == NeInt32, c->type);
return replaceCurrent(
builder.makeSequence(builder.makeDrop(ext), c));
}
}
} else if (auto* left = Properties::getSignExtValue(curr->left)) {
if (auto* right = Properties::getSignExtValue(curr->right)) {
auto bits = Properties::getSignExtBits(curr->left);
if (Properties::getSignExtBits(curr->right) == bits) {
// we are comparing two sign-exts with the same bits, so we may as
// well replace both with cheaper zexts
curr->left = makeZeroExt(left, bits);
curr->right = makeZeroExt(right, bits);
return replaceCurrent(curr);
}
} else if (auto* load = curr->right->dynCast<Load>()) {
// we are comparing a load to a sign-ext, we may be able to switch
// to zext
auto leftBits = Properties::getSignExtBits(curr->left);
if (load->signed_ && leftBits == load->bytes * 8) {
load->signed_ = false;
curr->left = makeZeroExt(left, leftBits);
return replaceCurrent(curr);
}
}
} else if (auto* load = curr->left->dynCast<Load>()) {
if (auto* right = Properties::getSignExtValue(curr->right)) {
// we are comparing a load to a sign-ext, we may be able to switch
// to zext
auto rightBits = Properties::getSignExtBits(curr->right);
if (load->signed_ && rightBits == load->bytes * 8) {
load->signed_ = false;
curr->right = makeZeroExt(right, rightBits);
return replaceCurrent(curr);
}
}
}
// note that both left and right may be consts, but then we let
// precompute compute the constant result
} else if (curr->op == AddInt32 || curr->op == AddInt64 ||
curr->op == SubInt32 || curr->op == SubInt64) {
if (auto* ret = optimizeAddedConstants(curr)) {
return replaceCurrent(ret);
}
} else if (curr->op == MulFloat32 || curr->op == MulFloat64 ||
curr->op == DivFloat32 || curr->op == DivFloat64) {
if (curr->left->type == curr->right->type) {
if (auto* leftUnary = curr->left->dynCast<Unary>()) {
if (leftUnary->op == Abstract::getUnary(curr->type, Abstract::Abs)) {
if (auto* rightUnary = curr->right->dynCast<Unary>()) {
if (leftUnary->op == rightUnary->op) { // both are abs ops
// abs(x) * abs(y) ==> abs(x * y)
// abs(x) / abs(y) ==> abs(x / y)
curr->left = leftUnary->value;
curr->right = rightUnary->value;
leftUnary->value = curr;
return replaceCurrent(leftUnary);
}
}
}
}
}
}
// a bunch of operations on a constant right side can be simplified
if (auto* right = curr->right->dynCast<Const>()) {
if (curr->op == AndInt32) {
auto mask = right->value.geti32();
// and with -1 does nothing (common in asm.js output)
if (mask == -1) {
return replaceCurrent(curr->left);
}
// small loads do not need to be masked, the load itself masks
if (auto* load = curr->left->dynCast<Load>()) {
if ((load->bytes == 1 && mask == 0xff) ||
(load->bytes == 2 && mask == 0xffff)) {
load->signed_ = false;
return replaceCurrent(curr->left);
}
} else if (auto maskedBits = Bits::getMaskedBits(mask)) {
if (Bits::getMaxBits(curr->left, this) <= maskedBits) {
// a mask of lower bits is not needed if we are already smaller
return replaceCurrent(curr->left);
}
}
}
// some math operations have trivial results
if (auto* ret = optimizeWithConstantOnRight(curr)) {
return replaceCurrent(ret);
}
if (auto* ret = optimizeDoubletonWithConstantOnRight(curr)) {
return replaceCurrent(ret);
}
if (right->type == Type::i32) {
BinaryOp op;
int32_t c = right->value.geti32();
// First, try to lower signed operations to unsigned if that is
// possible. Some unsigned operations like div_u or rem_u are usually
// faster on VMs. Also this opens more possibilities for further
// simplifications afterwards.
if (c >= 0 && (op = makeUnsignedBinaryOp(curr->op)) != InvalidBinary &&
Bits::getMaxBits(curr->left, this) <= 31) {
curr->op = op;
}
if (c < 0 && c > std::numeric_limits<int32_t>::min() &&
curr->op == DivUInt32) {
// u32(x) / C ==> u32(x) >= C iff C > 2^31
// We avoid applying this for C == 2^31 due to conflict
// with other rule which transform to more preferable
// right shift operation.
curr->op = c == -1 ? EqInt32 : GeUInt32;
return replaceCurrent(curr);
}
if (Bits::isPowerOf2((uint32_t)c)) {
switch (curr->op) {
case MulInt32:
return replaceCurrent(optimizePowerOf2Mul(curr, (uint32_t)c));
case RemUInt32:
return replaceCurrent(optimizePowerOf2URem(curr, (uint32_t)c));
case DivUInt32:
return replaceCurrent(optimizePowerOf2UDiv(curr, (uint32_t)c));
default:
break;
}
}
}
if (right->type == Type::i64) {
BinaryOp op;
int64_t c = right->value.geti64();
// See description above for Type::i32
if (c >= 0 && (op = makeUnsignedBinaryOp(curr->op)) != InvalidBinary &&
Bits::getMaxBits(curr->left, this) <= 63) {
curr->op = op;
}
if (getPassOptions().shrinkLevel == 0 && c < 0 &&
c > std::numeric_limits<int64_t>::min() && curr->op == DivUInt64) {
// u64(x) / C ==> u64(u64(x) >= C) iff C > 2^63
// We avoid applying this for C == 2^31 due to conflict
// with other rule which transform to more preferable
// right shift operation.
// And apply this only for shrinkLevel == 0 due to it
// increasing size by one byte.
curr->op = c == -1LL ? EqInt64 : GeUInt64;
curr->type = Type::i32;
return replaceCurrent(
Builder(*getModule()).makeUnary(ExtendUInt32, curr));
}
if (Bits::isPowerOf2((uint64_t)c)) {
switch (curr->op) {
case MulInt64:
return replaceCurrent(optimizePowerOf2Mul(curr, (uint64_t)c));
case RemUInt64:
return replaceCurrent(optimizePowerOf2URem(curr, (uint64_t)c));
case DivUInt64:
return replaceCurrent(optimizePowerOf2UDiv(curr, (uint64_t)c));
default:
break;
}
}
}
if (curr->op == DivFloat32) {
float c = right->value.getf32();
if (Bits::isPowerOf2InvertibleFloat(c)) {
return replaceCurrent(optimizePowerOf2FDiv(curr, c));
}
}
if (curr->op == DivFloat64) {
double c = right->value.getf64();
if (Bits::isPowerOf2InvertibleFloat(c)) {
return replaceCurrent(optimizePowerOf2FDiv(curr, c));
}
}
}
// a bunch of operations on a constant left side can be simplified
if (curr->left->is<Const>()) {
if (auto* ret = optimizeWithConstantOnLeft(curr)) {
return replaceCurrent(ret);
}
}
if (curr->op == AndInt32 || curr->op == OrInt32) {
if (curr->op == AndInt32) {
if (auto* ret = combineAnd(curr)) {
return replaceCurrent(ret);
}
if (auto* ret = optimizeAndNoOverlappingBits(curr)) {
return replaceCurrent(ret);
}
}
// for or, we can potentially combine
if (curr->op == OrInt32) {
if (auto* ret = combineOr(curr)) {
return replaceCurrent(ret);
}
}
// bitwise operations
// for and and or, we can potentially conditionalize
if (auto* ret = conditionalizeExpensiveOnBitwise(curr)) {
return replaceCurrent(ret);
}
}
if (curr->op == AndInt64) {
if (auto* ret = optimizeAndNoOverlappingBits(curr)) {
return replaceCurrent(ret);
}
}
// relation/comparisons allow for math optimizations
if (curr->isRelational()) {
if (auto* ret = optimizeRelational(curr)) {
return replaceCurrent(ret);
}
}
// see if we can infer this is a zero
if (auto* ret = replaceZeroBitsWithZero(curr)) {
return replaceCurrent(ret);
}
// finally, try more expensive operations on the curr in
// the case that they have no side effects
if (!effects(curr->left).hasSideEffects()) {
if (ExpressionAnalyzer::equal(curr->left, curr->right)) {
if (auto* ret = optimizeBinaryWithEqualEffectlessChildren(curr)) {
return replaceCurrent(ret);
}
}
}
if (auto* ret = deduplicateBinary(curr)) {
return replaceCurrent(ret);
}
}
void visitUnary(Unary* curr) {
if (curr->type == Type::unreachable) {
return;
}
{
using namespace Match;
using namespace Abstract;
Builder builder(*getModule());
{
// eqz(x - y) => x == y
Binary* inner;
if (matches(curr, unary(EqZ, binary(&inner, Sub, any(), any())))) {
inner->op = Abstract::getBinary(inner->left->type, Eq);
inner->type = Type::i32;
return replaceCurrent(inner);
}
}
{
// eqz(x + C) => x == -C
Const* c;
Binary* inner;
if (matches(curr, unary(EqZ, binary(&inner, Add, any(), ival(&c))))) {
c->value = c->value.neg();
inner->op = Abstract::getBinary(c->type, Eq);
inner->type = Type::i32;
return replaceCurrent(inner);
}
}
{
// eqz((signed)x % C_pot) => eqz(x & (abs(C_pot) - 1))
Const* c;
Binary* inner;
if (matches(curr, unary(EqZ, binary(&inner, RemS, any(), ival(&c)))) &&
(c->value.isSignedMin() ||
Bits::isPowerOf2(c->value.abs().getInteger()))) {
inner->op = Abstract::getBinary(c->type, And);
if (c->value.isSignedMin()) {
c->value = Literal::makeSignedMax(c->type);
} else {
c->value = c->value.abs().sub(Literal::makeOne(c->type));
}
return replaceCurrent(curr);
}
}
{
// i32.wrap_i64 can be removed if the operations inside it do not
// actually require 64 bits, e.g.:
//
// i32.wrap_i64(i64.extend_i32_u(x)) => x
if (matches(curr, unary(WrapInt64, any()))) {
if (auto* ret = optimizeWrappedResult(curr)) {
return replaceCurrent(ret);
}
}
}
{
// i32.eqz(i32.wrap_i64(x)) => i64.eqz(x)
// where maxBits(x) <= 32
Unary* inner;
Expression* x;
if (matches(curr, unary(EqZInt32, unary(&inner, WrapInt64, any(&x)))) &&
Bits::getMaxBits(x, this) <= 32) {
inner->op = EqZInt64;
return replaceCurrent(inner);
}
}
{
// i32.eqz(i32.eqz(x)) => i32(x) != 0
// i32.eqz(i64.eqz(x)) => i64(x) != 0
// iff shinkLevel == 0
// (1 instruction instead of 2, but 1 more byte)
if (getPassRunner()->options.shrinkLevel == 0) {
Expression* x;
if (matches(curr, unary(EqZInt32, unary(EqZ, any(&x))))) {
Builder builder(*getModule());
return replaceCurrent(builder.makeBinary(
getBinary(x->type, Ne),
x,
builder.makeConst(Literal::makeZero(x->type))));
}
}
}
{
// i64.extend_i32_s(i32.wrap_i64(x)) => x
// where maxBits(x) <= 31
//
// i64.extend_i32_u(i32.wrap_i64(x)) => x
// where maxBits(x) <= 32
Expression* x;
UnaryOp unaryOp;
if (matches(curr, unary(&unaryOp, unary(WrapInt64, any(&x))))) {
if (unaryOp == ExtendSInt32 || unaryOp == ExtendUInt32) {
auto maxBits = Bits::getMaxBits(x, this);
if ((unaryOp == ExtendSInt32 && maxBits <= 31) ||
(unaryOp == ExtendUInt32 && maxBits <= 32)) {
return replaceCurrent(x);
}
}
}
}
if (getModule()->features.hasSignExt()) {
// i64.extend_i32_s(i32.wrap_i64(x)) => i64.extend32_s(x)
Unary* inner;
if (matches(curr,
unary(ExtendSInt32, unary(&inner, WrapInt64, any())))) {
inner->op = ExtendS32Int64;
inner->type = Type::i64;
return replaceCurrent(inner);
}
}
}
if (curr->op == ExtendUInt32 || curr->op == ExtendSInt32) {
if (auto* load = curr->value->dynCast<Load>()) {
// i64.extend_i32_s(i32.load(_8|_16)(_u|_s)(x)) =>
// i64.load(_8|_16|_32)(_u|_s)(x)
//
// i64.extend_i32_u(i32.load(_8|_16)(_u|_s)(x)) =>
// i64.load(_8|_16|_32)(_u|_s)(x)
//
// but we can't do this in following cases:
//
// i64.extend_i32_u(i32.load8_s(x))
// i64.extend_i32_u(i32.load16_s(x))
//
// this mixed sign/zero extensions can't represent in single
// signed or unsigned 64-bit load operation. For example if `load8_s(x)`
// return i8(-1) (0xFF) than sign extended result will be
// i32(-1) (0xFFFFFFFF) and with zero extension to i64 we got
// finally 0x00000000FFFFFFFF. However with `i64.load8_s` in this
// situation we got `i64(-1)` (all ones) and with `i64.load8_u` it
// will be 0x00000000000000FF.
//
// Another limitation is atomics which only have unsigned loads.
// So we also avoid this only case:
//
// i64.extend_i32_s(i32.atomic.load(x))
// Special case for i32.load. In this case signedness depends on
// extend operation.
bool willBeSigned = curr->op == ExtendSInt32 && load->bytes == 4;
if (!(curr->op == ExtendUInt32 && load->bytes <= 2 && load->signed_) &&
!(willBeSigned && load->isAtomic())) {
if (willBeSigned) {
load->signed_ = true;
}
load->type = Type::i64;
return replaceCurrent(load);
}
}
}
// Simple sign extends can be removed if the value is already sign-extended.
auto signExtBits = getSignExtBits(curr->value);
if (signExtBits > 0) {
// Note that we can handle the case of |curr| having a larger sign-extend:
// if we have an 8-bit value in 32-bit, then there are 24 sign bits, and
// doing a sign-extend to 16 will only affect 16 of those 24, and the
// effect is to leave them as they are.
if ((curr->op == ExtendS8Int32 && signExtBits <= 8) ||
(curr->op == ExtendS16Int32 && signExtBits <= 16) ||
(curr->op == ExtendS8Int64 && signExtBits <= 8) ||
(curr->op == ExtendS16Int64 && signExtBits <= 16) ||
(curr->op == ExtendS32Int64 && signExtBits <= 32)) {
return replaceCurrent(curr->value);
}
}
if (Abstract::hasAnyReinterpret(curr->op)) {
// i32.reinterpret_f32(f32.reinterpret_i32(x)) => x
// i64.reinterpret_f64(f64.reinterpret_i64(x)) => x
// f32.reinterpret_i32(i32.reinterpret_f32(x)) => x
// f64.reinterpret_i64(i64.reinterpret_f64(x)) => x
if (auto* inner = curr->value->dynCast<Unary>()) {
if (Abstract::hasAnyReinterpret(inner->op)) {
if (inner->value->type == curr->type) {
return replaceCurrent(inner->value);
}
}
}
// f32.reinterpret_i32(i32.load(x)) => f32.load(x)
// f64.reinterpret_i64(i64.load(x)) => f64.load(x)
// i32.reinterpret_f32(f32.load(x)) => i32.load(x)
// i64.reinterpret_f64(f64.load(x)) => i64.load(x)
if (auto* load = curr->value->dynCast<Load>()) {
if (!load->isAtomic() && load->bytes == curr->type.getByteSize()) {
load->type = curr->type;
return replaceCurrent(load);
}
}
}
if (curr->op == EqZInt32) {
if (auto* inner = curr->value->dynCast<Binary>()) {
// Try to invert a relational operation using De Morgan's law
auto op = invertBinaryOp(inner->op);
if (op != InvalidBinary) {
inner->op = op;
return replaceCurrent(inner);
}
}
// eqz of a sign extension can be of zero-extension
if (auto* ext = Properties::getSignExtValue(curr->value)) {
// we are comparing a sign extend to a constant, which means we can
// use a cheaper zext
auto bits = Properties::getSignExtBits(curr->value);
curr->value = makeZeroExt(ext, bits);
return replaceCurrent(curr);
}
} else if (curr->op == AbsFloat32 || curr->op == AbsFloat64) {
// abs(-x) ==> abs(x)
if (auto* unaryInner = curr->value->dynCast<Unary>()) {
if (unaryInner->op ==
Abstract::getUnary(unaryInner->type, Abstract::Neg)) {
curr->value = unaryInner->value;
return replaceCurrent(curr);
}
}
// abs(x * x) ==> x * x
// abs(x / x) ==> x / x
if (auto* binary = curr->value->dynCast<Binary>()) {
if ((binary->op == Abstract::getBinary(binary->type, Abstract::Mul) ||
binary->op == Abstract::getBinary(binary->type, Abstract::DivS)) &&
areConsecutiveInputsEqual(binary->left, binary->right)) {
return replaceCurrent(binary);
}
// abs(0 - x) ==> abs(x),
// only for fast math
if (fastMath &&
binary->op == Abstract::getBinary(binary->type, Abstract::Sub)) {
if (auto* c = binary->left->dynCast<Const>()) {
if (c->value.isZero()) {
curr->value = binary->right;
return replaceCurrent(curr);
}
}
}
}
}
if (auto* ret = deduplicateUnary(curr)) {
return replaceCurrent(ret);
}
if (auto* ret = simplifyRoundingsAndConversions(curr)) {
return replaceCurrent(ret);
}
if (auto* ret = replaceZeroBitsWithZero(curr)) {
return replaceCurrent(ret);
}
}
void visitSelect(Select* curr) {
if (curr->type == Type::unreachable) {
return;
}
if (auto* ret = optimizeSelect(curr)) {
return replaceCurrent(ret);
}
optimizeTernary(curr);
}
void visitGlobalSet(GlobalSet* curr) {
if (curr->type == Type::unreachable) {
return;
}
// optimize out a set of a get
auto* get = curr->value->dynCast<GlobalGet>();
if (get && get->name == curr->name) {
ExpressionManipulator::nop(curr);
return replaceCurrent(curr);
}
}
void visitIf(If* curr) {
curr->condition = optimizeBoolean(curr->condition);
if (curr->ifFalse) {
if (auto* unary = curr->condition->dynCast<Unary>()) {
if (unary->op == EqZInt32) {
// flip if-else arms to get rid of an eqz
curr->condition = unary->value;
std::swap(curr->ifTrue, curr->ifFalse);
BranchHints::flip(curr, getFunction());
}
}
// Note that we do not consider metadata here. Like LLVM, we ignore
// metadata when trying to fold code together, preferring certain
// optimization over possible benefits of profiling data.
if (!neverFold && curr->condition->type != Type::unreachable &&
ExpressionAnalyzer::equal(curr->ifTrue, curr->ifFalse)) {
// The sides are identical, so fold. If we can replace the If with one
// arm and there are no side effects in the condition, replace it. But
// make sure not to change a concrete expression to an unreachable
// expression because we want to avoid having to refinalize.
bool needCondition = effects(curr->condition).hasSideEffects();
bool wouldBecomeUnreachable =
curr->type.isConcrete() && curr->ifTrue->type == Type::unreachable;
Builder builder(*getModule());
if (!wouldBecomeUnreachable && !needCondition) {
return replaceCurrent(curr->ifTrue);
} else if (!wouldBecomeUnreachable) {
return replaceCurrent(builder.makeSequence(
builder.makeDrop(curr->condition), curr->ifTrue));
} else {
// Emit a block with the original concrete type.
auto* ret = builder.makeBlock();
if (needCondition) {
ret->list.push_back(builder.makeDrop(curr->condition));
}
ret->list.push_back(curr->ifTrue);
ret->finalize(curr->type);
return replaceCurrent(ret);
}
}
optimizeTernary(curr);
}
}
void visitLocalSet(LocalSet* curr) {
// Interactions between local.set/tee and ref.as_non_null can be optimized
// in some cases, by removing or moving the ref.as_non_null operation.
if (auto* as = curr->value->dynCast<RefAs>()) {
if (as->op == RefAsNonNull &&
getFunction()->getLocalType(curr->index).isNullable()) {
// (local.tee (ref.as_non_null ..))
// =>
// (ref.as_non_null (local.tee ..))
//
// The reordering allows the ref.as to be potentially optimized further
// based on where the value flows to.
if (curr->isTee()) {
curr->value = as->value;
curr->finalize();
as->value = curr;
as->finalize();
replaceCurrent(as);
return;
}
// Otherwise, if this is not a tee, then no value falls through. The
// ref.as_non_null acts as a null check here, basically. If we are
// ignoring such traps, we can remove it.
auto& passOptions = getPassOptions();
if (passOptions.ignoreImplicitTraps || passOptions.trapsNeverHappen) {
curr->value = as->value;
}
}
}
}
void visitBreak(Break* curr) {
if (curr->condition) {
curr->condition = optimizeBoolean(curr->condition);
}
}
void visitLoad(Load* curr) {
if (curr->type == Type::unreachable) {
return;
}
optimizeMemoryAccess(curr->ptr, curr->offset, curr->memory);
}
void visitStore(Store* curr) {
if (curr->type == Type::unreachable) {
return;
}
optimizeMemoryAccess(curr->ptr, curr->offset, curr->memory);
optimizeStoredValue(curr->value, curr->bytes);
if (auto* unary = curr->value->dynCast<Unary>()) {
if (unary->op == WrapInt64) {
// instead of wrapping to 32, just store some of the bits in the i64
curr->valueType = Type::i64;
curr->value = unary->value;
} else if (!curr->isAtomic() && Abstract::hasAnyReinterpret(unary->op) &&
curr->bytes == curr->valueType.getByteSize()) {
// f32.store(y, f32.reinterpret_i32(x)) => i32.store(y, x)
// f64.store(y, f64.reinterpret_i64(x)) => i64.store(y, x)
// i32.store(y, i32.reinterpret_f32(x)) => f32.store(y, x)
// i64.store(y, i64.reinterpret_f64(x)) => f64.store(y, x)
curr->valueType = unary->value->type;
curr->value = unary->value;
}
}
}
void optimizeStoredValue(Expression*& value, Index bytes) {
if (!value->type.isInteger()) {
return;
}
// truncates constant values during stores
// (i32|i64).store(8|16|32)(p, C) ==>
// (i32|i64).store(8|16|32)(p, C & mask)
if (auto* c = value->dynCast<Const>()) {
if (value->type == Type::i64 && bytes == 4) {
c->value = c->value.and_(Literal(uint64_t(0xffffffff)));
} else {
c->value = c->value.and_(
Literal::makeFromInt32(Bits::lowBitMask(bytes * 8), value->type));
}
}
// stores of fewer bits truncates anyhow
if (auto* binary = value->dynCast<Binary>()) {
if (binary->op == AndInt32) {
if (auto* right = binary->right->dynCast<Const>()) {
if (right->type == Type::i32) {
auto mask = right->value.geti32();
if ((bytes == 1 && mask == 0xff) ||
(bytes == 2 && mask == 0xffff)) {
value = binary->left;
}
}
}
} else if (auto* ext = Properties::getSignExtValue(binary)) {
// if sign extending the exact bit size we store, we can skip the
// extension if extending something bigger, then we just alter bits we
// don't save anyhow
if (Properties::getSignExtBits(binary) >= Index(bytes) * 8) {
value = ext;
}
}
}
}
void visitMemoryCopy(MemoryCopy* curr) {
if (curr->type == Type::unreachable) {
return;
}
assert(getModule()->features.hasBulkMemoryOpt());
if (auto* ret = optimizeMemoryCopy(curr)) {
return replaceCurrent(ret);
}
}
void visitMemoryFill(MemoryFill* curr) {
if (curr->type == Type::unreachable) {
return;
}
assert(getModule()->features.hasBulkMemoryOpt());
if (auto* ret = optimizeMemoryFill(curr)) {
return replaceCurrent(ret);
}
}
void visitCallRef(CallRef* curr) {
skipNonNullCast(curr->target, curr);
if (trapOnNull(curr, curr->target)) {
return;
}
if (curr->target->type == Type::unreachable) {
// The call_ref is not reached; leave this for DCE.
return;
}
if (auto* ref = curr->target->dynCast<RefFunc>()) {
// We know the target!
replaceCurrent(
Builder(*getModule())
.makeCall(ref->func, curr->operands, curr->type, curr->isReturn));
return;
}
if (auto* get = curr->target->dynCast<TableGet>()) {
// (call_ref ..args.. (table.get $table (index))
// =>
// (call_indirect $table ..args.. (index))
replaceCurrent(Builder(*getModule())
.makeCallIndirect(get->table,
get->index,
curr->operands,
get->type.getHeapType(),
curr->isReturn));
return;
}
auto features = getModule()->features;
// It is possible the target is not a function reference, but we can infer
// the fallthrough value there. It takes more work to optimize this case,
// but it is pretty important to allow a call_ref to become a fast direct
// call, so make the effort.
if (auto* ref = getFallthrough(curr->target)->dynCast<RefFunc>()) {
// Check if the fallthrough make sense. We may have cast it to a different
// type, which would be a problem - we'd be replacing a call_ref to one
// type with a direct call to a function of another type. That would trap
// at runtime; be careful not to emit invalid IR here.
if (curr->target->type.getHeapType() != ref->type.getHeapType()) {
return;
}
Builder builder(*getModule());
if (curr->operands.empty()) {
// No operands, so this is simple and there is nothing to reorder: just
// emit:
//
// (block
// (drop curr->target)
// (call ref.func-from-curr->target)
// )
replaceCurrent(builder.makeSequence(
builder.makeDrop(curr->target),
builder.makeCall(ref->func, {}, curr->type, curr->isReturn)));
return;
}
// In the presence of operands, we must execute the code in curr->target
// after the last operand and before the call happens. Interpose at the
// last operand:
//
// (call ref.func-from-curr->target)
// (operand1)
// (..)
// (operandN-1)
// (block
// (local.set $temp (operandN))
// (drop curr->target)
// (local.get $temp)
// )
// )
auto* lastOperand = curr->operands.back();
auto lastOperandType = lastOperand->type;
if (lastOperandType == Type::unreachable) {
// The call_ref is not reached; leave this for DCE.
return;
}
Index tempLocal = builder.addVar(
getFunction(),
TypeUpdating::getValidLocalType(lastOperandType, features));
auto* set = builder.makeLocalSet(tempLocal, lastOperand);
auto* drop = builder.makeDrop(curr->target);
auto* get = TypeUpdating::fixLocalGet(
builder.makeLocalGet(tempLocal, lastOperandType), *getModule());
curr->operands.back() = builder.makeBlock({set, drop, get});
replaceCurrent(builder.makeCall(
ref->func, curr->operands, curr->type, curr->isReturn));
return;
}
// If the target is a select of two different constants, we can emit an if
// over two direct calls.
if (auto* calls = CallUtils::convertToDirectCalls(
curr,
[](Expression* target) -> CallUtils::IndirectCallInfo {
if (auto* refFunc = target->dynCast<RefFunc>()) {
return CallUtils::Known{refFunc->func};
}
return CallUtils::Unknown{};
},
*getFunction(),
*getModule())) {
replaceCurrent(calls);
}
}
void visitResume(Resume* curr) {
skipNonNullCast(curr->cont, curr);
if (trapOnNull(curr, curr->cont)) {
return;
}
if (curr->type == Type::unreachable) {
return;
}
// If this resume operates on a freshly-created continuation of an exact
// function that is known never to suspend, we can turn the resumption into
// a direct call, avoiding the continuation allocation and handler overhead.
// Continuations are single-shot, so resuming a continuation that has
// already been consumed will trap. If traps are assumed never to happen, we
// can assume this continuation will not be consumed on another path and
// look through tees and conditional branches. Otherwise, avoid looking
// through them to ensure the continuation cannot be consumed elsewhere.
auto behavior = getPassOptions().trapsNeverHappen
? Properties::FallthroughBehavior::AllowTeeBrIf
: Properties::FallthroughBehavior::NoTeeBrIf;
auto* contExpr = Properties::getFallthrough(
curr->cont, getPassOptions(), *getModule(), behavior);
auto* contNew = contExpr->dynCast<ContNew>();
if (!contNew) {
return;
}
if (contNew->func->type == Type::unreachable) {
return;
}
auto* funcExpr =
Properties::getFallthrough(contNew->func, getPassOptions(), *getModule());
auto* refFunc = funcExpr->dynCast<RefFunc>();
if (!refFunc) {
return;
}
auto* target = getModule()->getFunctionOrNull(refFunc->func);
if (!target || target->imported()) {
return;
}
if (!target->effects || target->effects->suspends) {
return;
}
auto* block =
ChildLocalizer(curr, getFunction(), *getModule(), getPassOptions())
.getChildrenReplacement();
Builder builder(*getModule());
Type results = target->getResults();
block->list.push_back(
builder.makeCall(target->name, curr->operands, results));
block->type = results;
replaceCurrent(block);
}
// Note on removing casts (which the following utilities, skipNonNullCast and
// skipCast do): removing a cast is potentially dangerous, as it removes
// information from the IR. For example:
//
// (ref.test (ref i31)
// (ref.cast (ref i31)
// (local.get $anyref)))
//
// The local has no useful type info here (it is anyref). The cast forces it
// to be an i31, so we know that if we do not trap then the ref.test will
// definitely be 1. But if we removed the ref.cast first (which we can do in
// traps-never-happen mode) then we'd not have the type info we need to
// optimize that way.
//
// To avoid such risks we should keep in mind the following:
//
// * Before removing a cast we should use its type information in the best
// way we can. Only after doing so should a cast be removed. In the example
// above, that means first seeing that the ref.test must return 1, and only
// then possibly removing the ref.cast.
// * Do not remove a cast if removing it might remove useful information for
// others. For example,
//
// (ref.cast (ref null $A)
// (ref.as_non_null ..))
//
// If we remove the inner cast then the outer cast becomes nullable. That
// means we'd be throwing away useful information, which we should not do,
// even in traps-never-happen mode and even if the wasm would validate
// without the cast. Only if we saw that the parents of the outer cast
// cannot benefit from non-nullability should we remove it.
// Another example:
//
// (struct.get $A 0
// (ref.cast $B ..))
//
// The cast only changes the type of the reference, which is consumed in
// this expression and so we don't have more parents to consider. But it is
// risky to remove this cast, since e.g. GUFA benefits from such info:
// it tells GUFA that we are reading from a $B here, and not the supertype
// $A. If $B may contain fewer values in field 0 than $A, then GUFA might
// be able to optimize better with this cast. Now, in traps-never-happen
// mode we can assume that only $B can arrive here, which means GUFA might
// be able to infer that even without the cast - but it might not, if we
// hit a limitation of GUFA. Some code patterns simply cannot be expected
// to be always inferred, say if a data structure has a tagged variant:
//
// {
// tag: i32,
// ref: anyref
// }
//
// Imagine that if tag == 0 then the reference always contains struct $A,
// and if tag == 1 then it always contains a struct $B, and so forth. We
// can't expect GUFA to figure out such invariants in general. But by
// having casts in the right places we can help GUFA optimize:
//
// (if
// (tag == 1)
// (struct.get $A 0
// (ref.cast $B ..))
//
// We know it must be a $B due to the tag. By keeping the cast there we can
// make sure that optimizations can benefit from that.
//
// Given the large amount of potential benefit we can get from a successful
// optimization in GUFA, any reduction there may be a bad idea, so we
// should be very careful and probably *not* remove such casts.
// If an instruction traps on a null input, there is no need for a
// ref.as_non_null on that input: we will trap either way (and the binaryen
// optimizer does not differentiate traps).
//
// See "notes on removing casts", above. However, in most cases removing a
// non-null cast is obviously safe to do, since we only remove one if another
// check will happen later.
//
// We also pass in the parent, because we need to be careful about ordering:
// if the parent has other children than |input| then we may not be able to
// remove the trap. For example,
//
// (struct.set
// (ref.as_non_null X)
// (call $foo)
// )
//
// If X is null we'd trap before the call to $foo. If we remove the
// ref.as_non_null then the struct.set will still trap, of course, but that
// will only happen *after* the call, which is wrong.
void skipNonNullCast(Expression*& input, Expression* parent) {
// If we must never reorder code, then we cannot remove a non-null cast.
// Such removals are valid because we move the trap later (see the
// struct.set in the example above: we can remove the ref.as_non_null
// because the set will trap anyhow, so we are pushing the trap onward; in
// the example above we have a call we can't move past, and in neverReorder
// mode we need to care about things like branch hints, which do not have
// effects, hence the need for the special neverReorder flag).
if (neverReorder) {
return;
}
// Check the other children for the ordering problem only if we find a
// possible optimization, to avoid wasted work.
bool checkedSiblings = false;
auto& options = getPassOptions();
while (1) {
if (auto* as = input->dynCast<RefAs>()) {
if (as->op == RefAsNonNull) {
// The problem with effect ordering that is described above is not an
// issue if traps are assumed to never happen anyhow.
if (!checkedSiblings && !options.trapsNeverHappen) {
// We need to see if a child with side effects exists after |input|.
// If there is such a child, it is a problem as mentioned above (it
// is fine for such a child to appear *before* |input|, as then we
// wouldn't be reordering effects). Thus, all we need to do is
// accumulate the effects in children after |input|, as we want to
// move the trap across those.
bool seenInput = false;
EffectAnalyzer crossedEffects(options, *getModule());
for (auto* child : ChildIterator(parent)) {
if (child == input) {
seenInput = true;
} else if (seenInput) {
crossedEffects.walk(child);
}
}
// Check if the effects we cross interfere with the effects of the
// trap we want to move. (We use a shallow effect analyzer since we
// will only move the ref.as_non_null itself.)
ShallowEffectAnalyzer movingEffects(options, *getModule(), input);
if (movingEffects.orderedBefore(crossedEffects)) {
return;
}
// If we got here, we've checked the siblings and found no problem.
checkedSiblings = true;
}
input = as->value;
continue;
}
}
break;
}
}
// As skipNonNullCast, but skips all casts if we can do so. This is useful in
// cases where we don't actually care about the type but just the value, that
// is, if casts of the type do not affect our behavior (which is the case in
// ref.eq for example).
//
// |requiredType| is the required supertype of the final output. We will not
// remove a cast that would leave something that would break that. If
// |requiredType| is not provided we will accept any type there.
//
// See "notes on removing casts", above, for when this is safe to do.
void skipCast(Expression*& input, Type requiredType = Type::none) {
// Traps-never-happen mode is a requirement for us to optimize here.
if (!getPassOptions().trapsNeverHappen) {
return;
}
while (1) {
if (auto* as = input->dynCast<RefAs>()) {
if (requiredType == Type::none ||
Type::isSubType(as->value->type, requiredType)) {
input = as->value;
continue;
}
} else if (auto* cast = input->dynCast<RefCast>()) {
if (requiredType == Type::none ||
Type::isSubType(cast->ref->type, requiredType)) {
input = cast->ref;
continue;
}
}
break;
}
}
// Appends a result after the dropped children, if we need them.
Expression*
getDroppedChildrenAndAppend(Expression* curr,
Expression* result,
DropMode mode = DropMode::NoticeParentEffects) {
return wasm::getDroppedChildrenAndAppend(
curr, *getModule(), getPassOptions(), result, mode);
}
Expression* getDroppedChildrenAndAppend(Expression* curr, Literal value) {
auto* result = Builder(*getModule()).makeConst(value);
return getDroppedChildrenAndAppend(curr, result);
}
Expression* getResultOfFirst(Expression* first, Expression* second) {
return wasm::getResultOfFirst(
first, second, getFunction(), getModule(), getPassOptions());
}
// Optimize an instruction and the reference it operates on, under the
// assumption that if the reference is a null then we will trap. Returns true
// if we replaced the expression with something simpler. Returns false if we
// found nothing to optimize, or if we just modified or replaced the ref (but
// not the expression itself).
bool trapOnNull(Expression* curr, Expression*& ref) {
Builder builder(*getModule());
if (getPassOptions().trapsNeverHappen) {
// We can ignore the possibility of the reference being an input, so
//
// (if
// (condition)
// (null)
// (other))
// =>
// (drop
// (condition))
// (other)
//
// That is, we will by assumption not read from the null, so remove that
// arm.
//
// TODO We could recurse here.
// TODO We could do similar things for casts (rule out an impossible arm).
// TODO Worth thinking about an 'assume' intrinsic of some form that
// annotates knowledge about a value, or another mechanism to allow
// that information to be passed around.
// Note that we must check that the null is actually flowed out, that is,
// that control flow is not transferred before:
//
// (if
// (1)
// (block (result null)
// (return)
// )
// (other))
//
// The true arm has a bottom type, but in fact it just returns out of the
// function and the null does not actually flow out. We can only optimize
// here if a null definitely flows out (as only that would cause a trap).
auto flowsOutNull = [&](Expression* child) {
return child->type.isNull() && !effects(child).transfersControlFlow();
};
if (auto* iff = ref->dynCast<If>()) {
if (iff->ifFalse) {
if (flowsOutNull(iff->ifTrue)) {
if (ref->type != iff->ifFalse->type) {
refinalize = true;
}
ref = builder.makeSequence(builder.makeDrop(iff->condition),
iff->ifFalse);
return false;
}
if (flowsOutNull(iff->ifFalse)) {
if (ref->type != iff->ifTrue->type) {
refinalize = true;
}
ref = builder.makeSequence(builder.makeDrop(iff->condition),
iff->ifTrue);
return false;
}
}
}
if (auto* select = ref->dynCast<Select>()) {
// We must check for unreachability explicitly here because a full
// refinalize only happens at the end. That is, the select may still be
// reachable after we turned one child into an unreachable, and we are
// calling getResultOfFirst which will error on unreachability.
if (flowsOutNull(select->ifTrue) &&
select->ifFalse->type != Type::unreachable) {
ref = builder.makeSequence(
builder.makeDrop(select->ifTrue),
getResultOfFirst(select->ifFalse,
builder.makeDrop(select->condition)));
return false;
}
if (flowsOutNull(select->ifFalse) &&
select->ifTrue->type != Type::unreachable) {
ref = getResultOfFirst(
select->ifTrue,
builder.makeSequence(builder.makeDrop(select->ifFalse),
builder.makeDrop(select->condition)));
return false;
}
}
}
// A nullable cast can be turned into a non-nullable one:
//
// (struct.get ;; or something else that traps on a null ref
// (ref.cast null
// =>
// (struct.get
// (ref.cast ;; now non-nullable
//
// Either way we trap here, but refining the type may have benefits later.
if (ref->type.isNullable()) {
if (auto* cast = ref->dynCast<RefCast>()) {
// Note that we must be the last child of the parent, otherwise effects
// in the middle may need to remain:
//
// (struct.set
// (ref.cast null
// (call ..
//
// The call here must execute before the trap in the struct.set. To
// avoid that problem, inspect all children after us. If there are no
// such children, then there is no problem; if there are, see below.
auto canOptimize = true;
auto seenRef = false;
for (auto* child : ChildIterator(curr)) {
if (child == ref) {
seenRef = true;
} else if (seenRef) {
// This is a child after the reference. Check it for effects. For
// simplicity, focus on the case of traps-never-happens: if we can
// assume no trap occurs in the parent, then there must not be a
// trap in the child either, unless control flow transfers and we
// might not reach the parent.
// TODO: handle more cases.
if (!getPassOptions().trapsNeverHappen ||
effects(child).transfersControlFlow()) {
canOptimize = false;
break;
}
}
}
if (canOptimize) {
cast->type = cast->type.with(NonNullable);
}
}
}
auto fallthrough = getFallthrough(ref);
if (fallthrough->type.isNull()) {
replaceCurrent(
getDroppedChildrenAndAppend(curr, builder.makeUnreachable()));
return true;
}
return false;
}
void visitRefEq(RefEq* curr) {
Builder builder(*getModule());
// Check for unreachability. Note we must check both the children and the
// ref.eq itself, as in e.g. optimizeTernary, as we only refinalize at the
// end, so unreachable children may not update the parent yet.
auto leftType = curr->left->type;
auto rightType = curr->right->type;
if (leftType == Type::unreachable || rightType == Type::unreachable ||
curr->type == Type::unreachable) {
// Leave this for DCE.
return;
}
// The types may prove that the same reference cannot appear on both sides.
auto leftHeapType = leftType.getHeapType();
auto rightHeapType = rightType.getHeapType();
auto leftIsHeapSubtype = HeapType::isSubType(leftHeapType, rightHeapType);
auto rightIsHeapSubtype = HeapType::isSubType(rightHeapType, leftHeapType);
if (!leftIsHeapSubtype && !rightIsHeapSubtype &&
(leftType.isNonNullable() || rightType.isNonNullable())) {
// The heap types have no intersection, so the only thing that can
// possibly appear on both sides is null, but one of the two is non-
// nullable, which rules that out. So there is no way that the same
// reference can appear on both sides.
replaceCurrent(
getDroppedChildrenAndAppend(curr, Literal::makeZero(Type::i32)));
return;
}
// Equality does not depend on the type, so casts may be removable.
//
// This is safe to do first because nothing farther down cares about the
// type, and we consume the two input references, so removing a cast could
// not help our parents (see "notes on removing casts").
Type nullableEq = Type(HeapType::eq, Nullable);
skipCast(curr->left, nullableEq);
skipCast(curr->right, nullableEq);
// Identical references compare equal. If the inputs are foldable, we need
// only keep one of them.
if (areConsecutiveInputsEqualAndFoldable(curr->left, curr->right)) {
replaceCurrent(
builder.makeSequence(builder.makeDrop(curr->left),
builder.makeConst(Literal::makeOne(Type::i32))));
return;
}
if (areConsecutiveInputsEqual(curr->left, curr->right)) {
replaceCurrent(
getDroppedChildrenAndAppend(curr, Literal::makeOne(Type::i32)));
return;
}
// Canonicalize to the pattern of a null on the right-hand side, if there is
// one. This makes pattern matching simpler.
if (curr->left->is<RefNull>()) {
std::swap(curr->left, curr->right);
}
// RefEq of a value to Null can be replaced with RefIsNull.
if (curr->right->is<RefNull>()) {
replaceCurrent(Builder(*getModule()).makeRefIsNull(curr->left));
}
}
void visitStructNew(StructNew* curr) {
if (curr->type == Type::unreachable) {
// Leave this for DCE.
return;
}
if (curr->desc) {
skipNonNullCast(curr->desc, curr);
trapOnNull(curr, curr->desc);
}
// If values are provided, but they are all the default, then we can remove
// them (in reachable code).
if (curr->isWithDefault()) {
return;
}
const auto& fields = curr->type.getHeapType().getStruct().fields;
assert(fields.size() == curr->operands.size());
for (Index i = 0; i < fields.size(); i++) {
// The field must be defaultable.
auto type = fields[i].type;
if (!type.isDefaultable()) {
return;
}
// The field must be written the default value.
auto* value = getFallthrough(curr->operands[i]);
if (!Properties::isSingleConstantExpression(value) ||
Properties::getLiteral(value) != Literal::makeZero(type)) {
return;
}
}
// Success! Drop the children and return a struct.new_with_default. We don't
// want the descriptor to be dropped, however, so temporarily remove it
// while we drop the other children. If the descriptor is null, this makes
// no difference.
auto* desc = curr->desc;
curr->desc = nullptr;
auto* rep = getDroppedChildrenAndAppend(curr, curr);
curr->operands.clear();
curr->desc = desc;
assert(curr->isWithDefault());
replaceCurrent(rep);
}
void visitStructGet(StructGet* curr) {
skipNonNullCast(curr->ref, curr);
trapOnNull(curr, curr->ref);
// Relax acquire loads of unshared fields to unordered because they cannot
// synchronize with other threads.
// TODO: Relax all other memory orderings as well.
if (curr->order == MemoryOrder::AcqRel && curr->ref->type.isRef() &&
!curr->ref->type.getHeapType().isShared()) {
curr->order = MemoryOrder::Unordered;
}
}
void visitStructSet(StructSet* curr) {
skipNonNullCast(curr->ref, curr);
if (trapOnNull(curr, curr->ref)) {
return;
}
if (curr->ref->type != Type::unreachable && curr->value->type.isInteger()) {
// We must avoid the case of a null type.
auto heapType = curr->ref->type.getHeapType();
if (heapType.isStruct()) {
const auto& fields = heapType.getStruct().fields;
optimizeStoredValue(curr->value, fields[curr->index].getByteSize());
}
}
// Relax release stores of unshared fields to unordered because they cannot
// synchronize with other threads.
// TODO: Relax all other memory orderings as well.
if (curr->order == MemoryOrder::AcqRel && curr->ref->type.isRef() &&
!curr->ref->type.getHeapType().isShared()) {
curr->order = MemoryOrder::Unordered;
}
}
void visitStructRMW(StructRMW* curr) {
skipNonNullCast(curr->ref, curr);
if (trapOnNull(curr, curr->ref)) {
return;
}
if (!curr->ref->type.isStruct()) {
return;
}
// We generally can't optimize seqcst RMWs on shared memory because they can
// act as both the source and sink of synchronization edges, even if they
// don't modify the in-memory value.
if (curr->ref->type.getHeapType().isShared() &&
curr->order == MemoryOrder::SeqCst) {
return;
}
Builder builder(*getModule());
// This RMW is either to non-shared memory or has acquire-release ordering.
// In the former case, it trivially does not synchronize with other threads
// and we can optimize to our heart's content. In the latter case, if we
// know the RMW does not change the value in memory, then we can consider
// all subsequent reads as reading from the previous write rather than from
// this RMW op, which means this RMW does not synchronize with later reads
// and we can optimize out the write part. This optimization wouldn't be
// valid for sequentially consistent RMW ops because the next reads from
// this location in the total order of seqcst ops would have to be
// considered to be reading from this RMW and therefore would synchronize
// with it.
auto* value = getFallthrough(curr->value);
if (Properties::isSingleConstantExpression(value)) {
auto val = Properties::getLiteral(value);
bool canOptimize = false;
switch (curr->op) {
case RMWAdd:
case RMWSub:
case RMWOr:
case RMWXor:
canOptimize = val.getInteger() == 0;
break;
case RMWAnd:
canOptimize = val == Literal::makeNegOne(val.type);
break;
case RMWXchg:
canOptimize = false;
break;
}
if (canOptimize) {
replaceCurrent(builder.makeStructGet(
curr->index,
getResultOfFirst(curr->ref, builder.makeDrop(curr->value)),
curr->order,
curr->type));
return;
}
}
// No further optimizations possible on RMWs to shared memory.
if (curr->ref->type.getHeapType().isShared()) {
return;
}
if (curr->type == Type::unreachable) {
// Leave this to DCE.
return;
}
// Lower the RMW to its more basic operations. Breaking the atomic
// operation into several non-atomic operations is safe because no other
// thread can observe an intermediate state in the unshared memory. This
// initially increases code size, but the more basic operations may be
// more optimizable than the original RMW.
// TODO: Experiment to determine whether this is worthwhile on real code.
// Maybe we should do this optimization only when optimizing for speed over
// size.
auto ref = builder.addVar(getFunction(), curr->ref->type);
auto val = builder.addVar(getFunction(), curr->type);
auto result = builder.addVar(getFunction(), curr->type);
auto* block = builder.makeBlock(
{builder.makeLocalSet(ref, curr->ref),
builder.makeLocalSet(val, curr->value),
builder.makeLocalSet(
result,
builder.makeStructGet(curr->index,
builder.makeLocalGet(ref, curr->ref->type),
MemoryOrder::Unordered,
curr->type))});
Expression* newVal = nullptr;
if (curr->op == RMWXchg) {
newVal = builder.makeLocalGet(val, curr->type);
} else {
Abstract::Op binop = Abstract::Add;
switch (curr->op) {
case RMWAdd:
binop = Abstract::Add;
break;
case RMWSub:
binop = Abstract::Sub;
break;
case RMWAnd:
binop = Abstract::And;
break;
case RMWOr:
binop = Abstract::Or;
break;
case RMWXor:
binop = Abstract::Xor;
break;
case RMWXchg:
WASM_UNREACHABLE("unexpected op");
}
newVal = builder.makeBinary(Abstract::getBinary(curr->type, binop),
builder.makeLocalGet(result, curr->type),
builder.makeLocalGet(val, curr->type));
}
block->list.push_back(
builder.makeStructSet(curr->index,
builder.makeLocalGet(ref, curr->ref->type),
newVal,
MemoryOrder::Unordered));
block->list.push_back(builder.makeLocalGet(result, curr->type));
block->type = curr->type;
replaceCurrent(block);
}
void visitStructCmpxchg(StructCmpxchg* curr) {
skipNonNullCast(curr->ref, curr);
if (trapOnNull(curr, curr->ref)) {
return;
}
if (!curr->ref->type.isStruct()) {
return;
}
Builder builder(*getModule());
// As with other RMW operations, we cannot optimize if the RMW is
// sequentially consistent and to shared memory.
if (curr->ref->type.getHeapType().isShared() &&
curr->order == MemoryOrder::SeqCst) {
return;
}
// Just like other RMW operations, unshared or release-acquire cmpxchg can
// be optimized to just a read if it is known not to change the in-memory
// value. This is the case when `expected` and `replacement` are known to be
// the same.
if (areConsecutiveInputsEqual(curr->expected, curr->replacement)) {
auto* ref = getResultOfFirst(
curr->ref,
builder.makeSequence(builder.makeDrop(curr->expected),
builder.makeDrop(curr->replacement)));
replaceCurrent(
builder.makeStructGet(curr->index, ref, curr->order, curr->type));
return;
}
if (curr->ref->type.getHeapType().isShared()) {
return;
}
if (curr->type == Type::unreachable) {
// Leave this to DCE.
return;
}
// Just like other RMW operations, lower to basic operations when operating
// on unshared memory.
auto ref = builder.addVar(getFunction(), curr->ref->type);
auto expectedType = curr->type;
if (expectedType.isRef()) {
expectedType =
Type(HeapTypes::eq.getBasic(expectedType.getHeapType().getShared()),
Nullable);
}
auto expected = builder.addVar(getFunction(), expectedType);
auto replacement = builder.addVar(getFunction(), curr->type);
auto result = builder.addVar(getFunction(), curr->type);
auto* block =
builder.makeBlock({builder.makeLocalSet(ref, curr->ref),
builder.makeLocalSet(expected, curr->expected),
builder.makeLocalSet(replacement, curr->replacement)});
auto* lhs = builder.makeLocalTee(
result,
builder.makeStructGet(curr->index,
builder.makeLocalGet(ref, curr->ref->type),
MemoryOrder::Unordered,
curr->type),
curr->type);
auto* rhs = builder.makeLocalGet(expected, expectedType);
Expression* pred = nullptr;
if (curr->type.isRef()) {
pred = builder.makeRefEq(lhs, rhs);
} else {
pred = builder.makeBinary(
Abstract::getBinary(curr->type, Abstract::Eq), lhs, rhs);
}
block->list.push_back(builder.makeIf(
pred,
builder.makeStructSet(curr->index,
builder.makeLocalGet(ref, curr->ref->type),
builder.makeLocalGet(replacement, curr->type),
MemoryOrder::Unordered)));
block->list.push_back(builder.makeLocalGet(result, curr->type));
block->type = curr->type;
replaceCurrent(block);
}
void visitArrayNew(ArrayNew* curr) {
// If a value is provided, we can optimize in some cases.
if (curr->type == Type::unreachable || curr->isWithDefault()) {
return;
}
Builder builder(*getModule());
// ArrayNew of size 1 is less efficient than ArrayNewFixed with one value
// (the latter avoids a Const, which ends up saving one byte).
// TODO: also look at the case with a fallthrough or effects on the size
if (auto* c = curr->size->dynCast<Const>()) {
if (c->value.geti32() == 1) {
// Optimize to ArrayNewFixed. Note that if the value is the default
// then we may end up optimizing further in visitArrayNewFixed.
replaceCurrent(
builder.makeArrayNewFixed(curr->type.getHeapType(), {curr->init}));
return;
}
}
// If the type is defaultable then perhaps the value here is the default.
auto type = curr->type.getHeapType().getArray().element.type;
if (!type.isDefaultable()) {
return;
}
// The value must be the default/zero.
auto zero = Literal::makeZero(type);
auto* value = getFallthrough(curr->init);
if (!Properties::isSingleConstantExpression(value) ||
Properties::getLiteral(value) != zero) {
return;
}
// Success! Drop the init and return an array.new_with_default.
auto* init = curr->init;
curr->init = nullptr;
assert(curr->isWithDefault());
replaceCurrent(builder.makeSequence(builder.makeDrop(init), curr));
}
void visitArrayNewFixed(ArrayNewFixed* curr) {
if (curr->type == Type::unreachable) {
return;
}
auto size = curr->values.size();
if (size == 0) {
// TODO: Consider what to do in the trivial case of an empty array: we can
// can use ArrayNew or ArrayNewFixed there. Measure which is best.
return;
}
// If all the values are equal then we can optimize, either to
// array.new_default (if they are all equal to the default) or array.new (if
// they are all equal to some other value). First, see if they are all
// equal, which we do by comparing in pairs: [0,1], then [1,2], etc.
for (Index i = 0; i < size - 1; i++) {
if (!areConsecutiveInputsEqual(curr->values[i], curr->values[i + 1])) {
return;
}
}
// Great, they are all equal!
Builder builder(*getModule());
// See if they are equal to a constant, and if that constant is the default.
auto type = curr->type.getHeapType().getArray().element.type;
if (type.isDefaultable()) {
auto* value = getFallthrough(curr->values[0]);
if (Properties::isSingleConstantExpression(value) &&
Properties::getLiteral(value) == Literal::makeZero(type)) {
// They are all equal to the default. Drop the children and return an
// array.new_with_default.
auto* withDefault = builder.makeArrayNew(
curr->type.getHeapType(), builder.makeConst(int32_t(size)));
replaceCurrent(getDroppedChildrenAndAppend(curr, withDefault));
return;
}
}
// They are all equal to each other, but not to the default value. If there
// are 2 or more elements here then we can save by using array.new. For
// example, with 2 elements we are doing this:
//
// (array.new_fixed
// (A)
// (A)
// )
// =>
// (array.new
// (A)
// (i32.const 2) ;; get two copies of (A)
// )
//
// However, with 1, ArrayNewFixed is actually more compact, and we optimize
// ArrayNew to it, above.
if (size == 1) {
return;
}
// Move children to locals, if we need to keep them around. We are removing
// them all, except from the first, when we remove the array.new_fixed's
// list of children and replace it with a single child + a constant for the
// number of children.
ChildLocalizer localizer(
curr, getFunction(), *getModule(), getPassOptions());
auto* block = localizer.getChildrenReplacement();
auto* arrayNew = builder.makeArrayNew(curr->type.getHeapType(),
builder.makeConst(int32_t(size)),
curr->values[0]);
block->list.push_back(arrayNew);
block->finalize();
replaceCurrent(block);
}
void visitArrayGet(ArrayGet* curr) {
skipNonNullCast(curr->ref, curr);
trapOnNull(curr, curr->ref);
}
void visitArraySet(ArraySet* curr) {
skipNonNullCast(curr->ref, curr);
if (trapOnNull(curr, curr->ref)) {
return;
}
if (curr->value->type.isInteger()) {
if (auto field = GCTypeUtils::getField(curr->ref->type)) {
optimizeStoredValue(curr->value, field->getByteSize());
}
}
}
void visitArrayLen(ArrayLen* curr) {
skipNonNullCast(curr->ref, curr);
trapOnNull(curr, curr->ref);
}
void visitArrayCopy(ArrayCopy* curr) {
skipNonNullCast(curr->destRef, curr);
skipNonNullCast(curr->srcRef, curr);
trapOnNull(curr, curr->destRef) || trapOnNull(curr, curr->srcRef);
}
void visitArrayRMW(ArrayRMW* curr) {
skipNonNullCast(curr->ref, curr);
trapOnNull(curr, curr->ref);
// TODO: more opts like StructRMW
}
void visitArrayCmpxchg(ArrayCmpxchg* curr) {
skipNonNullCast(curr->ref, curr);
trapOnNull(curr, curr->ref);
// TODO: more opts like StructCmpxchg
}
bool optimizeKnownCastResult(RefCast* curr, Type refType) {
Builder builder(*getModule());
// Given what we know about the type of the value, determine what we know
// about the results of the cast and optimize accordingly.
switch (GCTypeUtils::evaluateCastCheck(refType, curr->type)) {
case GCTypeUtils::Unknown:
// The cast may or may not succeed, so we cannot optimize.
return false;
case GCTypeUtils::Success:
case GCTypeUtils::SuccessOnlyIfNonNull: {
// Knowing the types match is not sufficient to know a descriptor cast
// succeeds. We must also know that the descriptor values match.
// However, if traps never happen, we can assume the descriptors will
// match and optimize anyway.
// TODO: Maybe we can determine that the descriptors values match in
// some cases.
if (curr->desc && !getPassOptions().trapsNeverHappen) {
// As a special case, we can still optimize if we know the value is
// null, because then we never get around to comparing the
// descriptors. We still need to preserve the trap on null
// descriptors, though.
if (refType.isNull()) {
assert(curr->type.isNullable());
if (curr->desc->type.isNullable()) {
curr->desc = builder.makeRefAs(RefAsNonNull, curr->desc);
}
replaceCurrent(getDroppedChildrenAndAppend(
curr,
builder.makeRefNull(curr->type.getHeapType()),
DropMode::IgnoreParentEffects));
return true;
}
return false;
}
// We know the cast will succeed, or at most requires a null check, so
// we can try to optimize it out. Find the best-typed fallthrough value
// to propagate.
auto** refp = Properties::getMostRefinedFallthrough(
&curr->ref, getPassOptions(), *getModule());
auto* ref = *refp;
assert(ref->type.isRef());
if (HeapType::isSubType(ref->type.getHeapType(),
curr->type.getHeapType())) {
// We know ref's heap type matches, but the knowledge that the
// nullabillity matches might come from somewhere else or we might not
// know at all whether the nullability matches, so we might need to
// emit a null check.
bool needsNullCheck = ref->type.getNullability() == Nullable &&
curr->type.getNullability() == NonNullable;
// Same with exactness.
bool needsExactCast = ref->type.getExactness() == Inexact &&
curr->type.getExactness() == Exact;
// If the best value to propagate is the argument to the cast, we can
// simply remove the cast (or downgrade it to a null check if
// necessary). This does not work if we need a cast to prove
// exactness.
if (ref == curr->ref && !needsExactCast) {
if (needsNullCheck) {
curr->ref = builder.makeRefAs(RefAsNonNull, curr->ref);
}
if (curr->desc) {
// We must move the ref past the descriptor operand.
auto* block =
ChildLocalizer(
curr, getFunction(), *getModule(), getPassOptions())
.getChildrenReplacement();
block->list.push_back(curr->ref);
block->type = curr->ref->type;
replaceCurrent(block);
} else {
replaceCurrent(curr->ref);
}
return true;
}
// Otherwise we can't just remove the cast and replace it with `ref`
// because the intermediate expressions might have had side effects or
// we need to check exactness. We can replace the cast with a drop
// followed by a direct return of the value, though.
if (ref->type.isNull()) {
// We can materialize the resulting null value directly.
//
// The type must be nullable for us to do that, which it normally
// would be, aside from the interesting corner case of
// uninhabitable types:
//
// (ref.cast (ref func)
// (block (result (ref nofunc))
// (unreachable)
// )
// )
//
// (ref nofunc) is a subtype of (ref func), so the cast might seem
// to be successful, but since the input is uninhabitable we won't
// even reach the cast. Such casts will be evaluated as
// Unreachable, so we'll not hit this assertion.
assert(curr->type.isNullable());
auto nullType = curr->type.getHeapType().getBottom();
replaceCurrent(
getDroppedChildrenAndAppend(curr,
builder.makeRefNull(nullType),
DropMode::IgnoreParentEffects));
return true;
}
// At this point we know the cast will succeed as long as nullability
// works out, but we still need the cast to recover the exactness that
// is not present in the value's static type, so there's nothing we
// can do.
if (needsExactCast) {
return false;
}
// We need to use a tee to return the value since we can't materialize
// it directly.
auto scratch = builder.addVar(getFunction(), ref->type);
*refp = builder.makeLocalTee(scratch, ref, ref->type);
Expression* get = builder.makeLocalGet(scratch, ref->type);
if (needsNullCheck) {
get = builder.makeRefAs(RefAsNonNull, get);
}
replaceCurrent(getDroppedChildrenAndAppend(
curr, get, DropMode::IgnoreParentEffects));
return true;
}
// If we get here, then we know that the heap type of the cast input is
// more refined than the heap type of the best available fallthrough
// expression. The only way this can happen is if we were able to infer
// that the input has bottom heap type because it was typed with
// multiple, incompatible heap types in different fallthrough
// expressions. For example:
//
// (ref.cast eqref
// (br_on_cast_fail $l anyref i31ref
// (br_on_cast_fail $l anyref structref
// ...)))
//
// In this case, the cast succeeds because the value must be null, so we
// can fall through to handle that case.
assert(Type::isSubType(refType, ref->type));
assert(refType.getHeapType().isBottom());
}
[[fallthrough]];
case GCTypeUtils::SuccessOnlyIfNull: {
// The cast either returns null or traps. In trapsNeverHappen mode
// we know the result, since by assumption it will not trap.
if (getPassOptions().trapsNeverHappen) {
replaceCurrent(getDroppedChildrenAndAppend(
curr,
builder.makeRefNull(curr->type.getHeapType()),
DropMode::IgnoreParentEffects));
return true;
}
return false;
}
case GCTypeUtils::Unreachable:
case GCTypeUtils::Failure:
// This cast cannot succeed, or it cannot even be reached, so we can
// trap.
replaceCurrent(getDroppedChildrenAndAppend(
curr, builder.makeUnreachable(), DropMode::IgnoreParentEffects));
return true;
}
WASM_UNREACHABLE("unexpected result");
}
void visitRefCast(RefCast* curr) {
// Note we must check the ref's type here and not our own, since we only
// refinalize at the end, which means our type may not have been updated yet
// after a change in the child.
// TODO: we could update unreachability up the stack perhaps, or just move
// all patterns that can add unreachability to a pass that does so
// already like vacuum or dce.
if (curr->ref->type == Type::unreachable ||
(curr->desc && curr->desc->type == Type::unreachable)) {
return;
}
if (curr->type.isNonNullable() && trapOnNull(curr, curr->ref)) {
return;
}
if (curr->desc) {
skipNonNullCast(curr->desc, curr);
if (trapOnNull(curr, curr->desc)) {
return;
}
}
Builder builder(*getModule());
// Look at all the fallthrough values to get the most precise possible type
// of the value we are casting.
Type refType = getFallthroughType(curr->ref);
// As a first step, we can tighten up the cast type. For normal casts we can
// use the greatest lower bound of the original cast type and the type we
// know the cast value to have. For descriptor casts we cannot change the
// target heap type because it is controlled by the descriptor operand, but
// we can improve nullability.
Type improvedType = curr->type;
if (curr->desc) {
if (curr->type.isNullable() && refType.isNonNullable()) {
improvedType = curr->type.with(NonNullable);
}
} else {
improvedType = Type::getGreatestLowerBound(curr->type, refType);
}
if (improvedType != Type::unreachable && improvedType != curr->type) {
curr->type = improvedType;
refinalize = true;
// Call replaceCurrent() to make us re-optimize this node, as we may
// have just unlocked further opportunities. (We could just continue
// down to the rest, but we'd need to do more work to make sure all the
// local state in this function is in sync which this change; it's
// easier to just do another clean pass on this node.)
replaceCurrent(curr);
return;
}
// Try to optimize based on what we know statically about the result of the
// cast.
if (optimizeKnownCastResult(curr, refType)) {
return;
}
// If we got past the optimizations above, it must be the case that we
// cannot tell statically whether the cast will succeed or not.
if (auto* child = curr->ref->dynCast<RefCast>()) {
// If the current cast is at least as strong as the child cast, then we
// can remove the child cast. If the child cast is a descriptor cast and
// traps are allowed, then we cannot remove the potentially-trapping
// child, though.
bool notWeaker = Type::isSubType(curr->type, child->type);
auto& options = getPassOptions();
auto canTrap = !options.trapsNeverHappen;
bool safe = !child->desc || !canTrap;
bool canOptimize = notWeaker && safe;
if (canOptimize && curr->desc && canTrap) {
// There is another child here, which might trap, and we need to
// consider that in this situation:
//
// (outer.cast
// (inner.cast (inner.ref))
// (descriptor with effects)
// )
//
// =>
//
// (outer.cast
// (inner.ref) ;; inner cast was removed
// (descriptor with effects)
// )
//
// It is safe to remove the inner cast, as if it trapped, the outer one
// would still trap. But if there is a descriptor, then we are moving
// the trap across the descriptor, and shouldn't cross effects there.
EffectAnalyzer descEffects(options, *getModule(), curr->desc);
ShallowEffectAnalyzer movingEffects(options, *getModule(), curr->ref);
if (movingEffects.orderedBefore(descEffects)) {
canOptimize = false;
}
}
if (canOptimize) {
if (child->desc) {
// Reorder the child's reference past its dropped descriptor if
// necessary.
auto* block =
ChildLocalizer(child, getFunction(), *getModule(), options)
.getChildrenReplacement();
block->list.push_back(child->ref);
block->type = child->ref->type;
curr->ref = block;
} else {
curr->ref = child->ref;
}
return;
}
}
// Similarly, ref.cast can be combined with ref.as_non_null.
//
// (ref.cast (ref null T) (ref.as_non_null ..))
// =>
// (ref.cast (ref T) ..)
//
if (auto* as = curr->ref->dynCast<RefAs>(); as && as->op == RefAsNonNull) {
if (curr->desc) {
// There is another child here, whose effects we must consider (the same
// ordering situation as in skipNonNullCast: we want to move a trap on
// null past later children).
auto& options = getPassRunner()->options;
EffectAnalyzer descEffects(options, *getModule(), curr->desc);
ShallowEffectAnalyzer movingEffects(options, *getModule(), curr->ref);
if (movingEffects.orderedBefore(descEffects)) {
return;
}
}
curr->ref = as->value;
curr->type = curr->type.with(NonNullable);
}
}
void visitRefTest(RefTest* curr) {
if (curr->type == Type::unreachable) {
return;
}
Builder builder(*getModule());
// Parallel to the code in visitRefCast: we look not just at the final type
// we are given, but at fallthrough values as well.
Type refType = getFallthroughType(curr->ref);
// Improve the cast type as much as we can without changing the results.
auto glb = Type::getGreatestLowerBound(curr->castType, refType);
if (glb != Type::unreachable && glb != curr->castType) {
curr->castType = glb;
}
switch (GCTypeUtils::evaluateCastCheck(refType, curr->castType)) {
case GCTypeUtils::Unknown:
break;
case GCTypeUtils::Success:
replaceCurrent(builder.makeBlock(
{builder.makeDrop(curr->ref), builder.makeConst(int32_t(1))}));
return;
case GCTypeUtils::Unreachable:
// Make sure to emit a block with the same type as us, to avoid other
// code in this pass needing to handle unexpected unreachable code
// (which is only properly propagated at the end of this pass when we
// refinalize).
replaceCurrent(builder.makeBlock(
{builder.makeDrop(curr->ref), builder.makeUnreachable()}, Type::i32));
return;
case GCTypeUtils::Failure:
replaceCurrent(builder.makeSequence(builder.makeDrop(curr->ref),
builder.makeConst(int32_t(0))));
return;
case GCTypeUtils::SuccessOnlyIfNull:
replaceCurrent(builder.makeRefIsNull(curr->ref));
return;
case GCTypeUtils::SuccessOnlyIfNonNull:
// This adds an EqZ, but code size does not regress since ref.test
// also encodes a type, and ref.is_null does not. The EqZ may also add
// some work, but a cast is likely more expensive than a null check +
// a fast int operation.
replaceCurrent(
builder.makeUnary(EqZInt32, builder.makeRefIsNull(curr->ref)));
return;
}
}
void visitBrOn(BrOn* curr) {
if (curr->desc) {
skipNonNullCast(curr->desc, curr);
trapOnNull(curr, curr->desc);
}
}
void visitRefIsNull(RefIsNull* curr) {
if (curr->type == Type::unreachable) {
return;
}
// Optimizing RefIsNull is not that obvious, since even if we know the
// result evaluates to 0 or 1 then the replacement may not actually save
// code size, since RefIsNull is a single byte while adding a Const of 0
// would be two bytes. Other factors are that we can remove the input and
// the added drop on it if it has no side effects, and that replacing with a
// constant may allow further optimizations later. For now, replace with a
// constant, but this warrants more investigation. TODO
Builder builder(*getModule());
if (curr->value->type.isNonNullable()) {
replaceCurrent(
builder.makeSequence(builder.makeDrop(curr->value),
builder.makeConst(Literal::makeZero(Type::i32))));
} else {
skipCast(curr->value);
}
}
void visitRefAs(RefAs* curr) {
if (curr->type == Type::unreachable) {
return;
}
if (curr->op == ExternConvertAny || curr->op == AnyConvertExtern) {
// These pass nulls through, and we can reorder them with null traps:
//
// (any.convert_extern/extern.convert_any (ref.as_non_null.. ))
// =>
// (ref.as_non_null (any.convert_extern/extern.convert_any ..))
//
// By moving the RefAsNonNull outside, it may reach a position where it
// can be optimized (e.g. if the parent traps anyhow). And,
// ExternConvertAny/AnyConvertExtern cannot be folded with anything, so
// there is no harm to moving them inside.
if (auto* refAsChild = curr->value->dynCast<RefAs>()) {
if (refAsChild->op == RefAsNonNull) {
// Reorder and fix up the types.
curr->value = refAsChild->value;
curr->finalize();
refAsChild->value = curr;
refAsChild->finalize();
replaceCurrent(refAsChild);
return;
}
// We can optimize away externalizations of internalizations and vice
// versa.
if ((curr->op == ExternConvertAny &&
refAsChild->op == AnyConvertExtern) ||
(curr->op == AnyConvertExtern &&
refAsChild->op == ExternConvertAny)) {
replaceCurrent(refAsChild->value);
return;
}
}
return;
}
assert(curr->op == RefAsNonNull);
if (trapOnNull(curr, curr->value)) {
return;
}
skipNonNullCast(curr->value, curr);
if (!curr->value->type.isNullable()) {
replaceCurrent(curr->value);
return;
}
// As we do in visitRefCast, ref.cast can be combined with ref.as_non_null.
// This code handles the case where the ref.as is on the outside:
//
// (ref.as_non_null (ref.cast null ..))
// =>
// (ref.cast ..)
//
if (auto* cast = curr->value->dynCast<RefCast>()) {
// The cast cannot be non-nullable, or we would have handled this right
// above by just removing the ref.as, since it would not be needed.
assert(!cast->type.isNonNullable());
cast->type = cast->type.with(NonNullable);
replaceCurrent(cast);
}
}
void visitRefGetDesc(RefGetDesc* curr) {
skipNonNullCast(curr->ref, curr);
trapOnNull(curr, curr->ref);
}
void visitPublish(Publish* curr) {
if (curr->type == Type::unreachable) {
return;
}
// Publish of a reference that cannot be a shared array or struct can be
// removed.
auto canBeSharedArrayOrStruct = [](Type type) {
if (!type.isRef()) {
return false;
}
auto ht = type.getHeapType();
if (ht.isBottom() || ht.isMaybeShared(HeapType::i31)) {
return false;
}
return HeapType::isSubType(ht, HeapTypes::any.getBasic(Shared));
};
if (!canBeSharedArrayOrStruct(getFallthroughType(curr->ref))) {
replaceCurrent(curr->ref);
return;
}
auto isAllocation = [](Expression* expr) {
return expr->is<StructNew>() || expr->is<ArrayNew>() ||
expr->is<ArrayNewData>() || expr->is<ArrayNewElem>() ||
expr->is<ArrayNewFixed>();
};
// Publish of publish or of a struct/array allocation can be removed.
Expression* fallthrough = curr->ref;
while (true) {
if (fallthrough->is<Publish>() || isAllocation(fallthrough)) {
replaceCurrent(curr->ref);
return;
}
auto* next = Properties::getImmediateFallthrough(
fallthrough, getPassOptions(), *getModule());
if (next == fallthrough) {
break;
}
fallthrough = next;
}
}
void visitTupleExtract(TupleExtract* curr) {
if (curr->type == Type::unreachable) {
return;
}
if (auto* make = curr->tuple->dynCast<TupleMake>()) {
Builder builder(*getModule());
// Store the value of the lane we want in a tee, and return that after a
// drop of the tuple (which might have side effects).
auto valueType = make->type[curr->index];
Index tempLocal = builder.addVar(getFunction(), valueType);
make->operands[curr->index] =
builder.makeLocalTee(tempLocal, make->operands[curr->index], valueType);
auto* get = builder.makeLocalGet(tempLocal, valueType);
replaceCurrent(getDroppedChildrenAndAppend(make, get));
}
}
Index getMaxBitsForLocal(LocalGet* get) {
// check what we know about the local (we may know nothing, if this local
// was added after the pass scanned for locals; in that case, full
// optimization may require another cycle)
if (get->index < localInfo.size()) {
return localInfo[get->index].maxBits;
}
return getBitsForType(get->type);
}
private:
// Information about our locals
std::vector<LocalInfo> localInfo;
// Checks if the first is a local.tee and the second a local.get of that same
// index. This is useful in the methods right below us, as it is a common
// pattern where two consecutive inputs are equal despite being syntactically
// different.
bool areMatchingTeeAndGet(Expression* left, Expression* right) {
if (auto* set = left->dynCast<LocalSet>()) {
if (auto* get = right->dynCast<LocalGet>()) {
if (set->isTee() && get->index == set->index) {
return true;
}
}
}
return false;
}
// Check if two consecutive inputs to an instruction are equal. As they are
// consecutive, no code can execute in between them, which simplifies the
// problem here (and which is the case we care about in this pass, which does
// simple peephole optimizations - all we care about is a single instruction
// at a time, and its inputs).
bool areConsecutiveInputsEqual(Expression* left, Expression* right) {
// The fallthrough expression of `left` produces its value. That value may
// depend on effects from other non-fallthrough expressions in `left`, but
// those expressions are generally executed before the fallthrough value and
// will affect the values of `left` and `right` equally, so we can ignore
// them. The exceptions are `local.tee` instructions and br_if conditions,
// which execute after the fallthrough and might affect only the value of
// `right`.
// TODO: We should use a custom getFallthrough that ignores whether br_if
// conditions and values can be reordered, since we can handle that more
// precisely here.
// TODO: When the fallthrough is an If (meaning the other branch must never
// return), we should ignore effects in that non-returning branch.
EffectAnalyzer interferingEffects(getPassOptions(), *getModule());
bool matchingTeeAndGet = false;
while (true) {
left =
Properties::getFallthrough(left,
getPassOptions(),
*getModule(),
Properties::FallthroughBehavior::NoTeeBrIf);
if (auto* tee = left->dynCast<LocalSet>()) {
assert(tee->isTee());
// If `right` reads directly from this local.tee, then we know their
// values are the same. We know no children of this tee will be executed
// after it, so we need not look for further effects. But there might be
// interfering sets in previous br_if conditions, so we cannot just
// return here.
// TODO: Calculate `right`'s fallthrough first in case the fallthrough
// is the matching get.
if (areMatchingTeeAndGet(left, right)) {
matchingTeeAndGet = true;
left = getFallthrough(left);
break;
}
interferingEffects.visit(tee);
left = tee->value;
continue;
}
if (auto* br = left->dynCast<Break>(); br && br->condition) {
assert(br->value);
// NB: We don't need to worry about the branch effect because any branch
// at runtime must skip past the parent expression, so it would not
// matter how that parent expression gets optimized.
interferingEffects.walk(br->condition);
left = br->value;
continue;
}
if (auto* cast = left->dynCast<RefCast>(); cast && cast->desc) {
interferingEffects.walk(cast->desc);
left = cast->ref;
continue;
}
if (auto* br = left->dynCast<BrOn>(); br && br->desc) {
interferingEffects.walk(br->desc);
left = br->ref;
continue;
}
// We have found the real fallthrough expression.
break;
}
// We similarly want to find the fallthrough expression of `right`. But this
// time, it is the expressions that execute before, not after, the
// fallthrough that can affect its value.
while (true) {
auto* next = Properties::getImmediateFallthrough(
right, getPassOptions(), *getModule());
if (next == right) {
// We have found the fallthrough expression.
break;
}
// Gather the effects of all the non-fallthrough children of the
// container.
for (auto* child : ChildIterator(right)) {
if (child == next) {
// Skip children that execute after the fallthrough value, such as
// br_if conditions.
break;
}
interferingEffects.walk(child);
}
right = next;
}
// We have both fallthrough expressions. See if they look the same.
if (!matchingTeeAndGet && !ExpressionAnalyzer::equal(left, right)) {
return false;
}
// They do look the same! Determine whether there are side effects that
// could make the values different. Note that we use the same
// EffectAnalyzers for effects from the left and right sides because they
// are the same.
EffectAnalyzer fallthroughChildEffects(getPassOptions(), *getModule());
for (auto* child : ChildIterator(right)) {
fallthroughChildEffects.walk(child);
}
EffectAnalyzer fallthroughEffects = fallthroughChildEffects;
fallthroughEffects.visit(right);
// Does the left-hand parent expression or side effects in the right-hand
// children affect the values flowing into the right-hand parent expression?
// Do not consider the right-hand parent expression because it could be an
// idempotent call and we don't want to mistakenly conclude that the
// left-hand side could interfere with it. (Non-idempotent functions will
// be rejected by the `isGenerative` check below.)
if (fallthroughEffects.orderedBefore(fallthroughChildEffects)) {
return false;
}
// Do any effects executed between the expressions affect the right-hand
// expression?
if (interferingEffects.orderedBefore(fallthroughEffects)) {
return false;
}
// To be equal, they must also be known to return the same result
// deterministically. We check the right side, as if the right is marked
// idempotent, that is enough (that tells us it does not generate a new
// value; logically, of course, as left is equal to right, they are calling
// the same thing, so it is odd to only annotate one, but this is consistent
// and easy to check).
return !Properties::isGenerative(right, getFunction(), *getModule());
}
// Check if two consecutive inputs to an instruction are equal and can also be
// folded into the first of the two (but we do not assume the caller will
// always fold them).
//
// The inputs here must be consecutive, but it is also ok to have code with no
// side effects at all in the middle. For example, a Const in between is ok.
bool areConsecutiveInputsEqualAndFoldable(Expression* left,
Expression* right) {
// TODO: We could probably consider fallthrough values for left, at least
// (since we fold into it).
if (areMatchingTeeAndGet(left, right)) {
return true;
}
// To fold the right side into the left, it must have no unremovable
// effects.
auto rightMightHaveEffects = true;
if (auto* call = right->dynCast<Call>()) {
// If these are a pair of idempotent calls, then the second has no
// effects. (We didn't check if left is a call, but the equality check
// below does that.)
if (Intrinsics(*getModule())
.getCallAnnotations(call, getFunction())
.idempotent) {
// We must still check for effects in the parameters. Imagine that we
// have
//
// (call $idempotent (global.get $g))
// (call $idempotent (global.get $g))
//
// Then the first call has effects, and those might alter $g if the
// global is mutable. That is, all that idempotency tells us is that
// the second call has no effects, but its parameters can still have
// read effects that interact. Also, the parameter might have write
// effects,
//
// (call $idempotent (call $other))
//
// We must check that as well.
EffectAnalyzer childEffects(getPassOptions(), *getModule());
for (auto* child : call->operands) {
childEffects.walk(child);
}
if (childEffects.hasUnremovableSideEffects()) {
return false;
}
ShallowEffectAnalyzer parentEffects(
getPassOptions(), *getModule(), call);
// Check if the first parent is ordered before the second child (in the
// example above, the first call vs the second global.get).
if (parentEffects.orderedBefore(childEffects)) {
return false;
}
// No effects are possible.
rightMightHaveEffects = false;
}
}
if (rightMightHaveEffects) {
// So far it looks like right has effects, so check fully.
if (EffectAnalyzer(getPassOptions(), *getModule(), right)
.hasUnremovableSideEffects()) {
return false;
}
}
return areConsecutiveInputsEqual(left, right);
}
// Canonicalizing the order of a symmetric binary helps us
// write more concise pattern matching code elsewhere.
void canonicalize(Binary* binary) {
assert(shouldCanonicalize(binary));
if (neverReorder) {
return;
}
auto swap = [&]() {
assert(canReorder(binary->left, binary->right));
if (binary->isRelational()) {
binary->op = reverseRelationalOp(binary->op);
}
std::swap(binary->left, binary->right);
};
auto maybeSwap = [&]() {
if (canReorder(binary->left, binary->right)) {
swap();
}
};
// Prefer a const on the right.
if (binary->left->is<Const>() && !binary->right->is<Const>()) {
swap();
}
if (auto* c = binary->right->dynCast<Const>()) {
// x - C ==> x + (-C)
// Prefer use addition if there is a constant on the right.
if (binary->op == Abstract::getBinary(c->type, Abstract::Sub)) {
c->value = c->value.neg();
binary->op = Abstract::getBinary(c->type, Abstract::Add);
return;
}
// Prefer to compare to 0 instead of to -1 or 1.
// (signed)x > -1 ==> x >= 0
if (binary->op == Abstract::getBinary(c->type, Abstract::GtS) &&
c->value.getInteger() == -1LL) {
binary->op = Abstract::getBinary(c->type, Abstract::GeS);
c->value = Literal::makeZero(c->type);
return;
}
// (signed)x <= -1 ==> x < 0
if (binary->op == Abstract::getBinary(c->type, Abstract::LeS) &&
c->value.getInteger() == -1LL) {
binary->op = Abstract::getBinary(c->type, Abstract::LtS);
c->value = Literal::makeZero(c->type);
return;
}
// (signed)x < 1 ==> x <= 0
if (binary->op == Abstract::getBinary(c->type, Abstract::LtS) &&
c->value.getInteger() == 1LL) {
binary->op = Abstract::getBinary(c->type, Abstract::LeS);
c->value = Literal::makeZero(c->type);
return;
}
// (signed)x >= 1 ==> x > 0
if (binary->op == Abstract::getBinary(c->type, Abstract::GeS) &&
c->value.getInteger() == 1LL) {
binary->op = Abstract::getBinary(c->type, Abstract::GtS);
c->value = Literal::makeZero(c->type);
return;
}
// (unsigned)x < 1 ==> x == 0
if (binary->op == Abstract::getBinary(c->type, Abstract::LtU) &&
c->value.getInteger() == 1LL) {
binary->op = Abstract::getBinary(c->type, Abstract::Eq);
c->value = Literal::makeZero(c->type);
return;
}
// (unsigned)x >= 1 ==> x != 0
if (binary->op == Abstract::getBinary(c->type, Abstract::GeU) &&
c->value.getInteger() == 1LL) {
binary->op = Abstract::getBinary(c->type, Abstract::Ne);
c->value = Literal::makeZero(c->type);
return;
}
// Prefer compare to signed min (s_min) instead of s_min + 1.
// (signed)x < s_min + 1 ==> x == s_min
if (binary->op == LtSInt32 && c->value.geti32() == INT32_MIN + 1) {
binary->op = EqInt32;
c->value = Literal::makeSignedMin(Type::i32);
return;
}
if (binary->op == LtSInt64 && c->value.geti64() == INT64_MIN + 1) {
binary->op = EqInt64;
c->value = Literal::makeSignedMin(Type::i64);
return;
}
// (signed)x >= s_min + 1 ==> x != s_min
if (binary->op == GeSInt32 && c->value.geti32() == INT32_MIN + 1) {
binary->op = NeInt32;
c->value = Literal::makeSignedMin(Type::i32);
return;
}
if (binary->op == GeSInt64 && c->value.geti64() == INT64_MIN + 1) {
binary->op = NeInt64;
c->value = Literal::makeSignedMin(Type::i64);
return;
}
// Prefer compare to signed max (s_max) instead of s_max - 1.
// (signed)x > s_max - 1 ==> x == s_max
if (binary->op == GtSInt32 && c->value.geti32() == INT32_MAX - 1) {
binary->op = EqInt32;
c->value = Literal::makeSignedMax(Type::i32);
return;
}
if (binary->op == GtSInt64 && c->value.geti64() == INT64_MAX - 1) {
binary->op = EqInt64;
c->value = Literal::makeSignedMax(Type::i64);
return;
}
// (signed)x <= s_max - 1 ==> x != s_max
if (binary->op == LeSInt32 && c->value.geti32() == INT32_MAX - 1) {
binary->op = NeInt32;
c->value = Literal::makeSignedMax(Type::i32);
return;
}
if (binary->op == LeSInt64 && c->value.geti64() == INT64_MAX - 1) {
binary->op = NeInt64;
c->value = Literal::makeSignedMax(Type::i64);
return;
}
// Prefer compare to unsigned max (u_max) instead of u_max - 1.
// (unsigned)x <= u_max - 1 ==> x != u_max
if (binary->op == Abstract::getBinary(c->type, Abstract::LeU) &&
c->value.getInteger() == (int64_t)(UINT64_MAX - 1)) {
binary->op = Abstract::getBinary(c->type, Abstract::Ne);
c->value = Literal::makeUnsignedMax(c->type);
return;
}
// (unsigned)x > u_max - 1 ==> x == u_max
if (binary->op == Abstract::getBinary(c->type, Abstract::GtU) &&
c->value.getInteger() == (int64_t)(UINT64_MAX - 1)) {
binary->op = Abstract::getBinary(c->type, Abstract::Eq);
c->value = Literal::makeUnsignedMax(c->type);
return;
}
return;
}
// Prefer a get on the right.
if (binary->left->is<LocalGet>() && !binary->right->is<LocalGet>()) {
return maybeSwap();
}
// Sort by the node id type, if different.
if (binary->left->_id != binary->right->_id) {
if (binary->left->_id > binary->right->_id) {
return maybeSwap();
}
return;
}
// If the children have the same node id, we have to go deeper.
if (auto* left = binary->left->dynCast<Unary>()) {
auto* right = binary->right->cast<Unary>();
if (left->op > right->op) {
return maybeSwap();
}
}
if (auto* left = binary->left->dynCast<Binary>()) {
auto* right = binary->right->cast<Binary>();
if (left->op > right->op) {
return maybeSwap();
}
}
if (auto* left = binary->left->dynCast<LocalGet>()) {
auto* right = binary->right->cast<LocalGet>();
if (left->index > right->index) {
return maybeSwap();
}
}
}
// Optimize given that the expression is flowing into a boolean context
Expression* optimizeBoolean(Expression* boolean) {
// TODO use a general getFallthroughs
if (auto* unary = boolean->dynCast<Unary>()) {
if (unary) {
if (unary->op == EqZInt32) {
auto* unary2 = unary->value->dynCast<Unary>();
if (unary2 && unary2->op == EqZInt32) {
// double eqz
return unary2->value;
}
if (auto* binary = unary->value->dynCast<Binary>()) {
// !(x <=> y) ==> x <!=> y
auto op = invertBinaryOp(binary->op);
if (op != InvalidBinary) {
binary->op = op;
return binary;
}
}
}
if (unary->op == EqZInt32 || unary->op == EqZInt64) {
if (auto* c = getFallthrough(unary->value)->dynCast<Const>()) {
return getDroppedChildrenAndAppend(
unary, Literal::makeFromInt32(c->value.isZero(), Type::i32));
}
}
}
} else if (auto* binary = boolean->dynCast<Binary>()) {
if (binary->op == SubInt32) {
if (auto* c = binary->left->dynCast<Const>()) {
if (c->value.geti32() == 0) {
// bool(0 - x) ==> bool(x)
return binary->right;
}
}
} else if (binary->op == OrInt32) {
// an or flowing into a boolean context can consider each input as
// boolean
binary->left = optimizeBoolean(binary->left);
binary->right = optimizeBoolean(binary->right);
} else if (binary->op == NeInt32) {
if (auto* c = binary->right->dynCast<Const>()) {
// x != 0 is just x if it's used as a bool
if (c->value.geti32() == 0) {
return binary->left;
}
// TODO: Perhaps use it for separate final pass???
// x != -1 ==> x ^ -1
// if (num->value.geti32() == -1) {
// binary->op = XorInt32;
// return binary;
// }
}
} else if (binary->op == RemSInt32) {
// bool(i32(x) % C_pot) ==> bool(x & (C_pot - 1))
// bool(i32(x) % min_s) ==> bool(x & max_s)
if (auto* c = binary->right->dynCast<Const>()) {
if (c->value.isSignedMin() ||
Bits::isPowerOf2(c->value.abs().geti32())) {
binary->op = AndInt32;
if (c->value.isSignedMin()) {
c->value = Literal::makeSignedMax(Type::i32);
} else {
c->value = c->value.abs().sub(Literal::makeOne(Type::i32));
}
return binary;
}
}
}
if (auto* ext = Properties::getSignExtValue(binary)) {
// use a cheaper zero-extent, we just care about the boolean value
// anyhow
return makeZeroExt(ext, Properties::getSignExtBits(binary));
}
} else if (auto* block = boolean->dynCast<Block>()) {
if (block->type == Type::i32 && block->list.size() > 0) {
block->list.back() = optimizeBoolean(block->list.back());
}
} else if (auto* iff = boolean->dynCast<If>()) {
if (iff->type == Type::i32) {
iff->ifTrue = optimizeBoolean(iff->ifTrue);
iff->ifFalse = optimizeBoolean(iff->ifFalse);
}
} else if (auto* select = boolean->dynCast<Select>()) {
select->ifTrue = optimizeBoolean(select->ifTrue);
select->ifFalse = optimizeBoolean(select->ifFalse);
} else if (auto* tryy = boolean->dynCast<Try>()) {
if (tryy->type == Type::i32) {
tryy->body = optimizeBoolean(tryy->body);
for (Index i = 0; i < tryy->catchBodies.size(); i++) {
tryy->catchBodies[i] = optimizeBoolean(tryy->catchBodies[i]);
}
}
}
// TODO: recurse into br values?
return boolean;
}
Expression* optimizeSelect(Select* curr) {
using namespace Match;
using namespace Abstract;
Builder builder(*getModule());
curr->condition = optimizeBoolean(curr->condition);
{
// Constant condition, we can just pick the correct side (barring side
// effects)
Expression *ifTrue, *ifFalse;
if (matches(curr, select(pure(&ifTrue), any(&ifFalse), i32(0)))) {
return ifFalse;
}
if (matches(curr, select(any(&ifTrue), any(&ifFalse), i32(0)))) {
return builder.makeSequence(builder.makeDrop(ifTrue), ifFalse);
}
int32_t cond;
if (matches(curr, select(any(&ifTrue), pure(&ifFalse), i32(&cond)))) {
// The condition must be non-zero because a zero would have matched one
// of the previous patterns.
assert(cond != 0);
return ifTrue;
}
// Don't bother when `ifFalse` isn't pure - we would need to reverse the
// order using a temp local, which would be bad
}
{
// TODO: Remove this after landing SCCP pass. See: #4161
// i32(x) ? i32(x) : 0 ==> x
Expression *x, *y;
if (matches(curr, select(any(&x), i32(0), any(&y))) &&
areConsecutiveInputsEqualAndFoldable(x, y)) {
return curr->ifTrue;
}
// i32(x) ? 0 : i32(x) ==> { x, 0 }
if (matches(curr, select(i32(0), any(&x), any(&y))) &&
areConsecutiveInputsEqualAndFoldable(x, y)) {
return builder.makeSequence(builder.makeDrop(x), curr->ifTrue);
}
// i64(x) == 0 ? 0 : i64(x) ==> x
// i64(x) != 0 ? i64(x) : 0 ==> x
if ((matches(curr, select(i64(0), any(&x), unary(EqZInt64, any(&y)))) ||
matches(
curr,
select(any(&x), i64(0), binary(NeInt64, any(&y), i64(0))))) &&
areConsecutiveInputsEqualAndFoldable(x, y)) {
return curr->condition->is<Unary>() ? curr->ifFalse : curr->ifTrue;
}
// i64(x) == 0 ? i64(x) : 0 ==> { x, 0 }
// i64(x) != 0 ? 0 : i64(x) ==> { x, 0 }
if ((matches(curr, select(any(&x), i64(0), unary(EqZInt64, any(&y)))) ||
matches(
curr,
select(i64(0), any(&x), binary(NeInt64, any(&y), i64(0))))) &&
areConsecutiveInputsEqualAndFoldable(x, y)) {
return builder.makeSequence(
builder.makeDrop(x),
curr->condition->is<Unary>() ? curr->ifFalse : curr->ifTrue);
}
}
{
// Simplify selects between 0 and 1
Expression* c;
bool reversed = matches(curr, select(ival(0), ival(1), any(&c)));
if (reversed || matches(curr, select(ival(1), ival(0), any(&c)))) {
if (reversed) {
c = optimizeBoolean(builder.makeUnary(EqZInt32, c));
}
if (!Properties::emitsBoolean(c)) {
// cond ? 1 : 0 ==> !!cond
c = builder.makeUnary(EqZInt32, builder.makeUnary(EqZInt32, c));
}
return curr->type == Type::i64 ? builder.makeUnary(ExtendUInt32, c) : c;
}
}
// Flip the arms if doing so might help later optimizations here.
if (auto* binary = curr->condition->dynCast<Binary>()) {
auto inv = invertBinaryOp(binary->op);
if (inv != InvalidBinary) {
// For invertible binary operations, we prefer to have non-zero values
// in the ifTrue, and zero values in the ifFalse, due to the
// optimization right after us. Even if this does not help there, it is
// a nice canonicalization. (To ensure convergence - that we don't keep
// doing work each time we get here - do nothing if both are zero, or
// if both are nonzero.)
Const* c;
if ((matches(curr->ifTrue, ival(0)) &&
!matches(curr->ifFalse, ival(0))) ||
(!matches(curr->ifTrue, ival()) &&
matches(curr->ifFalse, ival(&c)) && !c->value.isZero())) {
binary->op = inv;
std::swap(curr->ifTrue, curr->ifFalse);
}
}
}
if (curr->type == Type::i32 &&
Bits::getMaxBits(curr->condition, this) <= 1 &&
Bits::getMaxBits(curr->ifTrue, this) <= 1 &&
Bits::getMaxBits(curr->ifFalse, this) <= 1) {
// The condition and both arms are i32 booleans, which allows us to do
// boolean optimizations.
Expression* x;
Expression* y;
// x ? y : 0 ==> x & y
if (matches(curr, select(any(&y), ival(0), any(&x)))) {
return builder.makeBinary(AndInt32, y, x);
}
// x ? 1 : y ==> x | y
if (matches(curr, select(ival(1), any(&y), any(&x)))) {
return builder.makeBinary(OrInt32, y, x);
}
}
{
// Simplify x < 0 ? -1 : 1 or x >= 0 ? 1 : -1 to
// i32(x) >> 31 | 1
// i64(x) >> 63 | 1
Binary* bin;
if (matches(
curr,
select(ival(-1), ival(1), binary(&bin, LtS, any(), ival(0)))) ||
matches(
curr,
select(ival(1), ival(-1), binary(&bin, GeS, any(), ival(0))))) {
auto c = bin->right->cast<Const>();
auto type = curr->ifTrue->type;
if (type == c->type) {
bin->type = type;
bin->op = Abstract::getBinary(type, ShrS);
c->value = Literal::makeFromInt32(type.getByteSize() * 8 - 1, type);
curr->ifTrue->cast<Const>()->value = Literal::makeOne(type);
return builder.makeBinary(
Abstract::getBinary(type, Or), bin, curr->ifTrue);
}
}
}
{
// Flip select to remove eqz if we can reorder
Select* s;
Expression *ifTrue, *ifFalse, *c;
if (matches(
curr,
select(
&s, any(&ifTrue), any(&ifFalse), unary(EqZInt32, any(&c)))) &&
canReorder(ifTrue, ifFalse)) {
s->ifTrue = ifFalse;
s->ifFalse = ifTrue;
s->condition = c;
return s;
}
}
{
// If sides are identical, fold.
Expression *ifTrue, *ifFalse, *c;
// Note we do not compare metadata here: This is a select, so both arms
// execute anyhow, and things like branch hints were already being run.
// After optimization, we will only run fewer things, and run no risk of
// running new bad things.
if (matches(curr, select(any(&ifTrue), any(&ifFalse), any(&c)))) {
// Check if the values are identical and we can keep just the first one.
if (areConsecutiveInputsEqualAndFoldable(ifTrue, ifFalse)) {
// We can keep just the ifTrue branch, but we need to move its value
// past the condition. If there are side effects that force us to use
// a scratch local, then it is probably worth the extra local to
// execute the value expression once instead of twice.
if (canReorder(ifTrue, c)) {
return builder.makeSequence(builder.makeDrop(c), ifTrue);
}
// We generally skip optimizing unreachable selects, but an
// optimization on the children might have made them newly
// unreachable. We cannot create a scratch local in that case.
if (!ifTrue->type.isConcrete()) {
return nullptr;
}
auto scratch = builder.addVar(getFunction(), ifTrue->type);
return builder.makeBlock(
{builder.makeLocalSet(scratch, ifTrue),
builder.makeDrop(c),
builder.makeLocalGet(scratch, ifTrue->type)});
}
// Otherwise, check if the values are identical, but we would have to
// keep both. In this case the benefit is less, so we only optimize if
// we can avoid the scratch local.
if (areConsecutiveInputsEqual(ifTrue, ifFalse) &&
canReorder(ifFalse, c)) {
return builder.makeBlock(
{builder.makeDrop(ifTrue), builder.makeDrop(c), ifFalse});
}
}
}
return nullptr;
}
// find added constants in an expression tree, including multiplied/shifted,
// and combine them note that we ignore division/shift-right, as rounding
// makes this nonlinear, so not a valid opt
Expression* optimizeAddedConstants(Binary* binary) {
assert(binary->type.isInteger());
uint64_t constant = 0;
std::vector<Const*> constants;
struct SeekState {
Expression* curr;
uint64_t mul;
SeekState(Expression* curr, uint64_t mul) : curr(curr), mul(mul) {}
};
std::vector<SeekState> seekStack;
seekStack.emplace_back(binary, 1);
while (!seekStack.empty()) {
auto state = seekStack.back();
seekStack.pop_back();
auto curr = state.curr;
auto mul = state.mul;
if (auto* c = curr->dynCast<Const>()) {
uint64_t value = c->value.getInteger();
if (value != 0ULL) {
constant += value * mul;
constants.push_back(c);
}
continue;
} else if (auto* binary = curr->dynCast<Binary>()) {
if (binary->op == Abstract::getBinary(binary->type, Abstract::Add)) {
seekStack.emplace_back(binary->right, mul);
seekStack.emplace_back(binary->left, mul);
continue;
} else if (binary->op ==
Abstract::getBinary(binary->type, Abstract::Sub)) {
// if the left is a zero, ignore it, it's how we negate ints
auto* left = binary->left->dynCast<Const>();
seekStack.emplace_back(binary->right, -mul);
if (!left || !left->value.isZero()) {
seekStack.emplace_back(binary->left, mul);
}
continue;
} else if (binary->op ==
Abstract::getBinary(binary->type, Abstract::Shl)) {
if (auto* c = binary->right->dynCast<Const>()) {
seekStack.emplace_back(binary->left,
mul << Bits::getEffectiveShifts(c));
continue;
}
} else if (binary->op ==
Abstract::getBinary(binary->type, Abstract::Mul)) {
if (auto* c = binary->left->dynCast<Const>()) {
seekStack.emplace_back(binary->right,
mul * (uint64_t)c->value.getInteger());
continue;
} else if (auto* c = binary->right->dynCast<Const>()) {
seekStack.emplace_back(binary->left,
mul * (uint64_t)c->value.getInteger());
continue;
}
}
}
};
// find all factors
if (constants.size() <= 1) {
// nothing much to do, except for the trivial case of adding/subbing a
// zero
if (auto* c = binary->right->dynCast<Const>()) {
if (c->value.isZero()) {
return binary->left;
}
}
return nullptr;
}
// wipe out all constants, we'll replace with a single added one
for (auto* c : constants) {
c->value = Literal::makeZero(c->type);
}
// remove added/subbed zeros
struct ZeroRemover : public PostWalker<ZeroRemover> {
// TODO: we could save the Binary and Const nodes we drop, and reuse them
// later
PassOptions& passOptions;
ZeroRemover(PassOptions& passOptions) : passOptions(passOptions) {}
void visitBinary(Binary* curr) {
if (!curr->type.isInteger()) {
return;
}
auto type = curr->type;
auto* left = curr->left->dynCast<Const>();
auto* right = curr->right->dynCast<Const>();
// Canonicalization prefers an add instead of a subtract wherever
// possible. That prevents a subtracted constant on the right,
// as it would be added. And for a zero on the left, it can't be
// removed (it is how we negate ints).
if (curr->op == Abstract::getBinary(type, Abstract::Add)) {
if (left && left->value.isZero()) {
replaceCurrent(curr->right);
return;
}
if (right && right->value.isZero()) {
replaceCurrent(curr->left);
return;
}
} else if (curr->op == Abstract::getBinary(type, Abstract::Shl)) {
// shifting a 0 is a 0, or anything by 0 has no effect, all unless the
// shift has side effects
if (((left && left->value.isZero()) ||
(right && Bits::getEffectiveShifts(right) == 0)) &&
!EffectAnalyzer(passOptions, *getModule(), curr->right)
.hasSideEffects()) {
replaceCurrent(curr->left);
return;
}
} else if (curr->op == Abstract::getBinary(type, Abstract::Mul)) {
// multiplying by zero is a zero, unless the other side has side
// effects
if (left && left->value.isZero() &&
!EffectAnalyzer(passOptions, *getModule(), curr->right)
.hasSideEffects()) {
replaceCurrent(left);
return;
}
if (right && right->value.isZero() &&
!EffectAnalyzer(passOptions, *getModule(), curr->left)
.hasSideEffects()) {
replaceCurrent(right);
return;
}
}
}
};
// Noting the type here not only simplifies the code below, but is also
// necessary to avoid an error: if we look at walked->type then it may
// actually differ from the original type, say if the walk ended up turning
// |binary| into a simpler unreachable expression.
auto type = binary->type;
Expression* walked = binary;
ZeroRemover remover(getPassOptions());
remover.setModule(getModule());
remover.walk(walked);
if (constant == 0ULL) {
return walked; // nothing more to do
}
// Add the total constant value we computed to the value remaining here.
// Note that if the value is 32 bits then |makeFromInt64| will wrap to 32
// bits for us; as all the operations before us and the add below us are
// adds and subtracts, any overflow is not a problem.
auto toAdd = Literal::makeFromInt64(constant, type);
if (auto* c = walked->dynCast<Const>()) {
// This is a constant, so just add it immediately (we could also leave
// this for Precompute, in principle).
c->value = c->value.add(toAdd);
return c;
}
Builder builder(*getModule());
return builder.makeBinary(Abstract::getBinary(type, Abstract::Add),
walked,
builder.makeConst(toAdd));
}
// Given an i64.wrap operation, see if we can remove it. If all the things
// being operated on behave the same with or without wrapping, then we don't
// need to go to 64 bits at all, e.g.:
//
// int32_t(int64_t(x)) => x (extend, then wrap)
// int32_t(int64_t(x) + int64_t(10)) => x + int32_t(10) (also add)
//
Expression* optimizeWrappedResult(Unary* wrap) {
assert(wrap->op == WrapInt64);
// Core processing logic. This goes through the children, in one of two
// modes:
// * Scan: Find if there is anything we can't handle. Sets |canOptimize|
// with what it finds.
// * Optimize: Given we can handle everything, update things.
enum Mode { Scan, Optimize };
bool canOptimize = true;
auto processChildren = [&](Mode mode) {
// Use a simple stack as we go through the children. We use ** as we need
// to replace children for some optimizations.
SmallVector<Expression**, 2> stack;
stack.emplace_back(&wrap->value);
while (!stack.empty() && canOptimize) {
auto* currp = stack.back();
stack.pop_back();
auto* curr = *currp;
if (curr->type == Type::unreachable) {
// Leave unreachability for other passes.
canOptimize = false;
return;
} else if (auto* c = curr->dynCast<Const>()) {
// A i64 const can be handled by just turning it into an i32.
if (mode == Optimize) {
c->value = Literal(int32_t(c->value.getInteger()));
c->type = Type::i32;
}
} else if (auto* unary = curr->dynCast<Unary>()) {
switch (unary->op) {
case ExtendSInt32:
case ExtendUInt32: {
// Note that there is nothing to push to the stack here: the child
// is 32-bit already, so we can stop looking. We just need to skip
// the extend operation.
if (mode == Optimize) {
*currp = unary->value;
}
break;
}
default: {
// TODO: handle more cases here and below,
// https://github.com/WebAssembly/binaryen/issues/5004
canOptimize = false;
return;
}
}
} else if (auto* binary = curr->dynCast<Binary>()) {
// Turn the binary into a 32-bit one, if we can.
switch (binary->op) {
case AddInt64:
case SubInt64:
case MulInt64: {
// We can optimize these.
break;
}
default: {
canOptimize = false;
return;
}
}
if (mode == Optimize) {
switch (binary->op) {
case AddInt64: {
binary->op = AddInt32;
break;
}
case SubInt64: {
binary->op = SubInt32;
break;
}
case MulInt64: {
binary->op = MulInt32;
break;
}
default: {
WASM_UNREACHABLE("bad op");
}
}
// All things we can optimize change the type to i32.
binary->type = Type::i32;
}
stack.push_back(&binary->left);
stack.push_back(&binary->right);
} else {
// Anything else makes us give up.
canOptimize = false;
return;
}
}
};
processChildren(Scan);
if (!canOptimize) {
return nullptr;
}
// Optimize, and return the optimized results (in which we no longer need
// the wrap operation itself).
processChildren(Optimize);
return wrap->value;
}
// expensive1 | expensive2 can be turned into expensive1 ? 1 : expensive2,
// and expensive | cheap can be turned into cheap ? 1 : expensive,
// so that we can avoid one expensive computation, if it has no side effects.
Expression* conditionalizeExpensiveOnBitwise(Binary* binary) {
// this operation can increase code size, so don't always do it
auto& options = getPassRunner()->options;
if (options.optimizeLevel < 2 || options.shrinkLevel > 0) {
return nullptr;
}
const auto MIN_COST = 7;
assert(binary->op == AndInt32 || binary->op == OrInt32);
if (binary->right->is<Const>()) {
return nullptr; // trivial
}
// bitwise logical operator on two non-numerical values, check if they are
// boolean
auto* left = binary->left;
auto* right = binary->right;
if (!Properties::emitsBoolean(left) || !Properties::emitsBoolean(right)) {
return nullptr;
}
auto leftEffects = effects(left);
auto rightEffects = effects(right);
auto leftHasSideEffects = leftEffects.hasSideEffects();
auto rightHasSideEffects = rightEffects.hasSideEffects();
if (leftHasSideEffects && rightHasSideEffects) {
return nullptr; // both must execute
}
// canonicalize with side effects, if any, happening on the left
if (rightHasSideEffects) {
if (CostAnalyzer(left).cost < MIN_COST) {
return nullptr; // avoidable code is too cheap
}
if (leftEffects.invalidates(rightEffects)) {
return nullptr; // cannot reorder
}
std::swap(left, right);
} else if (leftHasSideEffects) {
if (CostAnalyzer(right).cost < MIN_COST) {
return nullptr; // avoidable code is too cheap
}
} else {
// no side effects, reorder based on cost estimation
auto leftCost = CostAnalyzer(left).cost;
auto rightCost = CostAnalyzer(right).cost;
if (std::max(leftCost, rightCost) < MIN_COST) {
return nullptr; // avoidable code is too cheap
}
// canonicalize with expensive code on the right
if (leftCost > rightCost) {
std::swap(left, right);
}
}
// worth it! perform conditionalization
Builder builder(*getModule());
if (binary->op == OrInt32) {
return builder.makeIf(
left, builder.makeConst(Literal(int32_t(1))), right);
} else { // &
return builder.makeIf(
left, right, builder.makeConst(Literal(int32_t(0))));
}
}
// We can combine `and` operations, e.g.
// (x == 0) & (y == 0) ==> (x | y) == 0
Expression* combineAnd(Binary* curr) {
assert(curr->op == AndInt32);
using namespace Abstract;
using namespace Match;
{
// (i32(x) == 0) & (i32(y) == 0) ==> i32(x | y) == 0
// (i64(x) == 0) & (i64(y) == 0) ==> i64(x | y) == 0
Expression *x, *y;
if (matches(curr->left, unary(EqZ, any(&x))) &&
matches(curr->right, unary(EqZ, any(&y))) && x->type == y->type) {
auto* inner = curr->left->cast<Unary>();
inner->value =
Builder(*getModule()).makeBinary(getBinary(x->type, Or), x, y);
return inner;
}
}
{
// Binary operations that inverse a bitwise AND can be
// reordered. If F(x) = binary(x, c), and F(x) preserves AND,
// that is,
//
// F(x) & F(y) == F(x | y)
//
// Then also
//
// binary(x, c) & binary(y, c) => binary(x | y, c)
Binary *bx, *by;
Expression *x, *y;
Const *cx, *cy;
if (matches(curr->left, binary(&bx, any(&x), ival(&cx))) &&
matches(curr->right, binary(&by, any(&y), ival(&cy))) &&
bx->op == by->op && x->type == y->type && cx->value == cy->value &&
inversesAnd(bx)) {
by->op = getBinary(x->type, Or);
by->type = x->type;
by->left = x;
by->right = y;
bx->left = by;
return bx;
}
}
{
// Binary operations that preserve a bitwise AND can be
// reordered. If F(x) = binary(x, c), and F(x) preserves AND,
// that is,
//
// F(x) & F(y) == F(x & y)
//
// Then also
//
// binary(x, c) & binary(y, c) => binary(x & y, c)
Binary *bx, *by;
Expression *x, *y;
Const *cx, *cy;
if (matches(curr->left, binary(&bx, any(&x), ival(&cx))) &&
matches(curr->right, binary(&by, any(&y), ival(&cy))) &&
bx->op == by->op && x->type == y->type && cx->value == cy->value &&
preserveAnd(bx)) {
by->op = getBinary(x->type, And);
by->type = x->type;
by->left = x;
by->right = y;
bx->left = by;
return bx;
}
}
return nullptr;
}
// Bitwise AND of a value with bits in [0, n) and a constant with no bits in
// [0, n) always yields 0. Replace with zero.
Expression* optimizeAndNoOverlappingBits(Binary* curr) {
assert(curr->op == AndInt32 || curr->op == AndInt64);
auto* left = curr->left;
auto* right = curr->right;
// Check left's max bits and right is constant.
auto leftMaxBits = Bits::getMaxBits(left, this);
uint64_t maskLeft;
// leftMaxBits may be greater than size if the lhs is unreachable but has
// not been refinalized yet.
if (!left->type.isNumber() || leftMaxBits >= left->type.getByteSize() * 8) {
// If we know nothing useful about the bits on the left,
// we cannot optimize.
return nullptr;
} else {
assert(leftMaxBits < 64);
maskLeft = (1ULL << leftMaxBits) - 1;
}
if (auto* c = right->dynCast<Const>()) {
uint64_t constantValue = c->value.getInteger();
if ((constantValue & maskLeft) == 0) {
return getDroppedChildrenAndAppend(
curr, LiteralUtils::makeZero(left->type, *getModule()));
}
}
return nullptr;
}
// We can combine `or` operations, e.g.
// (x > y) | (x == y) ==> x >= y
// (x != 0) | (y != 0) ==> (x | y) != 0
Expression* combineOr(Binary* curr) {
assert(curr->op == OrInt32);
using namespace Abstract;
using namespace Match;
if (auto* left = curr->left->dynCast<Binary>()) {
if (auto* right = curr->right->dynCast<Binary>()) {
if (left->op != right->op &&
ExpressionAnalyzer::equal(left->left, right->left) &&
ExpressionAnalyzer::equal(left->right, right->right) &&
!effects(left->left).hasSideEffects() &&
!effects(left->right).hasSideEffects()) {
switch (left->op) {
// (x > y) | (x == y) ==> x >= y
case EqInt32: {
if (right->op == GtSInt32) {
left->op = GeSInt32;
return left;
}
break;
}
default: {
}
}
}
}
}
{
// Binary operations that inverses a bitwise OR to AND.
// If F(x) = binary(x, c), and F(x) inverses OR,
// that is,
//
// F(x) | F(y) == F(x & y)
//
// Then also
//
// binary(x, c) | binary(y, c) => binary(x & y, c)
Binary *bx, *by;
Expression *x, *y;
Const *cx, *cy;
if (matches(curr->left, binary(&bx, any(&x), ival(&cx))) &&
matches(curr->right, binary(&by, any(&y), ival(&cy))) &&
bx->op == by->op && x->type == y->type && cx->value == cy->value &&
inversesOr(bx)) {
by->op = getBinary(x->type, And);
by->type = x->type;
by->left = x;
by->right = y;
bx->left = by;
return bx;
}
}
{
// Binary operations that preserve a bitwise OR can be
// reordered. If F(x) = binary(x, c), and F(x) preserves OR,
// that is,
//
// F(x) | F(y) == F(x | y)
//
// Then also
//
// binary(x, c) | binary(y, c) => binary(x | y, c)
Binary *bx, *by;
Expression *x, *y;
Const *cx, *cy;
if (matches(curr->left, binary(&bx, any(&x), ival(&cx))) &&
matches(curr->right, binary(&by, any(&y), ival(&cy))) &&
bx->op == by->op && x->type == y->type && cx->value == cy->value &&
preserveOr(bx)) {
by->op = getBinary(x->type, Or);
by->type = x->type;
by->left = x;
by->right = y;
bx->left = by;
return bx;
}
}
return nullptr;
}
// Check whether an operation preserves the Or operation through it, that is,
//
// F(x | y) = F(x) | F(y)
//
// Mathematically that means F is homomorphic with respect to the | operation.
//
// F(x) is seen as taking a single parameter of its first child. That is, the
// first child is |x|, and the rest is constant. For example, if we are given
// a binary with operation != and the right child is a constant 0, then
// F(x) = (x != 0).
bool preserveOr(Binary* curr) {
using namespace Abstract;
using namespace Match;
// (x != 0) | (y != 0) ==> (x | y) != 0
// This effectively checks if any bits are set in x or y.
if (matches(curr, binary(Ne, any(), ival(0)))) {
return true;
}
// (x < 0) | (y < 0) ==> (x | y) < 0
// This effectively checks if x or y have the sign bit set.
if (matches(curr, binary(LtS, any(), ival(0)))) {
return true;
}
return false;
}
// Check whether an operation inverses the Or operation to And, that is,
//
// F(x | y) = F(x) & F(y)
//
// Mathematically that means F is homomorphic with respect to the | operation.
//
// F(x) is seen as taking a single parameter of its first child. That is, the
// first child is |x|, and the rest is constant. For example, if we are given
// a binary with operation != and the right child is a constant 0, then
// F(x) = (x != 0).
bool inversesOr(Binary* curr) {
using namespace Abstract;
using namespace Match;
// (x >= 0) | (y >= 0) ==> (x & y) >= 0
if (matches(curr, binary(GeS, any(), ival(0)))) {
return true;
}
// (x !=-1) | (y !=-1) ==> (x & y) !=-1
if (matches(curr, binary(Ne, any(), ival(-1)))) {
return true;
}
return false;
}
// Check whether an operation preserves the And operation through it, that is,
//
// F(x & y) = F(x) & F(y)
//
// Mathematically that means F is homomorphic with respect to the & operation.
//
// F(x) is seen as taking a single parameter of its first child. That is, the
// first child is |x|, and the rest is constant. For example, if we are given
// a binary with operation != and the right child is a constant 0, then
// F(x) = (x != 0).
bool preserveAnd(Binary* curr) {
using namespace Abstract;
using namespace Match;
// (x < 0) & (y < 0) ==> (x & y) < 0
if (matches(curr, binary(LtS, any(), ival(0)))) {
return true;
}
// (x == -1) & (y == -1) ==> (x & y) == -1
if (matches(curr, binary(Eq, any(), ival(-1)))) {
return true;
}
return false;
}
// Check whether an operation inverses the And operation to Or, that is,
//
// F(x & y) = F(x) | F(y)
//
// Mathematically that means F is homomorphic with respect to the & operation.
//
// F(x) is seen as taking a single parameter of its first child. That is, the
// first child is |x|, and the rest is constant. For example, if we are given
// a binary with operation != and the right child is a constant 0, then
// F(x) = (x != 0).
bool inversesAnd(Binary* curr) {
using namespace Abstract;
using namespace Match;
// (x >= 0) & (y >= 0) ==> (x | y) >= 0
if (matches(curr, binary(GeS, any(), ival(0)))) {
return true;
}
return false;
}
// fold constant factors into the offset
void optimizeMemoryAccess(Expression*& ptr, Address& offset, Name memory) {
// ptr may be a const, but it isn't worth folding that in (we still have a
// const); in fact, it's better to do the opposite for gzip purposes as well
// as for readability.
auto* last = ptr->dynCast<Const>();
if (last) {
uint64_t value64 = last->value.getInteger();
uint64_t offset64 = offset;
auto mem = getModule()->getMemory(memory);
if (mem->is64()) {
// Check for a 64-bit overflow.
uint64_t sum;
if (!std::ckd_add(&sum, value64, offset64)) {
last->value = Literal(int64_t(sum));
offset = 0;
}
} else {
// don't do this if it would wrap the pointer
if (value64 <= uint64_t(std::numeric_limits<int32_t>::max()) &&
offset64 <= uint64_t(std::numeric_limits<int32_t>::max()) &&
value64 + offset64 <=
uint64_t(std::numeric_limits<int32_t>::max())) {
last->value = Literal(int32_t(value64 + offset64));
offset = 0;
}
}
}
}
// Optimize a multiply by a power of two on the right, which
// can be a shift.
// This doesn't shrink code size, and VMs likely optimize it anyhow,
// but it's still worth doing since
// * Often shifts are more common than muls.
// * The constant is smaller.
template<typename T> Expression* optimizePowerOf2Mul(Binary* binary, T c) {
static_assert(std::is_same<T, uint32_t>::value ||
std::is_same<T, uint64_t>::value,
"type mismatch");
auto shifts = Bits::countTrailingZeroes(c);
binary->op = std::is_same<T, uint32_t>::value ? ShlInt32 : ShlInt64;
binary->right->cast<Const>()->value = Literal(static_cast<T>(shifts));
return binary;
}
// Optimize an unsigned divide / remainder by a power of two on the right
// This doesn't shrink code size, and VMs likely optimize it anyhow,
// but it's still worth doing since
// * Usually ands are more common than urems.
// * The constant is slightly smaller.
template<typename T> Expression* optimizePowerOf2URem(Binary* binary, T c) {
static_assert(std::is_same<T, uint32_t>::value ||
std::is_same<T, uint64_t>::value,
"type mismatch");
binary->op = std::is_same<T, uint32_t>::value ? AndInt32 : AndInt64;
binary->right->cast<Const>()->value = Literal(c - 1);
return binary;
}
template<typename T> Expression* optimizePowerOf2UDiv(Binary* binary, T c) {
static_assert(std::is_same<T, uint32_t>::value ||
std::is_same<T, uint64_t>::value,
"type mismatch");
auto shifts = Bits::countTrailingZeroes(c);
binary->op = std::is_same<T, uint32_t>::value ? ShrUInt32 : ShrUInt64;
binary->right->cast<Const>()->value = Literal(static_cast<T>(shifts));
return binary;
}
template<typename T> Expression* optimizePowerOf2FDiv(Binary* binary, T c) {
//
// x / C_pot => x * (C_pot ^ -1)
//
// Explanation:
// Floating point numbers are represented as:
// ((-1) ^ sign) * (2 ^ (exp - bias)) * (1 + significand)
//
// If we have power of two numbers, then the mantissa (significand)
// is all zeros. Let's focus on the exponent, ignoring the sign part:
// (2 ^ (exp - bias))
//
// and for inverted power of two floating point:
// 1.0 / (2 ^ (exp - bias)) -> 2 ^ -(exp - bias)
//
// So inversion of C_pot is valid because it changes only the sign
// of the exponent part and doesn't touch the significand part,
// which remains the same (zeros).
static_assert(std::is_same<T, float>::value ||
std::is_same<T, double>::value,
"type mismatch");
double invDivisor = 1.0 / (double)c;
binary->op = std::is_same<T, float>::value ? MulFloat32 : MulFloat64;
binary->right->cast<Const>()->value = Literal(static_cast<T>(invDivisor));
return binary;
}
Expression* makeZeroExt(Expression* curr, int32_t bits) {
Builder builder(*getModule());
return builder.makeBinary(
AndInt32, curr, builder.makeConst(Literal(Bits::lowBitMask(bits))));
}
// given an "almost" sign extend - either a proper one, or it
// has too many shifts left - we remove the sign extend. If there are
// too many shifts, we split the shifts first, so this removes the
// two sign extend shifts and adds one (smaller one)
Expression* removeAlmostSignExt(Binary* outer) {
auto* inner = outer->left->cast<Binary>();
auto* outerConst = outer->right->cast<Const>();
auto* innerConst = inner->right->cast<Const>();
auto* value = inner->left;
if (outerConst->value == innerConst->value) {
return value;
}
// add a shift, by reusing the existing node
innerConst->value = innerConst->value.sub(outerConst->value);
return inner;
}
// Check if an expression is already sign-extended to an exact number of bits.
bool isSignExted(Expression* curr, Index bits) {
return getSignExtBits(curr) == bits;
}
// Returns the number of bits an expression is sign-extended (or 0 if it is
// not).
Index getSignExtBits(Expression* curr) {
if (Properties::getSignExtValue(curr)) {
return Properties::getSignExtBits(curr);
}
if (auto* get = curr->dynCast<LocalGet>()) {
// Check what we know about the local.
return localInfo[get->index].signExtBits;
}
return 0;
}
// optimize trivial math operations, given that the right side of a binary
// is a constant
Expression* optimizeWithConstantOnRight(Binary* curr) {
using namespace Match;
using namespace Abstract;
Builder builder(*getModule());
Expression* left;
auto* right = curr->right->cast<Const>();
auto type = curr->right->type;
// Operations on zero
if (matches(curr, binary(Shl, any(&left), ival(0))) ||
matches(curr, binary(ShrU, any(&left), ival(0))) ||
matches(curr, binary(ShrS, any(&left), ival(0))) ||
matches(curr, binary(Or, any(&left), ival(0))) ||
matches(curr, binary(Xor, any(&left), ival(0)))) {
return left;
}
if (matches(curr, binary(Mul, pure(&left), ival(0))) ||
matches(curr, binary(And, pure(&left), ival(0)))) {
return right;
}
// -x * C ==> x * -C, if shrinkLevel != 0 or C != C_pot
// -x * C ==> -(x * C), otherwise
// where x, C are integers
Binary* inner;
if (matches(
curr,
binary(Mul, binary(&inner, Sub, ival(0), any(&left)), ival()))) {
if (getPassOptions().shrinkLevel != 0 ||
!Bits::isPowerOf2(right->value.getInteger())) {
right->value = right->value.neg();
curr->left = left;
return curr;
} else {
curr->left = left;
Const* zero = inner->left->cast<Const>();
return builder.makeBinary(inner->op, zero, curr);
}
}
// x == 0 ==> eqz x
if (matches(curr, binary(Eq, any(&left), ival(0)))) {
return builder.makeUnary(Abstract::getUnary(type, EqZ), left);
}
// Operations on one
// (signed)x % 1 ==> 0
if (matches(curr, binary(RemS, pure(&left), ival(1)))) {
right->value = Literal::makeZero(type);
return right;
}
// (signed)x % C_pot != 0 ==> (x & (abs(C_pot) - 1)) != 0
{
Const* c;
Binary* inner;
if (matches(curr,
binary(Ne, binary(&inner, RemS, any(), ival(&c)), ival(0))) &&
(c->value.isSignedMin() ||
Bits::isPowerOf2(c->value.abs().getInteger()))) {
inner->op = Abstract::getBinary(c->type, And);
if (c->value.isSignedMin()) {
c->value = Literal::makeSignedMax(c->type);
} else {
c->value = c->value.abs().sub(Literal::makeOne(c->type));
}
return curr;
}
}
// i32(bool(x)) == 1 ==> i32(bool(x))
// i32(bool(x)) != 0 ==> i32(bool(x))
// i32(bool(x)) & 1 ==> i32(bool(x))
// i64(bool(x)) & 1 ==> i64(bool(x))
if ((matches(curr, binary(EqInt32, any(&left), i32(1))) ||
matches(curr, binary(NeInt32, any(&left), i32(0))) ||
matches(curr, binary(And, any(&left), ival(1)))) &&
Bits::getMaxBits(left, this) == 1) {
return left;
}
// i64(bool(x)) == 1 ==> i32(bool(x))
// i64(bool(x)) != 0 ==> i32(bool(x))
if ((matches(curr, binary(EqInt64, any(&left), i64(1))) ||
matches(curr, binary(NeInt64, any(&left), i64(0)))) &&
Bits::getMaxBits(left, this) == 1) {
return builder.makeUnary(WrapInt64, left);
}
// bool(x) != 1 ==> !bool(x)
if (matches(curr, binary(Ne, any(&left), ival(1))) &&
Bits::getMaxBits(left, this) == 1) {
return builder.makeUnary(Abstract::getUnary(type, EqZ), left);
}
// bool(x) ^ 1 ==> !bool(x)
if (matches(curr, binary(Xor, any(&left), ival(1))) &&
Bits::getMaxBits(left, this) == 1) {
auto* result = builder.makeUnary(Abstract::getUnary(type, EqZ), left);
if (left->type == Type::i64) {
// Xor's result is also an i64 in this case, but EqZ returns i32, so we
// must expand it so that we keep returning the same value as before.
// This means we replace a xor and a const with a xor and an extend,
// which is still smaller (the const is 2 bytes, the extend just 1), and
// also the extend may be removed by further work.
result = builder.makeUnary(ExtendUInt32, result);
}
return result;
}
// bool(x) | 1 ==> 1
if (matches(curr, binary(Or, any(&left), ival(1))) &&
Bits::getMaxBits(left, this) == 1) {
return getDroppedChildrenAndAppend(curr, right);
}
// Operations on all 1s
// x & -1 ==> x
if (matches(curr, binary(And, any(&left), ival(-1)))) {
return left;
}
// x | -1 ==> -1
if (matches(curr, binary(Or, any(&left), ival(-1)))) {
return getDroppedChildrenAndAppend(curr, right);
}
// (signed)x % -1 ==> 0
if (matches(curr, binary(RemS, pure(&left), ival(-1)))) {
right->value = Literal::makeZero(type);
return right;
}
// i32(x) / i32.min_s ==> x == i32.min_s
if (matches(
curr,
binary(DivSInt32, any(), i32(std::numeric_limits<int32_t>::min())))) {
curr->op = EqInt32;
return curr;
}
// i64(x) / i64.min_s ==> i64(x == i64.min_s)
// only for zero shrink level
if (getPassOptions().shrinkLevel == 0 &&
matches(
curr,
binary(DivSInt64, any(), i64(std::numeric_limits<int64_t>::min())))) {
curr->op = EqInt64;
curr->type = Type::i32;
return builder.makeUnary(ExtendUInt32, curr);
}
// (unsigned)x < 0 ==> i32(0)
if (matches(curr, binary(LtU, pure(&left), ival(0)))) {
right->value = Literal::makeZero(Type::i32);
right->type = Type::i32;
return right;
}
// (unsigned)x <= -1 ==> i32(1)
if (matches(curr, binary(LeU, any(&left), ival(-1)))) {
right->value = Literal::makeOne(Type::i32);
right->type = Type::i32;
return getDroppedChildrenAndAppend(curr, right);
}
// (unsigned)x > -1 ==> i32(0)
if (matches(curr, binary(GtU, any(&left), ival(-1)))) {
right->value = Literal::makeZero(Type::i32);
right->type = Type::i32;
return getDroppedChildrenAndAppend(curr, right);
}
// (unsigned)x >= 0 ==> i32(1)
if (matches(curr, binary(GeU, pure(&left), ival(0)))) {
right->value = Literal::makeOne(Type::i32);
right->type = Type::i32;
return right;
}
// (unsigned)x < -1 ==> x != -1
// Friendlier to JS emitting as we don't need to write an unsigned -1 value
// which is large.
if (matches(curr, binary(LtU, any(), ival(-1)))) {
curr->op = Abstract::getBinary(type, Ne);
return curr;
}
// (unsigned)x <= 0 ==> x == 0
if (matches(curr, binary(LeU, any(), ival(0)))) {
curr->op = Abstract::getBinary(type, Eq);
return curr;
}
// (unsigned)x > 0 ==> x != 0
if (matches(curr, binary(GtU, any(), ival(0)))) {
curr->op = Abstract::getBinary(type, Ne);
return curr;
}
// (unsigned)x >= -1 ==> x == -1
if (matches(curr, binary(GeU, any(), ival(-1)))) {
curr->op = Abstract::getBinary(type, Eq);
return curr;
}
{
Const* c;
// (signed)x < (i32|i64).min_s ==> i32(0)
if (matches(curr, binary(LtS, pure(&left), ival(&c))) &&
c->value.isSignedMin()) {
right->value = Literal::makeZero(Type::i32);
right->type = Type::i32;
return right;
}
// (signed)x <= (i32|i64).max_s ==> i32(1)
if (matches(curr, binary(LeS, pure(&left), ival(&c))) &&
c->value.isSignedMax()) {
right->value = Literal::makeOne(Type::i32);
right->type = Type::i32;
return right;
}
// (signed)x > (i32|i64).max_s ==> i32(0)
if (matches(curr, binary(GtS, pure(&left), ival(&c))) &&
c->value.isSignedMax()) {
right->value = Literal::makeZero(Type::i32);
right->type = Type::i32;
return right;
}
// (signed)x >= (i32|i64).min_s ==> i32(1)
if (matches(curr, binary(GeS, pure(&left), ival(&c))) &&
c->value.isSignedMin()) {
right->value = Literal::makeOne(Type::i32);
right->type = Type::i32;
return right;
}
// (signed)x < (i32|i64).max_s ==> x != (i32|i64).max_s
if (matches(curr, binary(LtS, any(), ival(&c))) &&
c->value.isSignedMax()) {
curr->op = Abstract::getBinary(type, Ne);
return curr;
}
// (signed)x <= (i32|i64).min_s ==> x == (i32|i64).min_s
if (matches(curr, binary(LeS, any(), ival(&c))) &&
c->value.isSignedMin()) {
curr->op = Abstract::getBinary(type, Eq);
return curr;
}
// (signed)x > (i32|i64).min_s ==> x != (i32|i64).min_s
if (matches(curr, binary(GtS, any(), ival(&c))) &&
c->value.isSignedMin()) {
curr->op = Abstract::getBinary(type, Ne);
return curr;
}
// (signed)x >= (i32|i64).max_s ==> x == (i32|i64).max_s
if (matches(curr, binary(GeS, any(), ival(&c))) &&
c->value.isSignedMax()) {
curr->op = Abstract::getBinary(type, Eq);
return curr;
}
}
// x * -1 ==> 0 - x
if (matches(curr, binary(Mul, any(&left), ival(-1)))) {
right->value = Literal::makeZero(type);
curr->op = Abstract::getBinary(type, Sub);
curr->left = right;
curr->right = left;
return curr;
}
{
// ~(1 << x) aka (1 << x) ^ -1 ==> rotl(-2, x)
Expression* x;
// Note that we avoid this in JS mode, as emitting a rotation would
// require lowering that rotation for JS in another cycle of work.
if (matches(curr, binary(Xor, binary(Shl, ival(1), any(&x)), ival(-1))) &&
!getPassOptions().targetJS) {
curr->op = Abstract::getBinary(type, RotL);
right->value = Literal::makeFromInt32(-2, type);
curr->left = right;
curr->right = x;
return curr;
}
}
{
// x * 2.0 ==> x + x
// but we apply this only for simple expressions like
// local.get and global.get for avoid using extra local
// variable.
Expression* x;
if (matches(curr, binary(Mul, any(&x), fval(2.0))) &&
(x->is<LocalGet>() || x->is<GlobalGet>())) {
curr->op = Abstract::getBinary(type, Abstract::Add);
curr->right = ExpressionManipulator::copy(x, *getModule());
return curr;
}
}
{
// x + (-0.0) ==> x
double value;
if (fastMath && matches(curr, binary(Add, any(), fval(&value))) &&
value == 0.0 && std::signbit(value)) {
return curr->left;
}
}
// -x * fval(C) ==> x * -C
// -x / fval(C) ==> x / -C
if (matches(curr, binary(Mul, unary(Neg, any(&left)), fval())) ||
matches(curr, binary(DivS, unary(Neg, any(&left)), fval()))) {
right->value = right->value.neg();
curr->left = left;
return curr;
}
// x * -1.0 ==>
// -x, if fastMath == true
// -0.0 - x, if fastMath == false
if (matches(curr, binary(Mul, any(), fval(-1.0)))) {
if (fastMath) {
return builder.makeUnary(Abstract::getUnary(type, Neg), left);
}
// x * -1.0 ==> -0.0 - x
curr->op = Abstract::getBinary(type, Sub);
right->value = Literal::makeZero(type).neg();
std::swap(curr->left, curr->right);
return curr;
}
if (matches(curr, binary(Mul, any(&left), constant(1))) ||
matches(curr, binary(DivS, any(&left), constant(1))) ||
matches(curr, binary(DivU, any(&left), constant(1)))) {
if (curr->type.isInteger() || fastMath) {
return left;
}
}
{
// x != NaN ==> 1
// x <=> NaN ==> 0
// x op NaN' ==> NaN', iff `op` != `copysign` and `x` != C
Const* c;
Binary* bin;
Expression* x;
if (matches(curr, binary(&bin, pure(&x), fval(&c))) &&
std::isnan(c->value.getFloat()) &&
bin->op != getBinary(x->type, CopySign)) {
if (bin->isRelational()) {
// reuse "c" (nan) constant
c->type = Type::i32;
if (bin->op == getBinary(x->type, Ne)) {
// x != NaN ==> 1
c->value = Literal::makeOne(Type::i32);
} else {
// x == NaN,
// x > NaN,
// x <= NaN
// x .. NaN ==> 0
c->value = Literal::makeZero(Type::i32);
}
return c;
}
// propagate NaN of RHS but canonicalize it
c->value = Literal::standardizeNaN(c->value);
return c;
}
}
return nullptr;
}
// Returns true if the given binary operation can overflow. If we can't be
// sure either way, we return true, assuming the worst.
//
// We can check for an unsigned overflow (more than the max number of bits) or
// a signed one (where even reaching the sign bit is an overflow, as that
// would turn us from positive to negative).
bool canOverflow(Binary* binary, bool signed_) {
using namespace Abstract;
// If we know nothing about a limit on the amount of bits on either side,
// give up.
auto typeMaxBits = getBitsForType(binary->type);
auto leftMaxBits = Bits::getMaxBits(binary->left, this);
auto rightMaxBits = Bits::getMaxBits(binary->right, this);
if (std::max(leftMaxBits, rightMaxBits) == typeMaxBits) {
return true;
}
if (binary->op == getBinary(binary->type, Add)) {
if (!signed_) {
// Proof this cannot overflow:
//
// left + right < 2^leftMaxBits + 2^rightMaxBits (1)
// <= 2^(typeMaxBits-1) + 2^(typeMaxBits-1) (2)
// = 2^typeMaxBits (3)
//
// (1) By the definition of the max bits (e.g. an int32 has 32 max bits,
// and its max value is 2^32 - 1, which is < 2^32).
// (2) By the above checks and early returns.
// (3) 2^x + 2^x === 2*2^x === 2^(x+1)
return false;
}
// For a signed comparison, check that the total cannot reach the sign
// bit.
return leftMaxBits + rightMaxBits >= typeMaxBits;
}
// TODO subtraction etc.
return true;
}
// Folding two expressions into one with similar operations and
// constants on RHSs
Expression* optimizeDoubletonWithConstantOnRight(Binary* curr) {
using namespace Match;
using namespace Abstract;
{
Binary* inner;
Const *c1, *c2 = curr->right->cast<Const>();
if (matches(curr->left, binary(&inner, any(), ival(&c1))) &&
inner->op == curr->op) {
Type type = inner->type;
BinaryOp op = inner->op;
// (x & C1) & C2 => x & (C1 & C2)
if (op == getBinary(type, And)) {
c1->value = c1->value.and_(c2->value);
return inner;
}
// (x | C1) | C2 => x | (C1 | C2)
if (op == getBinary(type, Or)) {
c1->value = c1->value.or_(c2->value);
return inner;
}
// (x ^ C1) ^ C2 => x ^ (C1 ^ C2)
if (op == getBinary(type, Xor)) {
c1->value = c1->value.xor_(c2->value);
return inner;
}
// (x * C1) * C2 => x * (C1 * C2)
if (op == getBinary(type, Mul)) {
c1->value = c1->value.mul(c2->value);
return inner;
}
// TODO:
// handle signed / unsigned divisions. They are more complex
// (x <<>> C1) <<>> C2 => x <<>> (C1 + C2)
if (hasAnyShift(op)) {
// shifts only use an effective amount from the constant, so
// adding must be done carefully
auto total =
Bits::getEffectiveShifts(c1) + Bits::getEffectiveShifts(c2);
auto effectiveTotal = Bits::getEffectiveShifts(total, c1->type);
if (total == effectiveTotal) {
// no overflow, we can do this
c1->value = Literal::makeFromInt32(total, c1->type);
return inner;
} else {
// overflow. Handle different scenarios
if (hasAnyRotateShift(op)) {
// overflow always accepted in rotation shifts
c1->value = Literal::makeFromInt32(effectiveTotal, c1->type);
return inner;
}
// handle overflows for general shifts
// x << C1 << C2 => 0 or { drop(x), 0 }
// x >>> C1 >>> C2 => 0 or { drop(x), 0 }
// iff `C1 + C2` -> overflows
if ((op == getBinary(type, Shl) || op == getBinary(type, ShrU))) {
auto* x = inner->left;
c1->value = Literal::makeZero(c1->type);
if (!effects(x).hasSideEffects()) {
// => 0
return c1;
} else {
// => { drop(x), 0 }
Builder builder(*getModule());
return builder.makeBlock({builder.makeDrop(x), c1});
}
}
// i32(x) >> C1 >> C2 => x >> 31
// i64(x) >> C1 >> C2 => x >> 63
// iff `C1 + C2` -> overflows
if (op == getBinary(type, ShrS)) {
c1->value = Literal::makeFromInt32(c1->type.getByteSize() * 8 - 1,
c1->type);
return inner;
}
}
}
}
}
{
// (x << C1) * C2 => x * (C2 << C1)
Binary* inner;
Const *c1, *c2;
if (matches(
curr,
binary(Mul, binary(&inner, Shl, any(), ival(&c1)), ival(&c2)))) {
inner->op = getBinary(inner->type, Mul);
c1->value = c2->value.shl(c1->value);
return inner;
}
}
{
// (x * C1) << C2 => x * (C1 << C2)
Binary* inner;
Const *c1, *c2;
if (matches(
curr,
binary(Shl, binary(&inner, Mul, any(), ival(&c1)), ival(&c2)))) {
c1->value = c1->value.shl(c2->value);
return inner;
}
}
{
// TODO: Add cancellation for some large constants when shrinkLevel > 0
// in FinalOptimizer.
// (x >> C) << C => x & -(1 << C)
// (x >>> C) << C => x & -(1 << C)
Binary* inner;
Const *c1, *c2;
if (matches(curr,
binary(Shl, binary(&inner, any(), ival(&c1)), ival(&c2))) &&
(inner->op == getBinary(inner->type, ShrS) ||
inner->op == getBinary(inner->type, ShrU)) &&
Bits::getEffectiveShifts(c1) == Bits::getEffectiveShifts(c2)) {
auto type = c1->type;
if (type == Type::i32) {
c1->value = Literal::makeFromInt32(
-(1U << Bits::getEffectiveShifts(c1)), Type::i32);
} else {
c1->value = Literal::makeFromInt64(
-(1ULL << Bits::getEffectiveShifts(c1)), Type::i64);
}
inner->op = getBinary(type, And);
return inner;
}
}
{
// TODO: Add cancellation for some large constants when shrinkLevel > 0
// in FinalOptimizer.
// (x << C) >>> C => x & (-1 >>> C)
// (x << C) >> C => skip
Binary* inner;
Const *c1, *c2;
if (matches(
curr,
binary(ShrU, binary(&inner, Shl, any(), ival(&c1)), ival(&c2))) &&
Bits::getEffectiveShifts(c1) == Bits::getEffectiveShifts(c2)) {
auto type = c1->type;
if (type == Type::i32) {
c1->value = Literal::makeFromInt32(
-1U >> Bits::getEffectiveShifts(c1), Type::i32);
} else {
c1->value = Literal::makeFromInt64(
-1ULL >> Bits::getEffectiveShifts(c1), Type::i64);
}
inner->op = getBinary(type, And);
return inner;
}
}
{
// TODO: Add canonicalization rotr to rotl and remove these rules.
// rotl(rotr(x, C1), C2) => rotr(x, C1 - C2)
// rotr(rotl(x, C1), C2) => rotl(x, C1 - C2)
Binary* inner;
Const *c1, *c2;
if (matches(
curr,
binary(RotL, binary(&inner, RotR, any(), ival(&c1)), ival(&c2))) ||
matches(
curr,
binary(RotR, binary(&inner, RotL, any(), ival(&c1)), ival(&c2)))) {
auto diff = Bits::getEffectiveShifts(c1) - Bits::getEffectiveShifts(c2);
c1->value = Literal::makeFromInt32(
Bits::getEffectiveShifts(diff, c2->type), c2->type);
return inner;
}
}
return nullptr;
}
// optimize trivial math operations, given that the left side of a binary
// is a constant. since we canonicalize constants to the right for symmetrical
// operations, we only need to handle asymmetrical ones here
// TODO: templatize on type?
Expression* optimizeWithConstantOnLeft(Binary* curr) {
using namespace Match;
using namespace Abstract;
auto type = curr->left->type;
auto* left = curr->left->cast<Const>();
// 0 <<>> x ==> 0
if (Abstract::hasAnyShift(curr->op) && left->value.isZero() &&
!effects(curr->right).hasSideEffects()) {
return curr->left;
}
// (signed)-1 >> x ==> -1
// rotl(-1, x) ==> -1
// rotr(-1, x) ==> -1
if ((curr->op == Abstract::getBinary(type, ShrS) ||
curr->op == Abstract::getBinary(type, RotL) ||
curr->op == Abstract::getBinary(type, RotR)) &&
left->value.getInteger() == -1LL &&
!effects(curr->right).hasSideEffects()) {
return curr->left;
}
{
// C1 - (x + C2) ==> (C1 - C2) - x
Const *c1, *c2;
Expression* x;
if (matches(curr,
binary(Sub, ival(&c1), binary(Add, any(&x), ival(&c2))))) {
left->value = c1->value.sub(c2->value);
curr->right = x;
return curr;
}
// C1 - (C2 - x) ==> x + (C1 - C2)
if (matches(curr,
binary(Sub, ival(&c1), binary(Sub, ival(&c2), any(&x))))) {
left->value = c1->value.sub(c2->value);
curr->op = Abstract::getBinary(type, Add);
curr->right = x;
std::swap(curr->left, curr->right);
return curr;
}
}
{
// fval(C) / -x ==> -C / x
Expression* right;
if (matches(curr, binary(DivS, fval(), unary(Neg, any(&right))))) {
left->value = left->value.neg();
curr->right = right;
return curr;
}
}
return nullptr;
}
// TODO: templatize on type?
Expression* optimizeRelational(Binary* curr) {
using namespace Abstract;
using namespace Match;
auto type = curr->right->type;
if (curr->left->type.isInteger()) {
if (curr->op == Abstract::getBinary(type, Abstract::Eq) ||
curr->op == Abstract::getBinary(type, Abstract::Ne)) {
if (auto* left = curr->left->dynCast<Binary>()) {
// TODO: inequalities can also work, if the constants do not overflow
// integer math, even on 2s complement, allows stuff like
// x + 5 == 7
// =>
// x == 2
if (left->op == Abstract::getBinary(type, Abstract::Add)) {
if (auto* leftConst = left->right->dynCast<Const>()) {
if (auto* rightConst = curr->right->dynCast<Const>()) {
return combineRelationalConstants(
curr, left, leftConst, nullptr, rightConst);
} else if (auto* rightBinary = curr->right->dynCast<Binary>()) {
if (rightBinary->op ==
Abstract::getBinary(type, Abstract::Add)) {
if (auto* rightConst = rightBinary->right->dynCast<Const>()) {
return combineRelationalConstants(
curr, left, leftConst, rightBinary, rightConst);
}
}
}
}
}
}
}
// x - y == 0 => x == y
// x - y != 0 => x != y
// unsigned(x - y) > 0 => x != y
// unsigned(x - y) <= 0 => x == y
{
Binary* inner;
// unsigned(x - y) > 0 => x != y
if (matches(curr,
binary(GtU, binary(&inner, Sub, any(), any()), ival(0)))) {
curr->op = Abstract::getBinary(type, Ne);
curr->right = inner->right;
curr->left = inner->left;
return curr;
}
// unsigned(x - y) <= 0 => x == y
if (matches(curr,
binary(LeU, binary(&inner, Sub, any(), any()), ival(0)))) {
curr->op = Abstract::getBinary(type, Eq);
curr->right = inner->right;
curr->left = inner->left;
return curr;
}
// x - y == 0 => x == y
// x - y != 0 => x != y
// This is not true for signed comparisons like x -y < 0 due to overflow
// effects (e.g. 8 - 0x80000000 < 0 is not the same as 8 < 0x80000000).
if (matches(curr,
binary(Eq, binary(&inner, Sub, any(), any()), ival(0))) ||
matches(curr,
binary(Ne, binary(&inner, Sub, any(), any()), ival(0)))) {
curr->right = inner->right;
curr->left = inner->left;
return curr;
}
}
// x + C1 > C2 ==> x > (C2-C1) if no overflowing, C2 >= C1
// x + C1 > C2 ==> x + (C1-C2) > 0 if no overflowing, C2 < C1
// And similarly for other relational operations on integers with a "+"
// on the left.
// TODO: support - and not just +
{
Binary* add;
Const* c1;
Const* c2;
if (matches(curr,
binary(binary(&add, Add, any(), ival(&c1)), ival(&c2))) &&
!canOverflow(add, isSignedOp(curr->op))) {
// We want to subtract C2-C1 or C1-C2. When doing so, we must avoid an
// overflow in that subtraction (so that we keep all the math here
// properly linear in the mathematical sense). Overflows that concern
// us include an underflow with unsigned values (e.g. 10 - 20, which
// flips the result to a large positive number), and a sign bit
// overflow for signed values (e.g. 0x80000000 - 1 = 0x7fffffff flips
// from a negative number, -1, to a positive one). We also need to be
// careful of signed handling of 0x80000000, for whom 0 - 0x80000000
// is equal to 0x80000000, leading to
// x + 0x80000000 > 0 ;; always false
// (apply the rule)
// x > 0 - 0x80000000 = 0x80000000 ;; depends on x
// The general principle in all of this is that when we go from
// (a) x + C1 > C2
// to
// (b) x > (C2-C1)
// then we want to adjust both sides in the same (linear) manner. That
// is, we can write the latter as
// (b') x + 0 > (C2-C1)
// Comparing (a) and (b'), we want the constants to change in a
// consistent way: C1 changes to 0, and C2 changes to C2-C1. Both
// transformations should decrease the value, which is violated in all
// the overflows described above:
// * Unsigned overflow: C1=20, C2=10, then C1 decreases but C2-C1
// is larger than C2.
// * Sign flip: C1=1, C2=0x80000000, then C1 decreases but C2-C1 is
// is larger than C2.
// * C1=0x80000000, C2=0, then C1 increases while C2-C1 stays the
// same.
// In the first and second case we can apply the other rule using
// C1-C2 rather than C2-C1. The third case, however, doesn't even work
// that way.
auto C1 = c1->value;
auto C2 = c2->value;
auto C1SubC2 = C1.sub(C2);
auto C2SubC1 = C2.sub(C1);
auto zero = Literal::makeZero(add->type);
auto doC1SubC2 = false;
auto doC2SubC1 = false;
// Ignore the case of C1 or C2 being zero, as then C2-C1 or C1-C2
// does not change anything (and we don't want the optimizer to think
// we improved anything, or we could infinite loop on the mirage of
// progress).
if (C1 != zero && C2 != zero) {
if (isSignedOp(curr->op)) {
if (C2SubC1.leS(C2).getInteger() && zero.leS(C1).getInteger()) {
// C2=>C2-C1 and C1=>0 both decrease, which means we can do the
// rule
// (a) x + C1 > C2
// (b') x (+ 0) > (C2-C1)
// That is, subtracting C1 from both sides is ok; the constants
// on both sides change in the same manner.
doC2SubC1 = true;
} else if (C1SubC2.leS(C1).getInteger() &&
zero.leS(C2).getInteger()) {
// N.B. this code path is not tested atm as other optimizations
// will canonicalize x + C into x - C, and so we would need to
// implement the TODO above on subtraction and not only support
// addition here.
doC1SubC2 = true;
}
} else {
// Unsigned.
if (C2SubC1.leU(C2).getInteger() && zero.leU(C1).getInteger()) {
doC2SubC1 = true;
} else if (C1SubC2.leU(C1).getInteger() &&
zero.leU(C2).getInteger()) {
doC1SubC2 = true;
}
// For unsigned, one of the cases must work out, as there are no
// corner cases with the sign bit.
assert(doC2SubC1 || doC1SubC2);
}
}
if (doC2SubC1) {
// This is the first line above, we turn into x > (C2-C1).
c2->value = C2SubC1;
curr->left = add->left;
return curr;
}
// This is the second line above, we turn into x + (C1-C2) > 0.
if (doC1SubC2) {
c1->value = C1SubC2;
c2->value = zero;
return curr;
}
}
}
// Comparisons can sometimes be simplified depending on the number of
// bits, e.g. (unsigned)x > y must be true if x has strictly more bits.
// A common case is a constant on the right, e.g. (x & 255) < 256 must be
// true.
// TODO: use getMinBits in more places, see ideas in
// https://github.com/WebAssembly/binaryen/issues/2898
{
// Check if there is a nontrivial amount of bits on the left, which may
// provide enough to optimize.
auto leftMaxBits = Bits::getMaxBits(curr->left, this);
auto type = curr->left->type;
if (leftMaxBits < getBitsForType(type)) {
using namespace Abstract;
auto rightMinBits = Bits::getMinBits(curr->right);
auto rightIsNegative = rightMinBits == getBitsForType(type);
if (leftMaxBits < rightMinBits) {
// There are not enough bits on the left for it to be equal to the
// right, making various comparisons obviously false:
// x == y
// (unsigned)x > y
// (unsigned)x >= y
// and the same for signed, if y does not have the sign bit set
// (in that case, the comparison is effectively unsigned).
//
// TODO: In addition to leftMaxBits < rightMinBits, we could
// handle the reverse, and also special cases like all bits
// being 1 on the right, things like (x & 255) <= 255 -> 1
if (curr->op == Abstract::getBinary(type, Eq) ||
curr->op == Abstract::getBinary(type, GtU) ||
curr->op == Abstract::getBinary(type, GeU) ||
(!rightIsNegative &&
(curr->op == Abstract::getBinary(type, GtS) ||
curr->op == Abstract::getBinary(type, GeS)))) {
return getDroppedChildrenAndAppend(curr,
Literal::makeZero(Type::i32));
}
// And some are obviously true:
// x != y
// (unsigned)x < y
// (unsigned)x <= y
// and likewise for signed, as above.
if (curr->op == Abstract::getBinary(type, Ne) ||
curr->op == Abstract::getBinary(type, LtU) ||
curr->op == Abstract::getBinary(type, LeU) ||
(!rightIsNegative &&
(curr->op == Abstract::getBinary(type, LtS) ||
curr->op == Abstract::getBinary(type, LeS)))) {
return getDroppedChildrenAndAppend(curr,
Literal::makeOne(Type::i32));
}
// For truly signed comparisons, where y's sign bit is set, we can
// also infer some things, since we know y is signed but x is not
// (since x does not have enough bits for the sign bit to be set).
if (rightIsNegative) {
// (signed, non-negative)x > (negative)y => 1
// (signed, non-negative)x >= (negative)y => 1
if (curr->op == Abstract::getBinary(type, GtS) ||
curr->op == Abstract::getBinary(type, GeS)) {
return getDroppedChildrenAndAppend(curr,
Literal::makeOne(Type::i32));
}
// (signed, non-negative)x < (negative)y => 0
// (signed, non-negative)x <= (negative)y => 0
if (curr->op == Abstract::getBinary(type, LtS) ||
curr->op == Abstract::getBinary(type, LeS)) {
return getDroppedChildrenAndAppend(
curr, Literal::makeZero(Type::i32));
}
}
}
}
}
}
return nullptr;
}
Expression* simplifyRoundingsAndConversions(Unary* curr) {
using namespace Abstract;
using namespace Match;
switch (curr->op) {
case TruncSFloat64ToInt32:
case TruncSatSFloat64ToInt32: {
// i32 -> f64 -> i32 roundtripping optimization:
// i32.trunc(_sat)_f64_s(f64.convert_i32_s(x)) ==> x
Expression* x;
if (matches(curr->value, unary(ConvertSInt32ToFloat64, any(&x)))) {
return x;
}
break;
}
case TruncUFloat64ToInt32:
case TruncSatUFloat64ToInt32: {
// u32 -> f64 -> u32 roundtripping optimization:
// i32.trunc(_sat)_f64_u(f64.convert_i32_u(x)) ==> x
Expression* x;
if (matches(curr->value, unary(ConvertUInt32ToFloat64, any(&x)))) {
return x;
}
break;
}
case CeilFloat32:
case CeilFloat64:
case FloorFloat32:
case FloorFloat64:
case TruncFloat32:
case TruncFloat64:
case NearestFloat32:
case NearestFloat64: {
// Rounding after integer to float conversion may be skipped
// ceil(float(int(x))) ==> float(int(x))
// floor(float(int(x))) ==> float(int(x))
// trunc(float(int(x))) ==> float(int(x))
// nearest(float(int(x))) ==> float(int(x))
Unary* inner;
if (matches(curr->value, unary(&inner, any()))) {
switch (inner->op) {
case ConvertSInt32ToFloat32:
case ConvertSInt32ToFloat64:
case ConvertUInt32ToFloat32:
case ConvertUInt32ToFloat64:
case ConvertSInt64ToFloat32:
case ConvertSInt64ToFloat64:
case ConvertUInt64ToFloat32:
case ConvertUInt64ToFloat64: {
return inner;
}
default: {
}
}
}
break;
}
default: {
}
}
return nullptr;
}
Expression* deduplicateUnary(Unary* unaryOuter) {
if (auto* unaryInner = unaryOuter->value->dynCast<Unary>()) {
if (unaryInner->op == unaryOuter->op) {
switch (unaryInner->op) {
case NegFloat32:
case NegFloat64: {
// neg(neg(x)) ==> x
return unaryInner->value;
}
case AbsFloat32:
case CeilFloat32:
case FloorFloat32:
case TruncFloat32:
case NearestFloat32:
case AbsFloat64:
case CeilFloat64:
case FloorFloat64:
case TruncFloat64:
case NearestFloat64: {
// unaryOp(unaryOp(x)) ==> unaryOp(x)
return unaryInner;
}
case ExtendS8Int32:
case ExtendS16Int32: {
assert(getModule()->features.hasSignExt());
return unaryInner;
}
case EqZInt32: {
// eqz(eqz(bool(x))) ==> bool(x)
if (Bits::getMaxBits(unaryInner->value, this) == 1) {
return unaryInner->value;
}
break;
}
default: {
}
}
}
}
return nullptr;
}
Expression* deduplicateBinary(Binary* outer) {
Type type = outer->type;
if (type.isInteger()) {
if (auto* inner = outer->right->dynCast<Binary>()) {
if (outer->op == inner->op) {
if (!EffectAnalyzer(getPassOptions(), *getModule(), outer->left)
.hasSideEffects()) {
if (ExpressionAnalyzer::equal(inner->left, outer->left)) {
// x - (x - y) ==> y
// x ^ (x ^ y) ==> y
if (outer->op == Abstract::getBinary(type, Abstract::Sub) ||
outer->op == Abstract::getBinary(type, Abstract::Xor)) {
return inner->right;
}
// x & (x & y) ==> x & y
// x | (x | y) ==> x | y
if (outer->op == Abstract::getBinary(type, Abstract::And) ||
outer->op == Abstract::getBinary(type, Abstract::Or)) {
return inner;
}
}
if (ExpressionAnalyzer::equal(inner->right, outer->left) &&
canReorder(outer->left, inner->left)) {
// x ^ (y ^ x) ==> y
// (note that we need the check for reordering here because if
// e.g. y writes to a local that x reads, the second appearance
// of x would be different from the first)
if (outer->op == Abstract::getBinary(type, Abstract::Xor)) {
return inner->left;
}
// x & (y & x) ==> y & x
// x | (y | x) ==> y | x
// (here we need the check for reordering for the more obvious
// reason that previously x appeared before y, and now y appears
// first; or, if we tried to emit x [&|] y here, reversing the
// order, we'd be in the same situation as the previous comment)
if (outer->op == Abstract::getBinary(type, Abstract::And) ||
outer->op == Abstract::getBinary(type, Abstract::Or)) {
return inner;
}
}
}
}
}
if (auto* inner = outer->left->dynCast<Binary>()) {
if (outer->op == inner->op) {
if (!EffectAnalyzer(getPassOptions(), *getModule(), outer->right)
.hasSideEffects()) {
if (ExpressionAnalyzer::equal(inner->right, outer->right)) {
// (x ^ y) ^ y ==> x
if (outer->op == Abstract::getBinary(type, Abstract::Xor)) {
return inner->left;
}
// (x % y) % y ==> x % y
// (x & y) & y ==> x & y
// (x | y) | y ==> x | y
if (outer->op == Abstract::getBinary(type, Abstract::RemS) ||
outer->op == Abstract::getBinary(type, Abstract::RemU) ||
outer->op == Abstract::getBinary(type, Abstract::And) ||
outer->op == Abstract::getBinary(type, Abstract::Or)) {
return inner;
}
}
// See comments in the parallel code earlier about ordering here.
if (ExpressionAnalyzer::equal(inner->left, outer->right) &&
canReorder(inner->left, inner->right)) {
// (x ^ y) ^ x ==> y
if (outer->op == Abstract::getBinary(type, Abstract::Xor)) {
return inner->right;
}
// (x & y) & x ==> x & y
// (x | y) | x ==> x | y
if (outer->op == Abstract::getBinary(type, Abstract::And) ||
outer->op == Abstract::getBinary(type, Abstract::Or)) {
return inner;
}
}
}
}
}
}
return nullptr;
}
// given a relational binary with a const on both sides, combine the constants
// left is also a binary, and has a constant; right may be just a constant, in
// which case right is nullptr
Expression* combineRelationalConstants(Binary* binary,
Binary* left,
Const* leftConst,
Binary* right,
Const* rightConst) {
auto type = binary->right->type;
// we fold constants to the right
Literal extra = leftConst->value;
if (left->op == Abstract::getBinary(type, Abstract::Sub)) {
extra = extra.neg();
}
if (right && right->op == Abstract::getBinary(type, Abstract::Sub)) {
extra = extra.neg();
}
rightConst->value = rightConst->value.sub(extra);
binary->left = left->left;
return binary;
}
Expression* optimizeMemoryCopy(MemoryCopy* memCopy) {
auto& options = getPassOptions();
if (options.ignoreImplicitTraps || options.trapsNeverHappen) {
if (areConsecutiveInputsEqual(memCopy->dest, memCopy->source)) {
// memory.copy(x, x, sz) ==> {drop(x), drop(x), drop(sz)}
Builder builder(*getModule());
return builder.makeBlock({builder.makeDrop(memCopy->dest),
builder.makeDrop(memCopy->source),
builder.makeDrop(memCopy->size)});
}
}
// memory.copy(dst, src, C) ==> store(dst, load(src))
if (auto* csize = memCopy->size->dynCast<Const>()) {
auto bytes = csize->value.getInteger();
Builder builder(*getModule());
switch (bytes) {
case 0: {
if (options.ignoreImplicitTraps || options.trapsNeverHappen) {
// memory.copy(dst, src, 0) ==> {drop(dst), drop(src)}
return builder.makeBlock({builder.makeDrop(memCopy->dest),
builder.makeDrop(memCopy->source)});
}
break;
}
case 1:
case 2:
case 4: {
return builder.makeStore(bytes, // bytes
0, // offset
1, // align
memCopy->dest,
builder.makeLoad(bytes,
false,
0,
1,
memCopy->source,
Type::i32,
memCopy->sourceMemory),
Type::i32,
memCopy->destMemory);
}
case 8: {
return builder.makeStore(bytes, // bytes
0, // offset
1, // align
memCopy->dest,
builder.makeLoad(bytes,
false,
0,
1,
memCopy->source,
Type::i64,
memCopy->sourceMemory),
Type::i64,
memCopy->destMemory);
}
case 16: {
if (options.shrinkLevel == 0) {
// This adds an extra 2 bytes so apply it only for
// minimal shrink level
if (getModule()->features.hasSIMD()) {
return builder.makeStore(bytes, // bytes
0, // offset
1, // align
memCopy->dest,
builder.makeLoad(bytes,
false,
0,
1,
memCopy->source,
Type::v128,
memCopy->sourceMemory),
Type::v128,
memCopy->destMemory);
}
}
break;
}
default: {
}
}
}
return nullptr;
}
Expression* optimizeMemoryFill(MemoryFill* memFill) {
if (memFill->type == Type::unreachable) {
return nullptr;
}
if (!memFill->size->is<Const>()) {
return nullptr;
}
auto& options = getPassOptions();
Builder builder(*getModule());
auto* csize = memFill->size->cast<Const>();
auto bytes = csize->value.getInteger();
if (bytes == 0LL &&
(options.ignoreImplicitTraps || options.trapsNeverHappen)) {
// memory.fill(d, v, 0) ==> { drop(d), drop(v) }
return builder.makeBlock(
{builder.makeDrop(memFill->dest), builder.makeDrop(memFill->value)});
}
const uint32_t offset = 0, align = 1;
if (auto* cvalue = memFill->value->dynCast<Const>()) {
uint32_t value = cvalue->value.geti32() & 0xFF;
// memory.fill(d, C1, C2) ==>
// store(d, (C1 & 0xFF) * (-1U / max(bytes)))
switch (bytes) {
case 1: {
return builder.makeStore(1, // bytes
offset,
align,
memFill->dest,
builder.makeConst<uint32_t>(value),
Type::i32,
memFill->memory);
}
case 2: {
return builder.makeStore(2,
offset,
align,
memFill->dest,
builder.makeConst<uint32_t>(value * 0x0101U),
Type::i32,
memFill->memory);
}
case 4: {
// transform only when "value" or shrinkLevel equal to zero due to
// it could increase size by several bytes
if (value == 0 || options.shrinkLevel == 0) {
return builder.makeStore(
4,
offset,
align,
memFill->dest,
builder.makeConst<uint32_t>(value * 0x01010101U),
Type::i32,
memFill->memory);
}
break;
}
case 8: {
// transform only when "value" or shrinkLevel equal to zero due to
// it could increase size by several bytes
if (value == 0 || options.shrinkLevel == 0) {
return builder.makeStore(
8,
offset,
align,
memFill->dest,
builder.makeConst<uint64_t>(value * 0x0101010101010101ULL),
Type::i64,
memFill->memory);
}
break;
}
case 16: {
if (options.shrinkLevel == 0) {
if (getModule()->features.hasSIMD()) {
uint8_t values[16];
std::fill_n(values, 16, (uint8_t)value);
return builder.makeStore(16,
offset,
align,
memFill->dest,
builder.makeConst<uint8_t[16]>(values),
Type::v128,
memFill->memory);
} else {
// { i64.store(d, C', 0), i64.store(d, C', 8) }
auto destType = memFill->dest->type;
Index tempLocal = builder.addVar(getFunction(), destType);
return builder.makeBlock({
builder.makeStore(
8,
offset,
align,
builder.makeLocalTee(tempLocal, memFill->dest, destType),
builder.makeConst<uint64_t>(value * 0x0101010101010101ULL),
Type::i64,
memFill->memory),
builder.makeStore(
8,
offset + 8,
align,
builder.makeLocalGet(tempLocal, destType),
builder.makeConst<uint64_t>(value * 0x0101010101010101ULL),
Type::i64,
memFill->memory),
});
}
}
break;
}
default: {
}
}
}
// memory.fill(d, v, 1) ==> store8(d, v)
if (bytes == 1LL) {
return builder.makeStore(1,
offset,
align,
memFill->dest,
memFill->value,
Type::i32,
memFill->memory);
}
return nullptr;
}
// given a binary expression with equal children and no side effects in
// either, we can fold various things
Expression* optimizeBinaryWithEqualEffectlessChildren(Binary* binary) {
// TODO add: perhaps worth doing 2*x if x is quite large?
switch (binary->op) {
case SubInt32:
case XorInt32:
case SubInt64:
case XorInt64:
return LiteralUtils::makeZero(binary->left->type, *getModule());
case NeInt32:
case LtSInt32:
case LtUInt32:
case GtSInt32:
case GtUInt32:
case NeInt64:
case LtSInt64:
case LtUInt64:
case GtSInt64:
case GtUInt64:
return LiteralUtils::makeZero(Type::i32, *getModule());
case AndInt32:
case OrInt32:
case AndInt64:
case OrInt64:
return binary->left;
case EqInt32:
case LeSInt32:
case LeUInt32:
case GeSInt32:
case GeUInt32:
case EqInt64:
case LeSInt64:
case LeUInt64:
case GeSInt64:
case GeUInt64:
return LiteralUtils::makeFromInt32(1, Type::i32, *getModule());
default:
return nullptr;
}
}
// Invert (negate) the opcode, so that it has the exact negative meaning as it
// had before.
BinaryOp invertBinaryOp(BinaryOp op) {
switch (op) {
case EqInt32:
return NeInt32;
case NeInt32:
return EqInt32;
case LtSInt32:
return GeSInt32;
case LtUInt32:
return GeUInt32;
case LeSInt32:
return GtSInt32;
case LeUInt32:
return GtUInt32;
case GtSInt32:
return LeSInt32;
case GtUInt32:
return LeUInt32;
case GeSInt32:
return LtSInt32;
case GeUInt32:
return LtUInt32;
case EqInt64:
return NeInt64;
case NeInt64:
return EqInt64;
case LtSInt64:
return GeSInt64;
case LtUInt64:
return GeUInt64;
case LeSInt64:
return GtSInt64;
case LeUInt64:
return GtUInt64;
case GtSInt64:
return LeSInt64;
case GtUInt64:
return LeUInt64;
case GeSInt64:
return LtSInt64;
case GeUInt64:
return LtUInt64;
case EqFloat32:
return NeFloat32;
case NeFloat32:
return EqFloat32;
case EqFloat64:
return NeFloat64;
case NeFloat64:
return EqFloat64;
default:
return InvalidBinary;
}
}
// Change the opcode so it is correct after reversing the operands. That is,
// we had X OP Y and we need OP' so that this is equivalent to that:
// Y OP' X
BinaryOp reverseRelationalOp(BinaryOp op) {
switch (op) {
case EqInt32:
return EqInt32;
case NeInt32:
return NeInt32;
case LtSInt32:
return GtSInt32;
case LtUInt32:
return GtUInt32;
case LeSInt32:
return GeSInt32;
case LeUInt32:
return GeUInt32;
case GtSInt32:
return LtSInt32;
case GtUInt32:
return LtUInt32;
case GeSInt32:
return LeSInt32;
case GeUInt32:
return LeUInt32;
case EqInt64:
return EqInt64;
case NeInt64:
return NeInt64;
case LtSInt64:
return GtSInt64;
case LtUInt64:
return GtUInt64;
case LeSInt64:
return GeSInt64;
case LeUInt64:
return GeUInt64;
case GtSInt64:
return LtSInt64;
case GtUInt64:
return LtUInt64;
case GeSInt64:
return LeSInt64;
case GeUInt64:
return LeUInt64;
case EqFloat32:
return EqFloat32;
case NeFloat32:
return NeFloat32;
case LtFloat32:
return GtFloat32;
case LeFloat32:
return GeFloat32;
case GtFloat32:
return LtFloat32;
case GeFloat32:
return LeFloat32;
case EqFloat64:
return EqFloat64;
case NeFloat64:
return NeFloat64;
case LtFloat64:
return GtFloat64;
case LeFloat64:
return GeFloat64;
case GtFloat64:
return LtFloat64;
case GeFloat64:
return LeFloat64;
default:
return InvalidBinary;
}
}
BinaryOp makeUnsignedBinaryOp(BinaryOp op) {
switch (op) {
case DivSInt32:
return DivUInt32;
case RemSInt32:
return RemUInt32;
case ShrSInt32:
return ShrUInt32;
case LtSInt32:
return LtUInt32;
case LeSInt32:
return LeUInt32;
case GtSInt32:
return GtUInt32;
case GeSInt32:
return GeUInt32;
case DivSInt64:
return DivUInt64;
case RemSInt64:
return RemUInt64;
case ShrSInt64:
return ShrUInt64;
case LtSInt64:
return LtUInt64;
case LeSInt64:
return LeUInt64;
case GtSInt64:
return GtUInt64;
case GeSInt64:
return GeUInt64;
default:
return InvalidBinary;
}
}
bool shouldCanonicalize(Binary* binary) {
if ((binary->op == SubInt32 || binary->op == SubInt64) &&
binary->right->is<Const>() && !binary->left->is<Const>()) {
return true;
}
if (Properties::isSymmetric(binary) || binary->isRelational()) {
return true;
}
switch (binary->op) {
case SubFloat32:
case SubFloat64: {
// Should apply x - C -> x + (-C)
return binary->right->is<Const>();
}
case AddFloat32:
case MulFloat32:
case AddFloat64:
case MulFloat64: {
// If the LHS is known to be non-NaN, the operands can commute.
// We don't care about the RHS because right now we only know if
// an expression is non-NaN if it is constant, but if the RHS is
// constant, then this expression is already canonicalized.
if (auto* c = binary->left->dynCast<Const>()) {
return !c->value.isNaN();
}
return false;
}
default:
return false;
}
}
// Optimize an if-else or a select, something with a condition and two
// arms with outputs.
template<typename T> void optimizeTernary(T* curr) {
using namespace Abstract;
using namespace Match;
Builder builder(*getModule());
// If one arm is an operation and the other is an appropriate constant, we
// can move the operation outside (where it may be further optimized), e.g.
//
// (select
// (i32.eqz (X))
// (i32.const 0|1)
// (Y)
// )
// =>
// (i32.eqz
// (select
// (X)
// (i32.const 1|0)
// (Y)
// )
// )
//
// Ignore unreachable code here; leave that for DCE.
if (curr->type != Type::unreachable &&
curr->ifTrue->type != Type::unreachable &&
curr->ifFalse->type != Type::unreachable) {
Unary* un;
Const* c;
auto check = [&](Expression* a, Expression* b) {
return matches(b, bval(&c)) && matches(a, unary(&un, EqZ, any()));
};
if (check(curr->ifTrue, curr->ifFalse) ||
check(curr->ifFalse, curr->ifTrue)) {
// The new type of curr will be that of the value of the unary, as after
// we move the unary out, its value is curr's direct child.
auto newType = un->value->type;
auto updateArm = [&](Expression* arm) -> Expression* {
if (arm == un) {
// This is the arm that had the eqz, which we need to remove.
return un->value;
} else {
// This is the arm with the constant, which we need to flip.
// Note that we also need to set the type to match the other arm.
c->value =
Literal::makeFromInt32(1 - c->value.getInteger(), newType);
c->type = newType;
return c;
}
};
curr->ifTrue = updateArm(curr->ifTrue);
curr->ifFalse = updateArm(curr->ifFalse);
un->value = curr;
curr->finalize();
return replaceCurrent(un);
}
}
if (!neverFold && curr->condition->type != Type::unreachable) {
// Identical code on both arms can be folded out, e.g.
//
// (select
// (i32.eqz (X))
// (i32.eqz (Y))
// (Z)
// )
// =>
// (i32.eqz
// (select
// (X)
// (Y)
// (Z)
// )
// )
//
// Continue doing this while we can, noting the chain of moved expressions
// as we go, then do a single replaceCurrent() at the end.
SmallVector<Expression*, 1> chain;
while (1) {
// Ignore control flow structures (which are handled in MergeBlocks).
if (!Properties::isControlFlowStructure(curr->ifTrue) &&
ExpressionAnalyzer::shallowEqual(curr->ifTrue, curr->ifFalse)) {
// TODO: consider the case with more than one child.
ChildIterator ifTrueChildren(curr->ifTrue);
if (ifTrueChildren.children.size() == 1) {
// ifTrue and ifFalse's children will become the direct children of
// curr, and so they must be compatible to allow for a proper new
// type after the transformation.
//
// The children must have a shared supertype. For example, we cannot
// fold out the `drop` here because there would be no valid result
// type for the if afterward:
//
// (if
// (condition)
// (drop (i32.const 1))
// (drop (f64.const 2.0))
// )
//
// However, having a shared supertype may not be enough. If $A and
// $B have a shared supertype, but that supertype does not have a
// field at index 1, then we cannot fold the duplicated struct.get
// here:
//
// (if
// (condition)
// (struct.get $A 1 (..))
// (struct.get $B 1 (..))
// )
//
// (Note: In that example, the problem only happens because the type
// is not part of the struct.get - we infer it from the reference.
// That is why after hoisting the struct.get out, and computing a
// new type for the if that is now the child of the single
// struct.get, we get a struct.get of a supertype. So in principle
// we could fix this by modifying the IR as well, but the problem is
// more general, so avoid that.)
//
// For now, only support the case where the types of the children
// are the same.
//
// TODO: We could analyze whether the LUB of the children types
// would satisfy the constraints imposed by the shared parent
// instruction. This would require a version of ChildTyper that
// allows for generalizing type annotations.
ChildIterator ifFalseChildren(curr->ifFalse);
auto* ifTrueChild = *ifTrueChildren.begin();
auto* ifFalseChild = *ifFalseChildren.begin();
bool validTypes = ifTrueChild->type == ifFalseChild->type;
// If the expression we are about to move outside has side effects,
// then we cannot do so in general with a select: we'd be reducing
// the amount of the effects as well as moving them. For an if,
// the side effects execute once, so there is no problem.
// TODO: handle certain side effects when possible in select
bool validEffects = std::is_same<T, If>::value ||
!ShallowEffectAnalyzer(
getPassOptions(), *getModule(), curr->ifTrue)
.hasSideEffects();
// In addition, check for specific limitations of select.
bool validChildren =
!std::is_same<T, Select>::value ||
Properties::canEmitSelectWithArms(ifTrueChild, ifFalseChild);
if (validTypes && validEffects && validChildren) {
// Replace ifTrue with its child.
curr->ifTrue = ifTrueChild;
// Replace ifFalse with its child, and reuse that node outside.
auto* reuse = curr->ifFalse;
curr->ifFalse = ifFalseChild;
// curr's type may have changed, if the instructions we moved out
// had different input types than output types.
curr->finalize();
// Point to curr from the code that is now outside of it.
*ChildIterator(reuse).begin() = curr;
if (!chain.empty()) {
// We've already moved things out, so chain them to there. That
// is, the end of the chain should now point to reuse (which
// in turn already points to curr).
*ChildIterator(chain.back()).begin() = reuse;
}
chain.push_back(reuse);
continue;
}
}
}
break;
}
if (!chain.empty()) {
// The beginning of the chain is the new top parent.
return replaceCurrent(chain[0]);
}
}
}
};
Pass* createOptimizeInstructionsPass() { return new OptimizeInstructions; }
} // namespace wasm