Google Git
Sign in
chromium / v8 / v8 / d8a922f9a688f0214a58eede7faf1a7b93bc700d / . / src / wasm / module-compiler.cc
blob: fb40c496e802c30f8d911eb1b2f26b33184e3ca9 [file] [log] [blame]
// Copyright 2017 the V8 project authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include "src/wasm/module-compiler.h"
#include <algorithm>
#include <atomic>
#include <memory>
#include <queue>
#include "src/api/api-inl.h"
#include "src/base/enum-set.h"
#include "src/base/fpu.h"
#include "src/base/platform/mutex.h"
#include "src/base/platform/semaphore.h"
#include "src/base/platform/time.h"
#include "src/codegen/compiler.h"
#include "src/compiler/wasm-compiler.h"
#include "src/debug/debug.h"
#include "src/handles/global-handles-inl.h"
#include "src/logging/counters-scopes.h"
#include "src/logging/metrics.h"
#include "src/tracing/trace-event.h"
#include "src/wasm/code-space-access.h"
#include "src/wasm/compilation-environment-inl.h"
#include "src/wasm/jump-table-assembler.h"
#include "src/wasm/module-decoder.h"
#include "src/wasm/pgo.h"
#include "src/wasm/std-object-sizes.h"
#include "src/wasm/streaming-decoder.h"
#include "src/wasm/wasm-code-manager.h"
#include "src/wasm/wasm-code-pointer-table-inl.h"
#include "src/wasm/wasm-engine.h"
#include "src/wasm/wasm-feature-flags.h"
#include "src/wasm/wasm-import-wrapper-cache.h"
#include "src/wasm/wasm-js.h"
#include "src/wasm/wasm-limits.h"
#include "src/wasm/wasm-objects-inl.h"
#include "src/wasm/wasm-result.h"
#include "src/wasm/wasm-serialization.h"
#define TRACE_COMPILE(...) \
do { \
if (v8_flags.trace_wasm_compiler) PrintF(__VA_ARGS__); \
} while (false)
#define TRACE_STREAMING(...) \
do { \
if (v8_flags.trace_wasm_streaming) PrintF(__VA_ARGS__); \
} while (false)
#define TRACE_LAZY(...) \
do { \
if (v8_flags.trace_wasm_lazy_compilation) PrintF(__VA_ARGS__); \
} while (false)
namespace v8::internal::wasm {
namespace {
enum class CompileStrategy : uint8_t {
// Compiles functions on first use. In this case, execution will block until
// the function's baseline is reached and top tier compilation starts in
// background (if applicable).
// Lazy compilation can help to reduce startup time and code size at the risk
// of blocking execution.
kLazy,
// Compiles baseline ahead of execution and starts top tier compilation in
// background (if applicable).
kEager,
// Triggers baseline compilation on first use (just like {kLazy}) with the
// difference that top tier compilation is started eagerly.
// This strategy can help to reduce startup time at the risk of blocking
// execution, but only in its early phase (until top tier compilation
// finishes).
kLazyBaselineEagerTopTier,
// Marker for default strategy.
kDefault = kEager,
};
class CompilationStateImpl;
class CompilationUnitBuilder;
class V8_NODISCARD BackgroundCompileScope {
public:
explicit BackgroundCompileScope(std::weak_ptr<NativeModule> native_module)
: native_module_(native_module.lock()) {}
NativeModule* native_module() const {
DCHECK(native_module_);
return native_module_.get();
}
inline CompilationStateImpl* compilation_state() const;
bool cancelled() const;
private:
// Keep the native module alive while in this scope.
std::shared_ptr<NativeModule> native_module_;
};
enum CompilationTier { kBaseline = 0, kTopTier = 1, kNumTiers = kTopTier + 1 };
// A set of work-stealing queues (vectors of units). Each background compile
// task owns one of the queues and steals from all others once its own queue
// runs empty.
class CompilationUnitQueues {
public:
// Public API for QueueImpl.
struct Queue {
bool ShouldPublish(int num_processed_units) const;
};
explicit CompilationUnitQueues(int num_imported_functions,
int num_declared_functions)
: num_imported_functions_(num_imported_functions),
num_declared_functions_(num_declared_functions) {
// Add one first queue, to add units to.
queues_.emplace_back(std::make_unique<QueueImpl>(0));
#if !defined(__cpp_lib_atomic_value_initialization) || \
__cpp_lib_atomic_value_initialization < 201911L
for (auto& atomic_counter : num_units_) {
std::atomic_init(&atomic_counter, size_t{0});
}
#endif
top_tier_compiled_ =
std::make_unique<std::atomic<bool>[]>(num_declared_functions);
#if !defined(__cpp_lib_atomic_value_initialization) || \
__cpp_lib_atomic_value_initialization < 201911L
for (int i = 0; i < num_declared_functions; i++) {
std::atomic_init(&top_tier_compiled_.get()[i], false);
}
#endif
}
Queue* GetQueueForTask(int task_id) {
int required_queues = task_id + 1;
{
base::MutexGuard queues_guard{&queues_mutex_};
if (V8_LIKELY(static_cast<int>(queues_.size()) >= required_queues)) {
return queues_[task_id].get();
}
}
// Otherwise increase the number of queues.
base::MutexGuard queues_guard{&queues_mutex_};
int num_queues = static_cast<int>(queues_.size());
while (num_queues < required_queues) {
int steal_from = num_queues + 1;
queues_.emplace_back(std::make_unique<QueueImpl>(steal_from));
++num_queues;
}
// Update the {publish_limit}s of all queues.
// We want background threads to publish regularly (to avoid contention when
// they are all publishing at the end). On the other side, each publishing
// has some overhead (part of it for synchronizing between threads), so it
// should not happen *too* often. Thus aim for 4-8 publishes per thread, but
// distribute it such that publishing is likely to happen at different
// times.
int units_per_thread = num_declared_functions_ / num_queues;
int min = std::max(10, units_per_thread / 8);
int queue_id = 0;
for (auto& queue : queues_) {
// Set a limit between {min} and {2*min}, but not smaller than {10}.
int limit = min + (min * queue_id / num_queues);
queue->publish_limit.store(limit, std::memory_order_relaxed);
++queue_id;
}
return queues_[task_id].get();
}
std::optional<WasmCompilationUnit> GetNextUnit(Queue* queue,
CompilationTier tier) {
DCHECK_LT(tier, CompilationTier::kNumTiers);
if (auto unit = GetNextUnitOfTier(queue, tier)) {
[[maybe_unused]] size_t old_units_count =
num_units_[tier].fetch_sub(1, std::memory_order_relaxed);
DCHECK_LE(1, old_units_count);
return unit;
}
return {};
}
void AddUnits(base::Vector<WasmCompilationUnit> baseline_units,
base::Vector<WasmCompilationUnit> top_tier_units,
const WasmModule* module) {
DCHECK_LT(0, baseline_units.size() + top_tier_units.size());
// Add to the individual queues in a round-robin fashion. No special care is
// taken to balance them; they will be balanced by work stealing.
QueueImpl* queue;
{
int queue_to_add = next_queue_to_add.load(std::memory_order_relaxed);
base::MutexGuard queues_guard{&queues_mutex_};
while (!next_queue_to_add.compare_exchange_weak(
queue_to_add, next_task_id(queue_to_add, queues_.size()),
std::memory_order_relaxed)) {
// Retry with updated {queue_to_add}.
}
queue = queues_[queue_to_add].get();
}
base::MutexGuard guard(&queue->mutex);
std::optional<base::MutexGuard> big_units_guard;
for (auto pair :
{std::make_pair(CompilationTier::kBaseline, baseline_units),
std::make_pair(CompilationTier::kTopTier, top_tier_units)}) {
int tier = pair.first;
base::Vector<WasmCompilationUnit> units = pair.second;
if (units.empty()) continue;
num_units_[tier].fetch_add(units.size(), std::memory_order_relaxed);
for (WasmCompilationUnit unit : units) {
size_t func_size = module->functions[unit.func_index()].code.length();
if (func_size <= kBigUnitsLimit) {
queue->units[tier].push_back(unit);
} else {
if (!big_units_guard) {
big_units_guard.emplace(&big_units_queue_.mutex);
}
big_units_queue_.has_units[tier].store(true,
std::memory_order_relaxed);
big_units_queue_.units[tier].emplace(func_size, unit);
}
}
}
}
void AddTopTierPriorityUnit(WasmCompilationUnit unit, size_t priority) {
base::MutexGuard queues_guard{&queues_mutex_};
// Add to the individual queues in a round-robin fashion. No special care is
// taken to balance them; they will be balanced by work stealing.
// Priorities should only be seen as a hint here; without balancing, we
// might pop a unit with lower priority from one queue while other queues
// still hold higher-priority units.
// Since updating priorities in a std::priority_queue is difficult, we just
// add new units with higher priorities, and use the
// {CompilationUnitQueues::top_tier_compiled_} array to discard units for
// functions which are already being compiled.
int queue_to_add = next_queue_to_add.load(std::memory_order_relaxed);
while (!next_queue_to_add.compare_exchange_weak(
queue_to_add, next_task_id(queue_to_add, queues_.size()),
std::memory_order_relaxed)) {
// Retry with updated {queue_to_add}.
}
{
auto* queue = queues_[queue_to_add].get();
base::MutexGuard guard(&queue->mutex);
queue->top_tier_priority_units.emplace(priority, unit);
num_priority_units_.fetch_add(1, std::memory_order_relaxed);
num_units_[CompilationTier::kTopTier].fetch_add(
1, std::memory_order_relaxed);
}
}
// Get the current number of units in the queue for |tier|. This is only a
// momentary snapshot, it's not guaranteed that {GetNextUnit} returns a unit
// if this method returns non-zero.
size_t GetSizeForTier(CompilationTier tier) const {
DCHECK_LT(tier, CompilationTier::kNumTiers);
return num_units_[tier].load(std::memory_order_relaxed);
}
void AllowAnotherTopTierJob(uint32_t func_index) {
top_tier_compiled_[declared_function_index(func_index)].store(
false, std::memory_order_relaxed);
}
size_t EstimateCurrentMemoryConsumption() const;
private:
// Functions bigger than {kBigUnitsLimit} will be compiled first, in ascending
// order of their function body size.
static constexpr size_t kBigUnitsLimit = 4096;
struct BigUnit {
BigUnit(size_t func_size, WasmCompilationUnit unit)
: func_size{func_size}, unit(unit) {}
size_t func_size;
WasmCompilationUnit unit;
bool operator<(const BigUnit& other) const {
return func_size < other.func_size;
}
};
struct TopTierPriorityUnit {
TopTierPriorityUnit(int priority, WasmCompilationUnit unit)
: priority(priority), unit(unit) {}
size_t priority;
WasmCompilationUnit unit;
bool operator<(const TopTierPriorityUnit& other) const {
return priority < other.priority;
}
};
struct BigUnitsQueue {
BigUnitsQueue() {
#if !defined(__cpp_lib_atomic_value_initialization) || \
__cpp_lib_atomic_value_initialization < 201911L
for (auto& atomic : has_units) std::atomic_init(&atomic, false);
#endif
}
mutable base::Mutex mutex;
// Can be read concurrently to check whether any elements are in the queue.
std::atomic<bool> has_units[CompilationTier::kNumTiers];
// Protected by {mutex}:
std::priority_queue<BigUnit> units[CompilationTier::kNumTiers];
};
struct QueueImpl : public Queue {
explicit QueueImpl(int next_steal_task_id)
: next_steal_task_id(next_steal_task_id) {}
// Number of units after which the task processing this queue should publish
// compilation results. Updated (reduced, using relaxed ordering) when new
// queues are allocated. If there is only one thread running, we can delay
// publishing arbitrarily.
std::atomic<int> publish_limit{kMaxInt};
base::Mutex mutex;
// All fields below are protected by {mutex}.
std::vector<WasmCompilationUnit> units[CompilationTier::kNumTiers];
std::priority_queue<TopTierPriorityUnit> top_tier_priority_units;
int next_steal_task_id;
};
int next_task_id(int task_id, size_t num_queues) const {
int next = task_id + 1;
return next == static_cast<int>(num_queues) ? 0 : next;
}
std::optional<WasmCompilationUnit> GetNextUnitOfTier(Queue* public_queue,
int tier) {
QueueImpl* queue = static_cast<QueueImpl*>(public_queue);
// First check whether there is a priority unit. Execute that first.
if (tier == CompilationTier::kTopTier) {
if (auto unit = GetTopTierPriorityUnit(queue)) {
return unit;
}
}
// Then check whether there is a big unit of that tier.
if (auto unit = GetBigUnitOfTier(tier)) return unit;
// Finally check whether our own queue has a unit of the wanted tier. If
// so, return it, otherwise get the task id to steal from.
int steal_task_id;
{
base::MutexGuard mutex_guard(&queue->mutex);
if (!queue->units[tier].empty()) {
auto unit = queue->units[tier].back();
queue->units[tier].pop_back();
return unit;
}
steal_task_id = queue->next_steal_task_id;
}
// Try to steal from all other queues. If this succeeds, return one of the
// stolen units.
{
base::MutexGuard guard{&queues_mutex_};
for (size_t steal_trials = 0; steal_trials < queues_.size();
++steal_trials, ++steal_task_id) {
if (steal_task_id >= static_cast<int>(queues_.size())) {
steal_task_id = 0;
}
if (auto unit = StealUnitsAndGetFirst(queue, steal_task_id, tier)) {
return unit;
}
}
}
// If we reach here, we didn't find any unit of the requested tier.
return {};
}
std::optional<WasmCompilationUnit> GetBigUnitOfTier(int tier) {
// Fast path without locking.
if (!big_units_queue_.has_units[tier].load(std::memory_order_relaxed)) {
return {};
}
base::MutexGuard guard(&big_units_queue_.mutex);
if (big_units_queue_.units[tier].empty()) return {};
WasmCompilationUnit unit = big_units_queue_.units[tier].top().unit;
big_units_queue_.units[tier].pop();
if (big_units_queue_.units[tier].empty()) {
big_units_queue_.has_units[tier].store(false, std::memory_order_relaxed);
}
return unit;
}
std::optional<WasmCompilationUnit> GetTopTierPriorityUnit(QueueImpl* queue) {
// Fast path without locking.
if (num_priority_units_.load(std::memory_order_relaxed) == 0) {
return {};
}
int steal_task_id;
{
base::MutexGuard mutex_guard(&queue->mutex);
while (!queue->top_tier_priority_units.empty()) {
auto unit = queue->top_tier_priority_units.top().unit;
queue->top_tier_priority_units.pop();
num_priority_units_.fetch_sub(1, std::memory_order_relaxed);
if (!top_tier_compiled_[declared_function_index(unit.func_index())]
.exchange(true, std::memory_order_relaxed)) {
return unit;
}
num_units_[CompilationTier::kTopTier].fetch_sub(
1, std::memory_order_relaxed);
}
steal_task_id = queue->next_steal_task_id;
}
// Try to steal from all other queues. If this succeeds, return one of the
// stolen units.
{
base::MutexGuard guard{&queues_mutex_};
for (size_t steal_trials = 0; steal_trials < queues_.size();
++steal_trials, ++steal_task_id) {
if (steal_task_id >= static_cast<int>(queues_.size())) {
steal_task_id = 0;
}
if (auto unit = StealTopTierPriorityUnit(queue, steal_task_id)) {
return unit;
}
}
}
return {};
}
// Steal units of {wanted_tier} from {steal_from_task_id} to {queue}. Return
// first stolen unit (rest put in queue of {task_id}), or {nullopt} if
// {steal_from_task_id} had no units of {wanted_tier}.
// Hold a shared lock on {queues_mutex_} when calling this method.
std::optional<WasmCompilationUnit> StealUnitsAndGetFirst(
QueueImpl* queue, int steal_from_task_id, int wanted_tier) {
auto* steal_queue = queues_[steal_from_task_id].get();
// Cannot steal from own queue.
if (steal_queue == queue) return {};
std::vector<WasmCompilationUnit> stolen;
std::optional<WasmCompilationUnit> returned_unit;
{
base::MutexGuard guard(&steal_queue->mutex);
auto* steal_from_vector = &steal_queue->units[wanted_tier];
if (steal_from_vector->empty()) return {};
size_t remaining = steal_from_vector->size() / 2;
auto steal_begin = steal_from_vector->begin() + remaining;
returned_unit = *steal_begin;
stolen.assign(steal_begin + 1, steal_from_vector->end());
steal_from_vector->erase(steal_begin, steal_from_vector->end());
}
base::MutexGuard guard(&queue->mutex);
auto* target_queue = &queue->units[wanted_tier];
target_queue->insert(target_queue->end(), stolen.begin(), stolen.end());
queue->next_steal_task_id = steal_from_task_id + 1;
return returned_unit;
}
// Steal one priority unit from {steal_from_task_id} to {task_id}. Return
// stolen unit, or {nullopt} if {steal_from_task_id} had no priority units.
// Hold a shared lock on {queues_mutex_} when calling this method.
std::optional<WasmCompilationUnit> StealTopTierPriorityUnit(
QueueImpl* queue, int steal_from_task_id) {
auto* steal_queue = queues_[steal_from_task_id].get();
// Cannot steal from own queue.
if (steal_queue == queue) return {};
std::optional<WasmCompilationUnit> returned_unit;
{
base::MutexGuard guard(&steal_queue->mutex);
while (true) {
if (steal_queue->top_tier_priority_units.empty()) return {};
auto unit = steal_queue->top_tier_priority_units.top().unit;
steal_queue->top_tier_priority_units.pop();
num_priority_units_.fetch_sub(1, std::memory_order_relaxed);
if (!top_tier_compiled_[declared_function_index(unit.func_index())]
.exchange(true, std::memory_order_relaxed)) {
returned_unit = unit;
break;
}
num_units_[CompilationTier::kTopTier].fetch_sub(
1, std::memory_order_relaxed);
}
}
base::MutexGuard guard(&queue->mutex);
queue->next_steal_task_id = steal_from_task_id + 1;
return returned_unit;
}
int declared_function_index(int func_index) const {
DCHECK_LE(num_imported_functions_, func_index);
DCHECK_LT(func_index, num_imported_functions_ + num_declared_functions_);
return func_index - num_imported_functions_;
}
// {queues_mutex_} protectes {queues_};
mutable base::Mutex queues_mutex_;
std::vector<std::unique_ptr<QueueImpl>> queues_;
const int num_imported_functions_;
const int num_declared_functions_;
BigUnitsQueue big_units_queue_;
std::atomic<size_t> num_units_[CompilationTier::kNumTiers];
std::atomic<size_t> num_priority_units_{0};
std::unique_ptr<std::atomic<bool>[]> top_tier_compiled_;
std::atomic<int> next_queue_to_add{0};
};
size_t CompilationUnitQueues::EstimateCurrentMemoryConsumption() const {
UPDATE_WHEN_CLASS_CHANGES(CompilationUnitQueues, 176);
UPDATE_WHEN_CLASS_CHANGES(QueueImpl, 112);
UPDATE_WHEN_CLASS_CHANGES(BigUnitsQueue, 88);
// Not including sizeof(CompilationUnitQueues) because that's included in
// sizeof(CompilationStateImpl).
size_t result = 0;
{
base::MutexGuard mutex_guard(&queues_mutex_);
result += ContentSize(queues_) + queues_.size() * sizeof(QueueImpl);
for (const auto& q : queues_) {
base::MutexGuard guard(&q->mutex);
result += ContentSize(*q->units);
result += q->top_tier_priority_units.size() * sizeof(TopTierPriorityUnit);
}
}
{
base::MutexGuard lock(&big_units_queue_.mutex);
result += big_units_queue_.units[0].size() * sizeof(BigUnit);
result += big_units_queue_.units[1].size() * sizeof(BigUnit);
}
// For {top_tier_compiled_}.
result += sizeof(std::atomic<bool>) * num_declared_functions_;
return result;
}
bool CompilationUnitQueues::Queue::ShouldPublish(
int num_processed_units) const {
auto* queue = static_cast<const QueueImpl*>(this);
return num_processed_units >=
queue->publish_limit.load(std::memory_order_relaxed);
}
// The {CompilationStateImpl} keeps track of the compilation state of the
// owning NativeModule, i.e. which functions are left to be compiled.
// It contains a task manager to allow parallel and asynchronous background
// compilation of functions.
// Its public interface {CompilationState} lives in compilation-environment.h.
class CompilationStateImpl {
public:
CompilationStateImpl(const std::shared_ptr<NativeModule>& native_module,
std::shared_ptr<Counters> async_counters,
DynamicTiering dynamic_tiering,
WasmDetectedFeatures detected_features);
~CompilationStateImpl() {
if (baseline_compile_job_->IsValid()) {
baseline_compile_job_->CancelAndDetach();
}
if (top_tier_compile_job_->IsValid()) {
top_tier_compile_job_->CancelAndDetach();
}
}
// Call right after the constructor, after the {compilation_state_} field in
// the {NativeModule} has been initialized.
void InitCompileJob();
// {kCancelUnconditionally}: Cancel all compilation.
// {kCancelInitialCompilation}: Cancel all compilation if initial (baseline)
// compilation is not finished yet.
enum CancellationPolicy { kCancelUnconditionally, kCancelInitialCompilation };
void CancelCompilation(CancellationPolicy);
bool cancelled() const;
// Apply a compilation hint to the initial compilation progress, updating all
// internal fields accordingly.
void ApplyCompilationHintToInitialProgress(const WasmCompilationHint& hint,
size_t hint_idx);
// Use PGO information to choose a better initial compilation progress
// (tiering decisions).
void ApplyPgoInfoToInitialProgress(ProfileInformation* pgo_info);
// Apply PGO information to a fully initialized compilation state. Also
// trigger compilation as needed.
void ApplyPgoInfoLate(ProfileInformation* pgo_info);
// Initialize compilation progress. Set compilation tiers to expect for
// baseline and top tier compilation. Must be set before
// {CommitCompilationUnits} is invoked which triggers background compilation.
void InitializeCompilationProgress(ProfileInformation* pgo_info);
void InitializeCompilationProgressAfterDeserialization(
base::Vector<const int> lazy_functions,
base::Vector<const int> eager_functions);
// Initializes compilation units based on the information encoded in the
// {compilation_progress_}.
void InitializeCompilationUnits(
std::unique_ptr<CompilationUnitBuilder> builder);
// Adds compilation units for another function to the
// {CompilationUnitBuilder}. This function is the streaming compilation
// equivalent to {InitializeCompilationUnits}.
void AddCompilationUnit(CompilationUnitBuilder* builder, int func_index);
// Add the callback to be called on compilation events. Needs to be
// set before {CommitCompilationUnits} is run to ensure that it receives all
// events. The callback object must support being deleted from any thread.
void AddCallback(std::unique_ptr<CompilationEventCallback> callback);
// Inserts new functions to compile and kicks off compilation.
void CommitCompilationUnits(base::Vector<WasmCompilationUnit> baseline_units,
base::Vector<WasmCompilationUnit> top_tier_units);
void CommitTopTierCompilationUnit(WasmCompilationUnit);
void AddTopTierPriorityCompilationUnit(WasmCompilationUnit, size_t);
CompilationUnitQueues::Queue* GetQueueForCompileTask(int task_id);
std::optional<WasmCompilationUnit> GetNextCompilationUnit(
CompilationUnitQueues::Queue*, CompilationTier tier);
void OnFinishedUnits(base::Vector<WasmCode*>);
void OnCompilationStopped(WasmDetectedFeatures detected);
void SchedulePublishCompilationResults(
std::vector<UnpublishedWasmCode> unpublished_code, CompilationTier tier);
WasmDetectedFeatures detected_features() const {
return detected_features_.load(std::memory_order_relaxed);
}
// Update the set of detected features; returns all features that were not
// detected before.
V8_WARN_UNUSED_RESULT WasmDetectedFeatures
UpdateDetectedFeatures(WasmDetectedFeatures);
size_t NumOutstandingCompilations(CompilationTier tier) const;
void SetError();
void WaitForCompilationEvent(CompilationEvent event);
void TierUpAllFunctions();
void AllowAnotherTopTierJob(uint32_t func_index) {
compilation_unit_queues_.AllowAnotherTopTierJob(func_index);
// Reset the stored priority; otherwise triggers might be ignored if the
// priority is not bumped to the next power of two.
TypeFeedbackStorage* feedback = &native_module_->module()->type_feedback;
base::MutexGuard mutex_guard(&feedback->mutex);
feedback->feedback_for_function[func_index].tierup_priority = 0;
}
void AllowAnotherTopTierJobForAllFunctions() {
const WasmModule* module = native_module_->module();
uint32_t fn_start = module->num_imported_functions;
uint32_t fn_end = fn_start + module->num_declared_functions;
base::MutexGuard mutex_guard(&module->type_feedback.mutex);
std::unordered_map<uint32_t, FunctionTypeFeedback>& feedback_map =
module->type_feedback.feedback_for_function;
for (uint32_t i = fn_start; i < fn_end; i++) {
compilation_unit_queues_.AllowAnotherTopTierJob(i);
// Reset the stored priority; otherwise triggers might be ignored if the
// priority is not bumped to the next power of two.
if (auto it = feedback_map.find(i); it != feedback_map.end()) {
it->second.tierup_priority = 0;
}
}
}
bool failed() const {
return compile_failed_.load(std::memory_order_relaxed);
}
bool baseline_compilation_finished() const {
base::MutexGuard guard(&callbacks_mutex_);
return outstanding_baseline_units_ == 0;
}
DynamicTiering dynamic_tiering() const { return dynamic_tiering_; }
Counters* counters() const { return async_counters_.get(); }
void SetWireBytesStorage(
std::shared_ptr<WireBytesStorage> wire_bytes_storage) {
base::MutexGuard guard(&mutex_);
wire_bytes_storage_ = std::move(wire_bytes_storage);
}
std::shared_ptr<WireBytesStorage> GetWireBytesStorage() const {
base::MutexGuard guard(&mutex_);
DCHECK_NOT_NULL(wire_bytes_storage_);
return wire_bytes_storage_;
}
void set_compilation_id(int compilation_id) {
DCHECK_EQ(compilation_id_, kInvalidCompilationID);
compilation_id_ = compilation_id;
}
size_t EstimateCurrentMemoryConsumption() const;
// Called from the delayed task to trigger caching if the timeout
// (--wasm-caching-timeout-ms) has passed since the last top-tier compilation.
// This either triggers caching or re-schedules the task if more code has
// been compiled to the top tier in the meantime.
void TriggerCachingAfterTimeout();
std::vector<WasmCode*> PublishCode(base::Vector<UnpublishedWasmCode> codes);
private:
void AddCompilationUnitInternal(CompilationUnitBuilder* builder,
int function_index,
uint8_t function_progress);
// Trigger callbacks according to the internal counters below
// (outstanding_...).
// Hold the {callbacks_mutex_} when calling this method.
void TriggerOutstandingCallbacks();
// Trigger an exact set of callbacks. Hold the {callbacks_mutex_} when calling
// this method.
void TriggerCallbacks(base::EnumSet<CompilationEvent>);
void PublishCompilationResults(
std::vector<UnpublishedWasmCode> unpublished_code);
NativeModule* const native_module_;
std::weak_ptr<NativeModule> const native_module_weak_;
const std::shared_ptr<Counters> async_counters_;
// Compilation error, atomically updated. This flag can be updated and read
// using relaxed semantics.
std::atomic<bool> compile_failed_{false};
// True if compilation was cancelled and worker threads should return. This
// flag can be updated and read using relaxed semantics.
std::atomic<bool> compile_cancelled_{false};
CompilationUnitQueues compilation_unit_queues_;
// Cache the dynamic tiering configuration to be consistent for the whole
// compilation.
const DynamicTiering dynamic_tiering_;
// This mutex protects all information of this {CompilationStateImpl} which is
// being accessed concurrently.
mutable base::Mutex mutex_;
// The compile job handles, initialized right after construction of
// {CompilationStateImpl}.
std::unique_ptr<JobHandle> baseline_compile_job_;
std::unique_ptr<JobHandle> top_tier_compile_job_;
// The compilation id to identify trace events linked to this compilation.
static constexpr int kInvalidCompilationID = -1;
int compilation_id_ = kInvalidCompilationID;
// Features detected to be used in this module. Features can be detected
// as a module is being compiled.
std::atomic<WasmDetectedFeatures> detected_features_;
//////////////////////////////////////////////////////////////////////////////
// Protected by {mutex_}:
// Abstraction over the storage of the wire bytes. Held in a shared_ptr so
// that background compilation jobs can keep the storage alive while
// compiling.
std::shared_ptr<WireBytesStorage> wire_bytes_storage_;
// End of fields protected by {mutex_}.
//////////////////////////////////////////////////////////////////////////////
// This mutex protects the callbacks vector, and the counters used to
// determine which callbacks to call. The counters plus the callbacks
// themselves need to be synchronized to ensure correct order of events.
mutable base::Mutex callbacks_mutex_;
//////////////////////////////////////////////////////////////////////////////
// Protected by {callbacks_mutex_}:
// Callbacks to be called on compilation events.
std::vector<std::unique_ptr<CompilationEventCallback>> callbacks_;
// Events that already happened.
base::EnumSet<CompilationEvent> finished_events_;
int outstanding_baseline_units_ = 0;
// The amount of generated top tier code since the last
// {kFinishedCompilationChunk} event.
size_t bytes_since_last_chunk_ = 0;
std::vector<uint8_t> compilation_progress_;
// The timestamp of the last top-tier compilation.
// This field is updated on every publishing of top-tier code, and is reset
// once caching is triggered. Hence it also informs whether a caching task is
// currently being scheduled (whenever this is set).
base::TimeTicks last_top_tier_compilation_timestamp_;
// End of fields protected by {callbacks_mutex_}.
//////////////////////////////////////////////////////////////////////////////
struct PublishState {
// {mutex_} protects {publish_queue_} and {publisher_running_}.
base::Mutex mutex_;
std::vector<UnpublishedWasmCode> publish_queue_;
bool publisher_running_ = false;
};
PublishState publish_state_[CompilationTier::kNumTiers];
// Encoding of fields in the {compilation_progress_} vector.
using RequiredBaselineTierField = base::BitField8<ExecutionTier, 0, 2>;
using RequiredTopTierField = base::BitField8<ExecutionTier, 2, 2>;
using ReachedTierField = base::BitField8<ExecutionTier, 4, 2>;
};
CompilationStateImpl* Impl(CompilationState* compilation_state) {
return reinterpret_cast<CompilationStateImpl*>(compilation_state);
}
const CompilationStateImpl* Impl(const CompilationState* compilation_state) {
return reinterpret_cast<const CompilationStateImpl*>(compilation_state);
}
CompilationStateImpl* BackgroundCompileScope::compilation_state() const {
DCHECK(native_module_);
return Impl(native_module_->compilation_state());
}
size_t CompilationStateImpl::EstimateCurrentMemoryConsumption() const {
UPDATE_WHEN_CLASS_CHANGES(CompilationStateImpl, 472);
size_t result = sizeof(CompilationStateImpl);
{
base::MutexGuard guard{&mutex_};
result += compilation_unit_queues_.EstimateCurrentMemoryConsumption();
}
// To read the size of {callbacks_} and {compilation_progress_}, we'd
// need to acquire the {callbacks_mutex_}, which can cause deadlocks
// when that mutex is already held elsewhere and another thread calls
// into this function. So we rely on heuristics and informed guesses
// instead: {compilation_progress_} contains an entry for every declared
// function in the module...
result += sizeof(uint8_t) * native_module_->module()->num_declared_functions;
// ...and there are typically no more than a handful of {callbacks_}.
constexpr size_t kAssumedNumberOfCallbacks = 4;
constexpr size_t size_of_vector =
kAssumedNumberOfCallbacks *
sizeof(std::unique_ptr<CompilationEventCallback>);
// Concrete subclasses of CompilationEventCallback will be bigger, but we
// can't know that here.
constexpr size_t size_of_payload =
kAssumedNumberOfCallbacks * sizeof(CompilationEventCallback);
result += size_of_vector + size_of_payload;
if (v8_flags.trace_wasm_offheap_memory) {
PrintF("CompilationStateImpl: %zu\n", result);
}
return result;
}
bool BackgroundCompileScope::cancelled() const {
return native_module_ == nullptr ||
Impl(native_module_->compilation_state())->cancelled();
}
} // namespace
//////////////////////////////////////////////////////
// PIMPL implementation of {CompilationState}.
CompilationState::~CompilationState() { Impl(this)->~CompilationStateImpl(); }
void CompilationState::InitCompileJob() { Impl(this)->InitCompileJob(); }
void CompilationState::CancelCompilation() {
Impl(this)->CancelCompilation(CompilationStateImpl::kCancelUnconditionally);
}
void CompilationState::CancelInitialCompilation() {
Impl(this)->CancelCompilation(
CompilationStateImpl::kCancelInitialCompilation);
}
void CompilationState::SetError() { Impl(this)->SetError(); }
void CompilationState::SetWireBytesStorage(
std::shared_ptr<WireBytesStorage> wire_bytes_storage) {
Impl(this)->SetWireBytesStorage(std::move(wire_bytes_storage));
}
std::shared_ptr<WireBytesStorage> CompilationState::GetWireBytesStorage()
const {
return Impl(this)->GetWireBytesStorage();
}
void CompilationState::AddCallback(
std::unique_ptr<CompilationEventCallback> callback) {
return Impl(this)->AddCallback(std::move(callback));
}
void CompilationState::TierUpAllFunctions() {
Impl(this)->TierUpAllFunctions();
}
void CompilationState::AllowAnotherTopTierJob(uint32_t func_index) {
Impl(this)->AllowAnotherTopTierJob(func_index);
}
void CompilationState::AllowAnotherTopTierJobForAllFunctions() {
Impl(this)->AllowAnotherTopTierJobForAllFunctions();
}
void CompilationState::InitializeAfterDeserialization(
base::Vector<const int> lazy_functions,
base::Vector<const int> eager_functions) {
Impl(this)->InitializeCompilationProgressAfterDeserialization(
lazy_functions, eager_functions);
}
bool CompilationState::failed() const { return Impl(this)->failed(); }
bool CompilationState::baseline_compilation_finished() const {
return Impl(this)->baseline_compilation_finished();
}
void CompilationState::set_compilation_id(int compilation_id) {
Impl(this)->set_compilation_id(compilation_id);
}
DynamicTiering CompilationState::dynamic_tiering() const {
return Impl(this)->dynamic_tiering();
}
size_t CompilationState::EstimateCurrentMemoryConsumption() const {
return Impl(this)->EstimateCurrentMemoryConsumption();
}
std::vector<WasmCode*> CompilationState::PublishCode(
base::Vector<UnpublishedWasmCode> unpublished_code) {
return Impl(this)->PublishCode(unpublished_code);
}
// static
std::unique_ptr<CompilationState> CompilationState::New(
const std::shared_ptr<NativeModule>& native_module,
std::shared_ptr<Counters> async_counters, DynamicTiering dynamic_tiering,
WasmDetectedFeatures detected_features) {
return std::unique_ptr<CompilationState>(
reinterpret_cast<CompilationState*>(new CompilationStateImpl(
std::move(native_module), std::move(async_counters), dynamic_tiering,
detected_features)));
}
WasmDetectedFeatures CompilationState::detected_features() const {
return Impl(this)->detected_features();
}
WasmDetectedFeatures CompilationState::UpdateDetectedFeatures(
WasmDetectedFeatures detected_features) {
return Impl(this)->UpdateDetectedFeatures(detected_features);
}
// End of PIMPL implementation of {CompilationState}.
//////////////////////////////////////////////////////
namespace {
ExecutionTier ApplyHintToExecutionTier(WasmCompilationHintTier hint,
ExecutionTier default_tier) {
switch (hint) {
case WasmCompilationHintTier::kDefault:
return default_tier;
case WasmCompilationHintTier::kBaseline:
return ExecutionTier::kLiftoff;
case WasmCompilationHintTier::kOptimized:
return ExecutionTier::kTurbofan;
}
UNREACHABLE();
}
const WasmCompilationHint* GetCompilationHint(const WasmModule* module,
uint32_t func_index) {
DCHECK_LE(module->num_imported_functions, func_index);
uint32_t hint_index = declared_function_index(module, func_index);
const std::vector<WasmCompilationHint>& compilation_hints =
module->compilation_hints;
if (hint_index < compilation_hints.size()) {
return &compilation_hints[hint_index];
}
return nullptr;
}
CompileStrategy GetCompileStrategy(const WasmModule* module,
WasmEnabledFeatures enabled_features,
uint32_t func_index, bool lazy_module) {
if (lazy_module) return CompileStrategy::kLazy;
if (!enabled_features.has_compilation_hints()) {
return CompileStrategy::kDefault;
}
auto* hint = GetCompilationHint(module, func_index);
if (hint == nullptr) return CompileStrategy::kDefault;
switch (hint->strategy) {
case WasmCompilationHintStrategy::kLazy:
return CompileStrategy::kLazy;
case WasmCompilationHintStrategy::kEager:
return CompileStrategy::kEager;
case WasmCompilationHintStrategy::kLazyBaselineEagerTopTier:
return CompileStrategy::kLazyBaselineEagerTopTier;
case WasmCompilationHintStrategy::kDefault:
return CompileStrategy::kDefault;
}
}
struct ExecutionTierPair {
ExecutionTier baseline_tier;
ExecutionTier top_tier;
};
// Pass the debug state as a separate parameter to avoid data races: the debug
// state may change between its use here and its use at the call site. To have
// a consistent view on the debug state, the caller reads the debug state once
// and then passes it to this function.
ExecutionTierPair GetDefaultTiersPerModule(NativeModule* native_module,
DynamicTiering dynamic_tiering,
DebugState is_in_debug_state,
bool lazy_module) {
const WasmModule* module = native_module->module();
if (lazy_module) {
return {ExecutionTier::kNone, ExecutionTier::kNone};
}
if (is_asmjs_module(module)) {
DCHECK(!is_in_debug_state);
return {ExecutionTier::kTurbofan, ExecutionTier::kTurbofan};
}
if (is_in_debug_state) {
return {ExecutionTier::kLiftoff, ExecutionTier::kLiftoff};
}
ExecutionTier baseline_tier =
v8_flags.liftoff ? ExecutionTier::kLiftoff : ExecutionTier::kTurbofan;
bool eager_tier_up = !dynamic_tiering && v8_flags.wasm_tier_up;
ExecutionTier top_tier =
eager_tier_up ? ExecutionTier::kTurbofan : baseline_tier;
return {baseline_tier, top_tier};
}
ExecutionTierPair GetLazyCompilationTiers(NativeModule* native_module,
uint32_t func_index,
DebugState is_in_debug_state) {
DynamicTiering dynamic_tiering =
Impl(native_module->compilation_state())->dynamic_tiering();
// For lazy compilation, get the tiers we would use if lazy compilation is
// disabled.
constexpr bool kNotLazy = false;
ExecutionTierPair tiers = GetDefaultTiersPerModule(
native_module, dynamic_tiering, is_in_debug_state, kNotLazy);
// If we are in debug mode, we ignore compilation hints.
if (is_in_debug_state) return tiers;
// Check if compilation hints override default tiering behaviour.
if (native_module->enabled_features().has_compilation_hints()) {
if (auto* hint = GetCompilationHint(native_module->module(), func_index)) {
tiers.baseline_tier =
ApplyHintToExecutionTier(hint->baseline_tier, tiers.baseline_tier);
tiers.top_tier = ApplyHintToExecutionTier(hint->top_tier, tiers.top_tier);
}
}
if (V8_UNLIKELY(v8_flags.wasm_tier_up_filter >= 0 &&
func_index !=
static_cast<uint32_t>(v8_flags.wasm_tier_up_filter))) {
tiers.top_tier = tiers.baseline_tier;
}
// Correct top tier if necessary.
static_assert(ExecutionTier::kLiftoff < ExecutionTier::kTurbofan,
"Assume an order on execution tiers");
if (tiers.baseline_tier > tiers.top_tier) {
tiers.top_tier = tiers.baseline_tier;
}
return tiers;
}
// The {CompilationUnitBuilder} builds compilation units and stores them in an
// internal buffer. The buffer is moved into the working queue of the
// {CompilationStateImpl} when {Commit} is called.
class CompilationUnitBuilder {
public:
explicit CompilationUnitBuilder(NativeModule* native_module)
: native_module_(native_module) {}
void AddBaselineUnit(int func_index, ExecutionTier tier) {
baseline_units_.emplace_back(func_index, tier, kNotForDebugging);
}
void AddTopTierUnit(int func_index, ExecutionTier tier) {
tiering_units_.emplace_back(func_index, tier, kNotForDebugging);
}
void Commit() {
if (baseline_units_.empty() && tiering_units_.empty()) return;
compilation_state()->CommitCompilationUnits(base::VectorOf(baseline_units_),
base::VectorOf(tiering_units_));
Clear();
}
void Clear() {
baseline_units_.clear();
tiering_units_.clear();
}
const WasmModule* module() { return native_module_->module(); }
private:
CompilationStateImpl* compilation_state() const {
return Impl(native_module_->compilation_state());
}
NativeModule* const native_module_;
std::vector<WasmCompilationUnit> baseline_units_;
std::vector<WasmCompilationUnit> tiering_units_;
};
DecodeResult ValidateSingleFunction(Zone* zone, const WasmModule* module,
int func_index,
base::Vector<const uint8_t> code,
WasmEnabledFeatures enabled_features,
WasmDetectedFeatures* detected_features) {
// Sometimes functions get validated unpredictably in the background, for
// debugging or when inlining one function into another. We check here if that
// is the case, and exit early if so.
if (module->function_was_validated(func_index)) return {};
const WasmFunction* func = &module->functions[func_index];
bool is_shared = module->type(func->sig_index).is_shared;
FunctionBody body{func->sig, func->code.offset(), code.begin(), code.end(),
is_shared};
DecodeResult result = ValidateFunctionBody(zone, enabled_features, module,
detected_features, body);
if (result.ok()) module->set_function_validated(func_index);
return result;
}
enum OnlyLazyFunctions : bool {
kAllFunctions = false,
kOnlyLazyFunctions = true,
};
bool IsLazyModule(const WasmModule* module) {
return v8_flags.wasm_lazy_compilation ||
(v8_flags.asm_wasm_lazy_compilation && is_asmjs_module(module));
}
class CompileLazyTimingScope {
public:
CompileLazyTimingScope(Counters* counters, NativeModule* native_module)
: counters_(counters), native_module_(native_module) {
timer_.Start();
}
~CompileLazyTimingScope() {
base::TimeDelta elapsed = timer_.Elapsed();
native_module_->AddLazyCompilationTimeSample(elapsed.InMicroseconds());
counters_->wasm_lazy_compile_time()->AddTimedSample(elapsed);
}
private:
Counters* counters_;
NativeModule* native_module_;
base::ElapsedTimer timer_;
};
} // namespace
bool CompileLazy(Isolate* isolate,
Tagged<WasmTrustedInstanceData> trusted_instance_data,
int func_index) {
DisallowGarbageCollection no_gc;
NativeModule* native_module = trusted_instance_data->native_module();
Counters* counters = isolate->counters();
// Put the timer scope around everything, including the {CodeSpaceWriteScope}
// and its destruction, to measure complete overhead (apart from the runtime
// function itself, which has constant overhead).
std::optional<CompileLazyTimingScope> lazy_compile_time_scope;
if (base::TimeTicks::IsHighResolution()) {
lazy_compile_time_scope.emplace(counters, native_module);
}
DCHECK(!native_module->lazy_compile_frozen());
TRACE_LAZY("Compiling wasm-function#%d.\n", func_index);
CompilationStateImpl* compilation_state =
Impl(native_module->compilation_state());
DebugState is_in_debug_state = native_module->IsInDebugState();
ExecutionTierPair tiers =
GetLazyCompilationTiers(native_module, func_index, is_in_debug_state);
DCHECK_LE(native_module->num_imported_functions(), func_index);
DCHECK_LT(func_index, native_module->num_functions());
WasmCompilationUnit baseline_unit{
func_index, tiers.baseline_tier,
is_in_debug_state ? kForDebugging : kNotForDebugging};
CompilationEnv env = CompilationEnv::ForModule(native_module);
WasmDetectedFeatures detected_features;
WasmCompilationResult result = baseline_unit.ExecuteCompilation(
&env, compilation_state->GetWireBytesStorage().get(), counters,
&detected_features);
compilation_state->OnCompilationStopped(detected_features);
// During lazy compilation, we can only get compilation errors when
// {--wasm-lazy-validation} is enabled. Otherwise, the module was fully
// verified before starting its execution.
CHECK_IMPLIES(result.failed(), v8_flags.wasm_lazy_validation);
if (result.failed()) {
return false;
}
WasmCodeRefScope code_ref_scope;
WasmCode* code =
native_module->PublishCode(native_module->AddCompiledCode(result));
DCHECK_EQ(func_index, code->index());
if (V8_UNLIKELY(native_module->log_code())) {
GetWasmEngine()->LogCode(base::VectorOf(&code, 1));
// Log the code immediately in the current isolate.
GetWasmEngine()->LogOutstandingCodesForIsolate(isolate);
}
counters->wasm_lazily_compiled_functions()->Increment();
const WasmModule* module = native_module->module();
const bool lazy_module = IsLazyModule(module);
if (GetCompileStrategy(module, native_module->enabled_features(), func_index,
lazy_module) == CompileStrategy::kLazy &&
tiers.baseline_tier < tiers.top_tier) {
WasmCompilationUnit tiering_unit{func_index, tiers.top_tier,
kNotForDebugging};
compilation_state->CommitTopTierCompilationUnit(tiering_unit);
}
return true;
}
void ThrowLazyCompilationError(Isolate* isolate,
const NativeModule* native_module,
int func_index) {
const WasmModule* module = native_module->module();
CompilationStateImpl* compilation_state =
Impl(native_module->compilation_state());
const WasmFunction* func = &module->functions[func_index];
base::Vector<const uint8_t> code =
compilation_state->GetWireBytesStorage()->GetCode(func->code);
auto enabled_features = native_module->enabled_features();
// This path is unlikely, so the overhead for creating an extra Zone is
// not important.
Zone validation_zone{GetWasmEngine()->allocator(), ZONE_NAME};
WasmDetectedFeatures unused_detected_features;
DecodeResult decode_result =
ValidateSingleFunction(&validation_zone, module, func_index, code,
enabled_features, &unused_detected_features);
CHECK(decode_result.failed());
wasm::ErrorThrower thrower(isolate, nullptr);
thrower.CompileFailed(GetWasmErrorWithName(native_module->wire_bytes(),
func_index, module,
std::move(decode_result).error()));
}
// The main purpose of this class is to copy the feedback vectors that live in
// `FixedArray`s on the JavaScript heap to a C++ datastructure on the `module`
// that is accessible to the background compilation threads.
// While we are at it, we also do some light processing here, e.g., mapping the
// feedback to functions, identified by their function index, and filtering out
// feedback for calls to imported functions (which we currently don't inline).
class TransitiveTypeFeedbackProcessor {
public:
static void Process(Isolate* isolate,
Tagged<WasmTrustedInstanceData> trusted_instance_data,
int func_index) {
TransitiveTypeFeedbackProcessor{isolate, trusted_instance_data, func_index}
.ProcessQueue();
}
private:
TransitiveTypeFeedbackProcessor(
Isolate* isolate, Tagged<WasmTrustedInstanceData> trusted_instance_data,
int func_index)
: isolate_(isolate),
instance_data_(trusted_instance_data),
module_(trusted_instance_data->module()),
mutex_guard(&module_->type_feedback.mutex),
feedback_for_function_(module_->type_feedback.feedback_for_function) {
queue_.insert(func_index);
}
~TransitiveTypeFeedbackProcessor() { DCHECK(queue_.empty()); }
void ProcessQueue() {
while (!queue_.empty()) {
auto next = queue_.cbegin();
ProcessFunction(*next);
queue_.erase(next);
}
}
void ProcessFunction(int func_index);
void EnqueueCallees(base::Vector<CallSiteFeedback> feedback) {
for (const CallSiteFeedback& csf : feedback) {
for (int j = 0; j < csf.num_cases(); j++) {
int func = csf.function_index(j);
// Don't spend time on calls that have never been executed.
if (csf.call_count(j) == 0) continue;
// Don't recompute feedback that has already been processed.
auto existing = feedback_for_function_.find(func);
if (existing != feedback_for_function_.end() &&
!existing->second.feedback_vector.empty()) {
if (!existing->second.needs_reprocessing_after_deopt) {
continue;
}
DCHECK(v8_flags.wasm_deopt);
existing->second.needs_reprocessing_after_deopt = false;
}
queue_.insert(func);
}
}
}
DisallowGarbageCollection no_gc_scope_;
Isolate* const isolate_;
const Tagged<WasmTrustedInstanceData> instance_data_;
const WasmModule* const module_;
// TODO(jkummerow): Check if it makes a difference to apply any updates
// as a single batch at the end.
base::MutexGuard mutex_guard;
std::unordered_map<uint32_t, FunctionTypeFeedback>& feedback_for_function_;
std::set<int> queue_;
};
bool IsCrossInstanceCall(Tagged<Object> obj, Isolate* const isolate) {
return obj == ReadOnlyRoots{isolate}.wasm_cross_instance_call_symbol();
}
class FeedbackMaker {
public:
FeedbackMaker(Isolate* const isolate,
Tagged<WasmTrustedInstanceData> trusted_instance_data,
int func_index, int num_calls)
: isolate_(isolate),
instance_data_(trusted_instance_data),
result_(
base::OwnedVector<CallSiteFeedback>::NewForOverwrite(num_calls)),
num_imported_functions_(static_cast<int>(
trusted_instance_data->module()->num_imported_functions)),
func_index_(func_index) {}
void AddCallRefCandidate(Tagged<WasmFuncRef> funcref, int count) {
Tagged<WasmInternalFunction> internal_function =
Cast<WasmFuncRef>(funcref)->internal(isolate_);
// Discard cross-instance calls, as we can only inline same-instance code.
if (internal_function->implicit_arg() != instance_data_) {
has_non_inlineable_targets_ = true;
return;
}
// Discard imports for now.
if (internal_function->function_index() < num_imported_functions_) {
has_non_inlineable_targets_ = true;
return;
}
AddCall(internal_function->function_index(), count);
}
void AddCallIndirectCandidate(Tagged<Object> target_truncated_obj,
int count) {
// Discard cross-instance calls, as we can only inline same-instance code.
if (IsCrossInstanceCall(target_truncated_obj, isolate_)) {
has_non_inlineable_targets_ = true;
return;
}
Tagged<Smi> target_truncated_smi = Cast<Smi>(target_truncated_obj);
// We need to map a truncated call target back to a function index.
// Generally there may be multiple jump tables if code spaces are far apart
// (to ensure that direct calls can always use a near call to the closest
// jump table).
// However, here we are always handling call targets that are originally
// from the `WasmDispatchTable`, whose entries are always targets pointing
// into the main jump table, so we only need to check against that.
WasmCodePointer handle =
WasmCodePointer{static_cast<uint32_t>(target_truncated_smi.value())};
Address entry = GetProcessWideWasmCodePointerTable()
->GetEntrypointWithoutSignatureCheck(handle);
wasm::WasmCode* code =
wasm::GetWasmCodeManager()->LookupCode(nullptr, entry);
if (!code || code->native_module() != instance_data_->native_module() ||
code->IsAnonymous()) {
// Was not in the main table (e.g., because it's an imported function).
has_non_inlineable_targets_ = true;
return;
}
DCHECK_EQ(code->kind(), WasmCode::Kind::kWasmFunction);
uint32_t func_idx = code->index();
AddCall(func_idx, count);
}
void AddCall(int target, int count) {
// If we add too many calls, treat it as megamorphic.
if (static_cast<size_t>(cache_usage_) == targets_cache_.size() ||
is_megamorphic_) {
is_megamorphic_ = true;
return;
}
// Keep the cache sorted (using insertion-sort), highest count first.
int insertion_index = 0;
while (insertion_index < cache_usage_ &&
counts_cache_[insertion_index] >= count) {
insertion_index++;
}
for (int shifted_index = cache_usage_ - 1; shifted_index >= insertion_index;
shifted_index--) {
targets_cache_[shifted_index + 1] = targets_cache_[shifted_index];
counts_cache_[shifted_index + 1] = counts_cache_[shifted_index];
}
targets_cache_[insertion_index] = target;
counts_cache_[insertion_index] = count;
cache_usage_++;
}
bool HasTargetCached(int target) {
auto end = targets_cache_.begin() + cache_usage_;
DCHECK_LE(end, targets_cache_.end());
return std::find(targets_cache_.begin(), end, target) != end;
}
void AddResult(CallSiteFeedback feedback) {
DCHECK_LT(seen_calls_, result_.size());
result_[seen_calls_] = feedback;
++seen_calls_;
}
void FinalizeCall() {
if (is_megamorphic_) {
if (v8_flags.trace_wasm_inlining) {
PrintF("[function %d: call #%d: megamorphic]\n", func_index_,
seen_calls_);
}
AddResult(CallSiteFeedback::CreateMegamorphic());
} else if (cache_usage_ == 0) {
AddResult(CallSiteFeedback{});
} else if (cache_usage_ == 1) {
if (v8_flags.trace_wasm_inlining) {
PrintF("[function %d: call #%d inlineable (monomorphic)]\n",
func_index_, seen_calls_);
}
AddResult(CallSiteFeedback{targets_cache_[0], counts_cache_[0]});
} else {
if (v8_flags.trace_wasm_inlining) {
PrintF("[function %d: call #%d inlineable (polymorphic %d)]\n",
func_index_, seen_calls_, cache_usage_);
}
DCHECK_LE(cache_usage_, kMaxPolymorphism);
CallSiteFeedback::PolymorphicCase* polymorphic =
new CallSiteFeedback::PolymorphicCase[cache_usage_];
for (int i = 0; i < cache_usage_; i++) {
polymorphic[i].function_index = targets_cache_[i];
polymorphic[i].absolute_call_frequency = counts_cache_[i];
}
AddResult(CallSiteFeedback{polymorphic, cache_usage_});
}
result_[seen_calls_ - 1].set_has_non_inlineable_targets(
has_non_inlineable_targets_);
// TODO(mliedtke): Have a better representation that merges these properties
// into one object.
has_non_inlineable_targets_ = false;
is_megamorphic_ = false;
cache_usage_ = 0;
}
void set_has_non_inlineable_targets() { has_non_inlineable_targets_ = true; }
void set_megamorphic() { is_megamorphic_ = true; }
// {GetResult} can only be called on a r-value reference to make it more
// obvious at call sites that {this} should not be used after this operation.
base::OwnedVector<CallSiteFeedback> GetResult() && {
return std::move(result_);
}
private:
Isolate* const isolate_;
const Tagged<WasmTrustedInstanceData> instance_data_;
base::OwnedVector<CallSiteFeedback> result_;
int seen_calls_ = 0;
const int num_imported_functions_;
const int func_index_;
int cache_usage_{0};
std::array<int, kMaxPolymorphism> targets_cache_;
std::array<int, kMaxPolymorphism> counts_cache_;
bool has_non_inlineable_targets_ = false;
// If we add more call targets than kMaxPolymorphism while processing the
// feedback, treat it as megamorphic.
bool is_megamorphic_ = false;
};
void TransitiveTypeFeedbackProcessor::ProcessFunction(int func_index) {
int which_vector = declared_function_index(module_, func_index);
Tagged<Object> maybe_feedback =
instance_data_->feedback_vectors()->get(which_vector);
if (!IsFixedArray(maybe_feedback)) return;
Tagged<FixedArray> feedback = Cast<FixedArray>(maybe_feedback);
base::Vector<uint32_t> call_targets =
module_->type_feedback.feedback_for_function[func_index]
.call_targets.as_vector();
// For each entry in {call_targets}, there are two {Object} slots in the
// {feedback} vector:
// +--------------------------+-----------------------------+----------------+
// | Call Type | Feedback: Entry 1 | Entry 2 |
// +-------------------------+------------------------------+----------------+
// | direct | Smi(count) | Smi(0), unused |
// +--------------------------+-----------------------------+----------------+
// | ref, uninitialized | Smi(0) | Smi(0) |
// | ref, monomorphic | WasmFuncRef(target) | Smi(count>0) |
// | ref, polymorphic | FixedArray | Undefined |
// | ref, megamorphic | MegamorphicSymbol | Undefined |
// +--------------------------+-----------------------------+----------------+
// | indirect, uninitialized | Smi(0) | Smi(0) |
// | indirect, monomorphic | Smi(truncated_target) | Smi(count>0) |
// | indirect, wrong instance | WasmCrossInstanceCallSymbol | Smi(count>0) |
// | indirect, polymorphic | FixedArray | Undefined |
// | indirect, megamorphic | MegamorphicSymbol | Undefined |
// +--------------------------+-----------------------------+----------------+
// The FixedArray entries for the polymorphic cases look like the monomorphic
// entries in the feedback vector itself, i.e., they can a (truncated) target,
// or the wrong instance sentinel (for cross-instance call_indirect).
// See {UpdateCallRefOrIndirectIC} in {wasm.tq} for how this is written.
// Since this is combining untrusted data ({feedback} vector on the JS heap)
// with trusted data ({call_targets}), make sure to avoid an OOB access.
int checked_feedback_length = feedback->length();
SBXCHECK_EQ(checked_feedback_length, call_targets.size() * 2);
FeedbackMaker fm(isolate_, instance_data_, func_index,
checked_feedback_length / 2);
for (int i = 0; i < checked_feedback_length; i += 2) {
uint32_t sentinel_or_target = call_targets[i / 2];
Tagged<Object> first_slot = feedback->get(i);
Tagged<Object> second_slot = feedback->get(i + 1);
if (sentinel_or_target != FunctionTypeFeedback::kCallRef &&
sentinel_or_target != FunctionTypeFeedback::kCallIndirect) {
// Direct call counts.
int count = Smi::ToInt(first_slot);
DCHECK_EQ(Smi::ToInt(second_slot), 0);
// TODO(dlehmann): Currently, TurboFan assumes that we add feedback even
// if the call count is zero. Once TurboFan is gone, revisit if we can
// avoid this (similar to how we do for call_ref/call_indirect today).
fm.AddCall(static_cast<int>(sentinel_or_target), count);
} else if (IsSmi(second_slot) && Smi::ToInt(second_slot) == 0) {
// Uninitialized call_ref or call_indirect.
DCHECK_EQ(Smi::ToInt(first_slot), 0);
if (v8_flags.trace_wasm_inlining) {
PrintF("[function %d: call #%d: uninitialized]\n", func_index, i / 2);
}
} else if (IsWasmFuncRef(first_slot)) {
// Monomorphic call_ref.
DCHECK_EQ(sentinel_or_target, FunctionTypeFeedback::kCallRef);
int count = Smi::ToInt(second_slot);
fm.AddCallRefCandidate(Cast<WasmFuncRef>(first_slot), count);
} else if (IsSmi(first_slot) || IsCrossInstanceCall(first_slot, isolate_)) {
// Monomorphic call_indirect.
DCHECK_EQ(sentinel_or_target, FunctionTypeFeedback::kCallIndirect);
int count = Smi::ToInt(second_slot);
fm.AddCallIndirectCandidate(first_slot, count);
} else if (IsFixedArray(first_slot)) {
// Polymorphic call_ref or call_indirect.
Tagged<FixedArray> polymorphic = Cast<FixedArray>(first_slot);
DCHECK(IsUndefined(second_slot));
int checked_polymorphic_length = polymorphic->length();
SBXCHECK_LE(checked_polymorphic_length, 2 * kMaxPolymorphism);
if (sentinel_or_target == FunctionTypeFeedback::kCallRef) {
for (int j = 0; j < checked_polymorphic_length; j += 2) {
Tagged<WasmFuncRef> target = Cast<WasmFuncRef>(polymorphic->get(j));
int count = Smi::ToInt(polymorphic->get(j + 1));
fm.AddCallRefCandidate(target, count);
}
} else {
DCHECK_EQ(sentinel_or_target, FunctionTypeFeedback::kCallIndirect);
for (int j = 0; j < checked_polymorphic_length; j += 2) {
Tagged<Object> target = polymorphic->get(j);
int count = Smi::ToInt(polymorphic->get(j + 1));
fm.AddCallIndirectCandidate(target, count);
}
}
} else if (first_slot == ReadOnlyRoots{isolate_}.megamorphic_symbol()) {
DCHECK(IsUndefined(second_slot));
fm.set_megamorphic();
} else {
UNREACHABLE();
}
if (v8_flags.wasm_deopt &&
first_slot != ReadOnlyRoots{isolate_}.megamorphic_symbol()) {
// If we already had feedback for this call, also add the already existing
// feedback to prevent deopt loops where two different instantiations
// (which have their own on-heap feedback vector) to "flip-flop" between
// their inlining decisions potentially causing deopt loops.
const base::OwnedVector<CallSiteFeedback>& existing =
feedback_for_function_[func_index].feedback_vector;
size_t feedback_index = i / 2;
if (feedback_index < existing.size()) {
const CallSiteFeedback& old_feedback = existing[feedback_index];
if (old_feedback.has_non_inlineable_targets()) {
fm.set_has_non_inlineable_targets();
}
if (old_feedback.is_megamorphic()) {
fm.set_megamorphic();
}
for (int j = 0; j < old_feedback.num_cases(); ++j) {
int old_target_function_index = old_feedback.function_index(j);
// If the new feedback already contains the target, we do not touch
// the call count.
if (!fm.HasTargetCached(old_target_function_index)) {
fm.AddCall(old_target_function_index, old_feedback.call_count(j));
// There shouldn't be any imported functions in there as they can't
// be inlined. If this DCHECK is invalidated,
// has_non_inlineable_targets_ would need to be updated here to
// reflect that.
DCHECK_GE(static_cast<uint32_t>(old_target_function_index),
instance_data_->module()->num_imported_functions);
}
}
}
}
fm.FinalizeCall();
}
base::OwnedVector<CallSiteFeedback> result = std::move(fm).GetResult();
EnqueueCallees(result.as_vector());
DCHECK_EQ(result.size(),
feedback_for_function_[func_index].call_targets.size());
feedback_for_function_[func_index].feedback_vector = std::move(result);
}
void TriggerTierUp(Isolate* isolate,
Tagged<WasmTrustedInstanceData> trusted_instance_data,
int func_index) {
NativeModule* native_module = trusted_instance_data->native_module();
CompilationStateImpl* compilation_state =
Impl(native_module->compilation_state());
WasmCompilationUnit tiering_unit{func_index, ExecutionTier::kTurbofan,
kNotForDebugging};
const WasmModule* module = native_module->module();
int priority;
{
base::MutexGuard mutex_guard(&module->type_feedback.mutex);
int array_index = wasm::declared_function_index(module, func_index);
trusted_instance_data->tiering_budget_array()[array_index].store(
v8_flags.wasm_tiering_budget, std::memory_order_relaxed);
int& stored_priority =
module->type_feedback.feedback_for_function[func_index].tierup_priority;
if (stored_priority < kMaxInt) ++stored_priority;
priority = stored_priority;
}
// Only create a compilation unit if this is the first time we detect this
// function as hot (priority == 1), or if the priority increased
// significantly. The latter is assumed to be the case if the priority
// increased at least to four, and is a power of two.
if (priority == 2 || !base::bits::IsPowerOfTwo(priority)) return;
// Before adding the tier-up unit or increasing priority, process type
// feedback for best code generation.
if (v8_flags.wasm_inlining) {
// TODO(jkummerow): we could have collisions here if different instances
// of the same module have collected different feedback. If that ever
// becomes a problem, figure out a solution.
TransitiveTypeFeedbackProcessor::Process(isolate, trusted_instance_data,
func_index);
}
compilation_state->AddTopTierPriorityCompilationUnit(tiering_unit, priority);
}
void TierUpNowForTesting(Isolate* isolate,
Tagged<WasmTrustedInstanceData> trusted_instance_data,
int func_index) {
NativeModule* native_module = trusted_instance_data->native_module();
if (v8_flags.wasm_inlining) {
TransitiveTypeFeedbackProcessor::Process(isolate, trusted_instance_data,
func_index);
}
wasm::GetWasmEngine()->CompileFunction(isolate->counters(), native_module,
func_index,
wasm::ExecutionTier::kTurbofan);
CHECK(!native_module->compilation_state()->failed());
}
void TierUpAllForTesting(
Isolate* isolate, Tagged<WasmTrustedInstanceData> trusted_instance_data) {
NativeModule* native_module = trusted_instance_data->native_module();
const WasmModule* mod = native_module->module();
WasmCodeRefScope code_ref_scope;
uint32_t start = mod->num_imported_functions;
uint32_t end = start + mod->num_declared_functions;
for (uint32_t func_index = start; func_index < end; func_index++) {
if (!native_module->HasCodeWithTier(func_index, ExecutionTier::kTurbofan)) {
TierUpNowForTesting(isolate, trusted_instance_data, func_index);
}
}
}
void InitializeCompilationForTesting(NativeModule* native_module) {
Impl(native_module->compilation_state())
->InitializeCompilationProgress(nullptr);
}
void PublishDetectedFeatures(WasmDetectedFeatures detected_features,
Isolate* isolate, bool is_initial_compilation) {
using Feature = v8::Isolate::UseCounterFeature;
static constexpr std::pair<WasmDetectedFeature, Feature> kUseCounters[] = {
{WasmDetectedFeature::shared_memory, Feature::kWasmSharedMemory},
{WasmDetectedFeature::reftypes, Feature::kWasmRefTypes},
{WasmDetectedFeature::simd, Feature::kWasmSimdOpcodes},
{WasmDetectedFeature::threads, Feature::kWasmThreadOpcodes},
{WasmDetectedFeature::legacy_eh, Feature::kWasmExceptionHandling},
{WasmDetectedFeature::memory64, Feature::kWasmMemory64},
{WasmDetectedFeature::multi_memory, Feature::kWasmMultiMemory},
{WasmDetectedFeature::gc, Feature::kWasmGC},
{WasmDetectedFeature::imported_strings, Feature::kWasmImportedStrings},
{WasmDetectedFeature::imported_strings_utf8,
Feature::kWasmImportedStringsUtf8},
{WasmDetectedFeature::return_call, Feature::kWasmReturnCall},
{WasmDetectedFeature::extended_const, Feature::kWasmExtendedConst},
{WasmDetectedFeature::relaxed_simd, Feature::kWasmRelaxedSimd},
{WasmDetectedFeature::type_reflection, Feature::kWasmTypeReflection},
{WasmDetectedFeature::exnref, Feature::kWasmExnRef},
{WasmDetectedFeature::typed_funcref, Feature::kWasmTypedFuncRef},
{WasmDetectedFeature::jspi, Feature::kWasmJavaScriptPromiseIntegration},
{WasmDetectedFeature::branch_hinting, Feature::kWasmBranchHinting},
};
// Check that every staging or shipping feature has a use counter as that is
// the main point of tracking used features.
auto check_use_counter = [](WasmDetectedFeature feat) constexpr -> bool {
// Some features intentionally do not have a use counter.
constexpr WasmDetectedFeature kIntentionallyNoUseCounter[] = {
WasmDetectedFeature::stringref, // Deprecated / unlikely to ship.
WasmDetectedFeature::js_inlining, // Not a user-visible feature.
};
for (auto no_use_counter_feature : kIntentionallyNoUseCounter) {
if (feat == no_use_counter_feature) return true;
}
for (auto [feature, use_counter] : kUseCounters) {
if (feat == feature) return true;
}
return false;
};
#define CHECK_USE_COUNTER(feat, ...) \
static_assert(check_use_counter(WasmDetectedFeature::feat));
FOREACH_WASM_STAGING_FEATURE_FLAG(CHECK_USE_COUNTER)
FOREACH_WASM_SHIPPED_FEATURE_FLAG(CHECK_USE_COUNTER)
FOREACH_WASM_NON_FLAG_FEATURE(CHECK_USE_COUNTER)
#undef CHECK_USE_COUNTER
static constexpr size_t kMaxFeatures = arraysize(kUseCounters) + 1;
base::SmallVector<Feature, kMaxFeatures> use_counter_features;
if (is_initial_compilation) {
// Always set the WasmModuleCompilation feature as a baseline for the other
// features. Note that we also track instantiation, but the number of
// compilations and instantiations are pretty unrelated.
use_counter_features.push_back(Feature::kWasmModuleCompilation);
}
for (auto [wasm_feature, feature] : kUseCounters) {
if (!detected_features.contains(wasm_feature)) continue;
use_counter_features.push_back(feature);
}
if (use_counter_features.empty()) return;
isolate->CountUsage(base::VectorOf(use_counter_features));
// Help differential fuzzers avoid detecting known/intentional platform-
// specific differences.
if (v8_flags.correctness_fuzzer_suppressions) {
if (detected_features.has_relaxed_simd()) {
PrintF("Warning: This run cannot be compared across architectures.\n");
}
}
}
namespace {
bool IsI16Array(wasm::ValueType type, const WasmModule* module) {
if (!type.is_object_reference() || !type.has_index()) return false;
ModuleTypeIndex reftype = type.ref_index();
if (!module->has_array(reftype)) return false;
return module->canonical_type_id(reftype) ==
TypeCanonicalizer::kPredefinedArrayI16Index;
}
bool IsI8Array(wasm::ValueType type, const WasmModule* module,
bool allow_nullable) {
if (!type.is_object_reference() || !type.has_index()) return false;
if (!allow_nullable && type.is_nullable()) return false;
ModuleTypeIndex reftype = type.ref_index();
if (!module->has_array(reftype)) return false;
return module->canonical_type_id(reftype) ==
TypeCanonicalizer::kPredefinedArrayI8Index;
}
// Returns the start offset of a given import, for use in error messages.
// The module_name payload is preceded by an i32v giving its length. That i32v
// is preceded by another i32v, which is either a type index (specifying the
// type of the previous import) or the imports count (in case of the first
// import). So we scan backwards as long as we find non-last LEB bytes there.
uint32_t ImportStartOffset(base::Vector<const uint8_t> wire_bytes,
uint32_t module_name_start) {
DCHECK_LT(0, module_name_start);
uint32_t offset = module_name_start - 1; // Last byte of the string length.
DCHECK_EQ(wire_bytes[offset] & 0x80, 0);
while (offset > 0 && (wire_bytes[offset - 1] & 0x80) != 0) {
offset--;
}
return offset;
}
} // namespace
// Validates the signatures of recognized compile-time imports, and stores
// them on the {module}'s {well_known_imports} list.
WasmError ValidateAndSetBuiltinImports(const WasmModule* module,
base::Vector<const uint8_t> wire_bytes,
const CompileTimeImports& imports,
WasmDetectedFeatures* detected) {
DCHECK_EQ(module->origin, kWasmOrigin);
if (imports.empty()) return {};
static constexpr ValueType kRefExtern = kWasmRefExtern;
static constexpr ValueType kExternRef = kWasmExternRef;
static constexpr ValueType kI32 = kWasmI32;
// Shorthands: "r" = nullable "externref", "e" = non-nullable "ref extern".
static constexpr ValueType kReps_e_i[] = {kRefExtern, kI32};
static constexpr ValueType kReps_e_rr[] = {kRefExtern, kExternRef,
kExternRef};
static constexpr ValueType kReps_e_rii[] = {kRefExtern, kExternRef, kI32,
kI32};
static constexpr ValueType kReps_i_ri[] = {kI32, kExternRef, kI32};
static constexpr ValueType kReps_i_rr[] = {kI32, kExternRef, kExternRef};
static constexpr FunctionSig kSig_e_i(1, 1, kReps_e_i);
static constexpr FunctionSig kSig_e_r(1, 1, kReps_e_rr);
static constexpr FunctionSig kSig_e_rr(1, 2, kReps_e_rr);
static constexpr FunctionSig kSig_e_rii(1, 3, kReps_e_rii);
static constexpr FunctionSig kSig_i_r(1, 1, kReps_i_ri);
static constexpr FunctionSig kSig_i_ri(1, 2, kReps_i_ri);
static constexpr FunctionSig kSig_i_rr(1, 2, kReps_i_rr);
std::vector<WellKnownImport> statuses;
statuses.reserve(module->num_imported_functions);
for (size_t i = 0; i < module->import_table.size(); i++) {
const WasmImport& import = module->import_table[i];
// When magic string imports are requested, check that imports with the
// string constant module name are globals of the right type.
if (imports.has_string_constants(wire_bytes.SubVector(
import.module_name.offset(), import.module_name.end_offset()))) {
if (import.kind != kExternalGlobal ||
!module->globals[import.index].type.is_reference_to(
HeapType::kExtern) ||
module->globals[import.index].mutability != false) {
TruncatedUserString<> name(
wire_bytes.data() + import.field_name.offset(),
import.field_name.length());
return WasmError(
ImportStartOffset(wire_bytes, import.module_name.offset()),
"String constant import #%zu \"%.*s\" must be an immutable global "
"subtyping externref",
i, name.length(), name.start());
}
}
// Check compile-time imported functions.
if (import.kind != kExternalFunction) continue;
base::Vector<const uint8_t> module_name = wire_bytes.SubVector(
import.module_name.offset(), import.module_name.end_offset());
constexpr size_t kMinInterestingLength = 10;
if (module_name.size() < kMinInterestingLength ||
module_name.SubVector(0, 5) != base::StaticOneByteVector("wasm:")) {
statuses.push_back(WellKnownImport::kUninstantiated);
continue;
}
base::Vector<const uint8_t> collection = module_name.SubVectorFrom(5);
WellKnownImport status = WellKnownImport::kUninstantiated;
const WasmFunction& func = module->functions[import.index];
const FunctionSig* sig = func.sig;
WireBytesRef field_name = import.field_name;
base::Vector<const uint8_t> name =
wire_bytes.SubVector(field_name.offset(), field_name.end_offset());
if (collection == base::StaticOneByteVector("js-string") &&
imports.contains(CompileTimeImport::kJsString)) {
#define RETURN_ERROR(module_name_string, import_name) \
uint32_t error_offset = \
ImportStartOffset(wire_bytes, import.module_name.offset()); \
return WasmError(error_offset, \
"Imported builtin function \"wasm:" module_name_string \
"\" \"" import_name "\" has incorrect signature")
#define CHECK_SIG(import_name, kSigName, kEnumName) \
if (name == base::StaticOneByteVector(#import_name)) { \
if (*sig != kSigName) { \
RETURN_ERROR("js-string", #import_name); \
} \
status = WellKnownImport::kEnumName; \
detected->add_imported_strings(); \
} else // NOLINT(readability/braces)
CHECK_SIG(cast, kSig_e_r, kStringCast)
CHECK_SIG(test, kSig_i_r, kStringTest)
CHECK_SIG(fromCharCode, kSig_e_i, kStringFromCharCode)
CHECK_SIG(fromCodePoint, kSig_e_i, kStringFromCodePoint)
CHECK_SIG(charCodeAt, kSig_i_ri, kStringCharCodeAt)
CHECK_SIG(codePointAt, kSig_i_ri, kStringCodePointAt)
CHECK_SIG(length, kSig_i_r, kStringLength)
CHECK_SIG(concat, kSig_e_rr, kStringConcat)
CHECK_SIG(substring, kSig_e_rii, kStringSubstring)
CHECK_SIG(equals, kSig_i_rr, kStringEquals)
CHECK_SIG(compare, kSig_i_rr, kStringCompare)
if (name == base::StaticOneByteVector("fromCharCodeArray")) {
if (sig->parameter_count() != 3 || sig->return_count() != 1 ||
!IsI16Array(sig->GetParam(0), module) || // --
sig->GetParam(1) != kI32 || // --
sig->GetParam(2) != kI32 || // --
sig->GetReturn() != kRefExtern) {
RETURN_ERROR("js-string", "fromCharCodeArray");
}
detected->add_imported_strings();
status = WellKnownImport::kStringFromWtf16Array;
} else if (name == base::StaticOneByteVector("intoCharCodeArray")) {
if (sig->parameter_count() != 3 || sig->return_count() != 1 ||
sig->GetParam(0) != kExternRef ||
!IsI16Array(sig->GetParam(1), module) || // --
sig->GetParam(2) != kI32 || // --
sig->GetReturn() != kI32) {
RETURN_ERROR("js-string", "intoCharCodeArray");
}
status = WellKnownImport::kStringToWtf16Array;
detected->add_imported_strings();
}
#undef CHECK_SIG
} else if (collection == base::StaticOneByteVector("text-encoder") &&
imports.contains(CompileTimeImport::kTextEncoder)) {
if (name == base::StaticOneByteVector("measureStringAsUTF8")) {
if (*sig != kSig_i_r) {
RETURN_ERROR("text-encoder", "measureStringAsUTF8");
}
status = WellKnownImport::kStringMeasureUtf8;
detected->add_imported_strings_utf8();
} else if (name ==
base::StaticOneByteVector("encodeStringIntoUTF8Array")) {
if (sig->parameter_count() != 3 || sig->return_count() != 1 ||
sig->GetParam(0) != kExternRef || // --
!IsI8Array(sig->GetParam(1), module, true) || // --
sig->GetParam(2) != kI32 || // --
sig->GetReturn() != kI32) {
RETURN_ERROR("text-encoder", "encodeStringIntoUTF8Array");
}
status = WellKnownImport::kStringIntoUtf8Array;
detected->add_imported_strings_utf8();
} else if (name == base::StaticOneByteVector("encodeStringToUTF8Array")) {
if (sig->parameter_count() != 1 || sig->return_count() != 1 ||
sig->GetParam(0) != kExternRef ||
!IsI8Array(sig->GetReturn(), module, false)) {
RETURN_ERROR("text-encoder", "encodeStringToUTF8Array");
}
status = WellKnownImport::kStringToUtf8Array;
detected->add_imported_strings_utf8();
}
} else if (collection == base::StaticOneByteVector("text-decoder") &&
imports.contains(CompileTimeImport::kTextDecoder)) {
if (name == base::StaticOneByteVector("decodeStringFromUTF8Array")) {
if (sig->parameter_count() != 3 || sig->return_count() != 1 ||
!IsI8Array(sig->GetParam(0), module, true) || // --
sig->GetParam(1) != kI32 || // --
sig->GetParam(2) != kI32 || // --
sig->GetReturn() != kRefExtern) {
RETURN_ERROR("text-decoder", "decodeStringFromUTF8Array");
}
status = WellKnownImport::kStringFromUtf8Array;
detected->add_imported_strings_utf8();
}
}
#undef RETURN_ERROR
statuses.push_back(status);
}
// We're operating on a fresh WasmModule instance here, so we don't need to
// check for incompatibilities with previously seen imports.
DCHECK_EQ(module->num_imported_functions, statuses.size());
// The "Initialize" call is currently only safe when the decoder has allocated
// storage, which it allocates when there is an imports section.
if (module->num_imported_functions != 0) {
module->type_feedback.well_known_imports.Initialize(
base::VectorOf(statuses));
}
return {};
}
namespace {
enum CompilationExecutionResult : int8_t { kNoMoreUnits, kYield };
const char* GetCompilationEventName(const WasmCompilationUnit& unit,
const CompilationEnv& env) {
ExecutionTier tier = unit.tier();
if (tier == ExecutionTier::kLiftoff) {
return "wasm.BaselineCompilation";
}
if (tier == ExecutionTier::kTurbofan) {
return "wasm.TopTierCompilation";
}
if (unit.func_index() <
static_cast<int>(env.module->num_imported_functions)) {
return "wasm.WasmToJSWrapperCompilation";
}
return "wasm.OtherCompilation";
}
constexpr uint8_t kMainTaskId = 0;
// Run by the {BackgroundCompileJob} (on any thread).
CompilationExecutionResult ExecuteCompilationUnits(
std::weak_ptr<NativeModule> native_module, Counters* counters,
JobDelegate* delegate, CompilationTier tier) {
TRACE_EVENT0("v8.wasm", "wasm.ExecuteCompilationUnits");
// Compilation must be disabled in jitless mode.
CHECK(!v8_flags.wasm_jitless);
// These fields are initialized in a {BackgroundCompileScope} before
// starting compilation.
std::optional<CompilationEnv> env;
std::optional<base::FlushDenormalsScope> disable_denormals;
std::shared_ptr<WireBytesStorage> wire_bytes;
std::shared_ptr<const WasmModule> module;
// Task 0 is any main thread (there might be multiple from multiple isolates),
// worker threads start at 1 (thus the "+ 1").
static_assert(kMainTaskId == 0);
int task_id = delegate ? (int{delegate->GetTaskId()} + 1) : kMainTaskId;
DCHECK_LE(0, task_id);
CompilationUnitQueues::Queue* queue;
std::optional<WasmCompilationUnit> unit;
WasmDetectedFeatures global_detected_features;
// Preparation (synchronized): Initialize the fields above and get the first
// compilation unit.
{
BackgroundCompileScope compile_scope(native_module);
if (compile_scope.cancelled()) return kYield;
env.emplace(CompilationEnv::ForModule(compile_scope.native_module()));
// We only really need this for optimized compilation, but for simplicity
// set it just once for everything.
disable_denormals.emplace(
compile_scope.native_module()->compile_imports().contains(
CompileTimeImport::kDisableDenormalFloats));
wire_bytes = compile_scope.compilation_state()->GetWireBytesStorage();
module = compile_scope.native_module()->shared_module();
queue = compile_scope.compilation_state()->GetQueueForCompileTask(task_id);
unit =
compile_scope.compilation_state()->GetNextCompilationUnit(queue, tier);
if (!unit) return kNoMoreUnits;
}
TRACE_COMPILE("ExecuteCompilationUnits (task id %d)\n", task_id);
std::vector<WasmCompilationResult> results_to_publish;
while (true) {
ExecutionTier current_tier = unit->tier();
const char* event_name = GetCompilationEventName(unit.value(), env.value());
TRACE_EVENT0("v8.wasm", event_name);
while (unit->tier() == current_tier) {
// Track detected features on a per-function basis before collecting them
// into {global_detected_features}.
WasmDetectedFeatures per_function_detected_features;
// (asynchronous): Execute the compilation.
WasmCompilationResult result =
unit->ExecuteCompilation(&env.value(), wire_bytes.get(), counters,
&per_function_detected_features);
global_detected_features.Add(per_function_detected_features);
bool compilation_succeeded = result.succeeded();
ExecutionTier result_tier = result.result_tier;
// We don't eagerly compile import wrappers any more.
DCHECK_GE(unit->func_index(), env->module->num_imported_functions);
results_to_publish.emplace_back(std::move(result));
bool yield = delegate && delegate->ShouldYield();
// (synchronized): Publish the compilation result and get the next unit.
BackgroundCompileScope compile_scope(native_module);
if (compile_scope.cancelled()) return kYield;
if (!compilation_succeeded) {
compile_scope.compilation_state()->SetError();
return kNoMoreUnits;
}
if (!unit->for_debugging() && result_tier != current_tier) {
compile_scope.native_module()->AddLiftoffBailout();
}
// Yield or get next unit.
if (yield ||
!(unit = compile_scope.compilation_state()->GetNextCompilationUnit(
queue, tier))) {
std::vector<UnpublishedWasmCode> unpublished_code =
compile_scope.native_module()->AddCompiledCode(
base::VectorOf(results_to_publish));
results_to_publish.clear();
compile_scope.compilation_state()->SchedulePublishCompilationResults(
std::move(unpublished_code), tier);
compile_scope.compilation_state()->OnCompilationStopped(
global_detected_features);
return yield ? kYield : kNoMoreUnits;
}
// Publish after finishing a certain amount of units, to avoid
// contention when all threads publish at the end.
bool batch_full =
queue->ShouldPublish(static_cast<int>(results_to_publish.size()));
// Also publish each time the compilation tier changes from Liftoff to
// TurboFan, such that we immediately publish the baseline compilation
// results to start execution, and do not wait for a batch to fill up.
bool liftoff_finished = unit->tier() != current_tier &&
unit->tier() == ExecutionTier::kTurbofan;
if (batch_full || liftoff_finished) {
std::vector<UnpublishedWasmCode> unpublished_code =
compile_scope.native_module()->AddCompiledCode(
base::VectorOf(results_to_publish));
results_to_publish.clear();
compile_scope.compilation_state()->SchedulePublishCompilationResults(
std::move(unpublished_code), tier);
}
}
}
UNREACHABLE();
}
std::unique_ptr<CompilationUnitBuilder> InitializeCompilation(
Isolate* isolate, NativeModule* native_module,
ProfileInformation* pgo_info) {
CompilationStateImpl* compilation_state =
Impl(native_module->compilation_state());
auto builder = std::make_unique<CompilationUnitBuilder>(native_module);
compilation_state->InitializeCompilationProgress(pgo_info);
return builder;
}
bool MayCompriseLazyFunctions(const WasmModule* module,
WasmEnabledFeatures enabled_features) {
if (IsLazyModule(module)) return true;
if (enabled_features.has_compilation_hints()) return true;
#ifdef ENABLE_SLOW_DCHECKS
int start = module->num_imported_functions;
int end = start + module->num_declared_functions;
for (int func_index = start; func_index < end; func_index++) {
SLOW_DCHECK(GetCompileStrategy(module, enabled_features, func_index,
false) != CompileStrategy::kLazy);
}
#endif
return false;
}
class CompilationTimeCallback : public CompilationEventCallback {
public:
enum CompileMode { kSynchronous, kAsync, kStreaming };
explicit CompilationTimeCallback(
std::shared_ptr<Counters> async_counters,
std::shared_ptr<metrics::Recorder> metrics_recorder,
v8::metrics::Recorder::ContextId context_id,
std::weak_ptr<NativeModule> native_module, CompileMode compile_mode)
: start_time_(base::TimeTicks::Now()),
async_counters_(std::move(async_counters)),
metrics_recorder_(std::move(metrics_recorder)),
context_id_(context_id),
native_module_(std::move(native_module)),
compile_mode_(compile_mode) {}
void call(CompilationEvent compilation_event) override {
DCHECK(base::TimeTicks::IsHighResolution());
std::shared_ptr<NativeModule> native_module = native_module_.lock();
if (!native_module) return;
auto now = base::TimeTicks::Now();
auto duration = now - start_time_;
if (compilation_event == CompilationEvent::kFinishedBaselineCompilation) {
// Reset {start_time_} to measure tier-up time.
start_time_ = now;
if (compile_mode_ != kSynchronous) {
TimedHistogram* histogram =
compile_mode_ == kAsync
? async_counters_->wasm_async_compile_wasm_module_time()
: async_counters_->wasm_streaming_compile_wasm_module_time();
histogram->AddSample(static_cast<int>(duration.InMicroseconds()));
}
v8::metrics::WasmModuleCompiled event{
(compile_mode_ != kSynchronous), // async
(compile_mode_ == kStreaming), // streamed
false, // cached
false, // deserialized
v8_flags.wasm_lazy_compilation, // lazy
true, // success
native_module->liftoff_code_size(), // code_size_in_bytes
native_module->liftoff_bailout_count(), // liftoff_bailout_count
duration.InMicroseconds()}; // wall_clock_duration_in_us
metrics_recorder_->DelayMainThreadEvent(event, context_id_);
}
if (compilation_event == CompilationEvent::kFailedCompilation) {
v8::metrics::WasmModuleCompiled event{
(compile_mode_ != kSynchronous), // async
(compile_mode_ == kStreaming), // streamed
false, // cached
false, // deserialized
v8_flags.wasm_lazy_compilation, // lazy
false, // success
native_module->liftoff_code_size(), // code_size_in_bytes
native_module->liftoff_bailout_count(), // liftoff_bailout_count
duration.InMicroseconds()}; // wall_clock_duration_in_us
metrics_recorder_->DelayMainThreadEvent(event, context_id_);
}
}
private:
base::TimeTicks start_time_;
const std::shared_ptr<Counters> async_counters_;
std::shared_ptr<metrics::Recorder> metrics_recorder_;
v8::metrics::Recorder::ContextId context_id_;
std::weak_ptr<NativeModule> native_module_;
const CompileMode compile_mode_;
};
WasmError ValidateFunctions(const WasmModule* module,
base::Vector<const uint8_t> wire_bytes,
WasmEnabledFeatures enabled_features,
OnlyLazyFunctions only_lazy_functions,
WasmDetectedFeatures* detected_features) {
DCHECK_EQ(module->origin, kWasmOrigin);
if (only_lazy_functions &&
!MayCompriseLazyFunctions(module, enabled_features)) {
return {};
}
std::function<bool(int)> filter; // Initially empty for "all functions".
if (only_lazy_functions) {
const bool is_lazy_module = IsLazyModule(module);
filter = [module, enabled_features, is_lazy_module](int func_index) {
CompileStrategy strategy = GetCompileStrategy(module, enabled_features,
func_index, is_lazy_module);
return strategy == CompileStrategy::kLazy ||
strategy == CompileStrategy::kLazyBaselineEagerTopTier;
};
}
// Call {ValidateFunctions} in the module decoder.
return ValidateFunctions(module, enabled_features, wire_bytes, filter,
detected_features);
}
WasmError ValidateFunctions(const NativeModule& native_module,
OnlyLazyFunctions only_lazy_functions) {
WasmDetectedFeatures detected_features;
WasmError result =
ValidateFunctions(native_module.module(), native_module.wire_bytes(),
native_module.enabled_features(), only_lazy_functions,
&detected_features);
if (!result.has_error()) {
// This function is called before the NativeModule is finished; all detected
// features will be published afterwards anyway, so ignore the return value
// here.
USE(native_module.compilation_state()->UpdateDetectedFeatures(
detected_features));
}
return result;
}
void CompileNativeModule(Isolate* isolate,
v8::metrics::Recorder::ContextId context_id,
ErrorThrower* thrower,
std::shared_ptr<NativeModule> native_module,
ProfileInformation* pgo_info) {
CHECK(!v8_flags.jitless || v8_flags.wasm_jitless);
const WasmModule* module = native_module->module();
// The callback captures a shared ptr to the semaphore.
auto* compilation_state = Impl(native_module->compilation_state());
if (base::TimeTicks::IsHighResolution()) {
compilation_state->AddCallback(std::make_unique<CompilationTimeCallback>(
isolate->async_counters(), isolate->metrics_recorder(), context_id,
native_module, CompilationTimeCallback::kSynchronous));
}
// Initialize the compilation units and kick off background compile tasks.
std::unique_ptr<CompilationUnitBuilder> builder =
InitializeCompilation(isolate, native_module.get(), pgo_info);
compilation_state->InitializeCompilationUnits(std::move(builder));
// Validate wasm modules for lazy compilation if requested. Never validate
// asm.js modules as these are valid by construction (additionally a CHECK
// will catch this during lazy compilation).
if (!v8_flags.wasm_lazy_validation && module->origin == kWasmOrigin) {
DCHECK(!thrower->error());
if (WasmError validation_error =
ValidateFunctions(*native_module, kOnlyLazyFunctions)) {
thrower->CompileFailed(std::move(validation_error));
return;
}
}
if (!compilation_state->failed()) {
compilation_state->WaitForCompilationEvent(
CompilationEvent::kFinishedBaselineCompilation);
}
if (compilation_state->failed()) {
DCHECK_IMPLIES(IsLazyModule(module), !v8_flags.wasm_lazy_validation);
WasmError validation_error =
ValidateFunctions(*native_module, kAllFunctions);
CHECK(validation_error.has_error());
thrower->CompileFailed(std::move(validation_error));
}
}
class BackgroundCompileJob final : public JobTask {
public:
explicit BackgroundCompileJob(std::weak_ptr<NativeModule> native_module,
std::shared_ptr<Counters> async_counters,
CompilationTier tier)
: native_module_(std::move(native_module)),
engine_barrier_(GetWasmEngine()->GetBarrierForBackgroundCompile()),
async_counters_(std::move(async_counters)),
tier_(tier) {}
void Run(JobDelegate* delegate) override {
auto engine_scope = engine_barrier_->TryLock();
if (!engine_scope) return;
ExecuteCompilationUnits(native_module_, async_counters_.get(), delegate,
tier_);
}
size_t GetMaxConcurrency(size_t worker_count) const override {
BackgroundCompileScope compile_scope(native_module_);
if (compile_scope.cancelled()) return 0;
size_t flag_limit = static_cast<size_t>(
std::max(1, v8_flags.wasm_num_compilation_tasks.value()));
// NumOutstandingCompilations() does not reflect the units that running
// workers are processing, thus add the current worker count to that number.
return std::min(flag_limit,
worker_count + compile_scope.compilation_state()
->NumOutstandingCompilations(tier_));
}
private:
std::weak_ptr<NativeModule> native_module_;
std::shared_ptr<OperationsBarrier> engine_barrier_;
const std::shared_ptr<Counters> async_counters_;
const CompilationTier tier_;
};
std::shared_ptr<NativeModule> GetOrCompileNewNativeModule(
Isolate* isolate, WasmEnabledFeatures enabled_features,
WasmDetectedFeatures detected_features, CompileTimeImports compile_imports,
ErrorThrower* thrower, std::shared_ptr<const WasmModule> module,
base::OwnedVector<const uint8_t> wire_bytes, int compilation_id,
v8::metrics::Recorder::ContextId context_id, ProfileInformation* pgo_info) {
std::shared_ptr<NativeModule> native_module =
GetWasmEngine()->MaybeGetNativeModule(
module->origin, wire_bytes.as_vector(), compile_imports, isolate);
if (native_module) return native_module;
// Otherwise compile a new NativeModule.
std::optional<TimedHistogramScope> wasm_compile_module_time_scope;
if (base::TimeTicks::IsHighResolution()) {
wasm_compile_module_time_scope.emplace(SELECT_WASM_COUNTER(
isolate->counters(), module->origin, wasm_compile, module_time));
}
const bool include_liftoff =
module->origin == kWasmOrigin && v8_flags.liftoff;
size_t code_size_estimate =
wasm::WasmCodeManager::EstimateNativeModuleCodeSize(
module.get(), include_liftoff,
DynamicTiering{v8_flags.wasm_dynamic_tiering.value()});
native_module = GetWasmEngine()->NewNativeModule(
isolate, enabled_features, detected_features, std::move(compile_imports),
module, code_size_estimate);
native_module->SetWireBytes(std::move(wire_bytes));
native_module->compilation_state()->set_compilation_id(compilation_id);
if (!v8_flags.wasm_jitless) {
// Compile / validate the new module.
CompileNativeModule(isolate, context_id, thrower, native_module, pgo_info);
}
if (thrower->error()) {
GetWasmEngine()->UpdateNativeModuleCache(true, std::move(native_module),
isolate);
return {};
}
// Finally, put the new module in the cache; this can return the passed
// NativeModule pointer, or another one (for a previously cached module).
return GetWasmEngine()->UpdateNativeModuleCache(false, native_module,
isolate);
}
} // namespace
std::shared_ptr<NativeModule> CompileToNativeModule(
Isolate* isolate, WasmEnabledFeatures enabled_features,
WasmDetectedFeatures detected_features, CompileTimeImports compile_imports,
ErrorThrower* thrower, std::shared_ptr<const WasmModule> module,
base::OwnedVector<const uint8_t> wire_bytes, int compilation_id,
v8::metrics::Recorder::ContextId context_id, ProfileInformation* pgo_info) {
std::shared_ptr<NativeModule> native_module = GetOrCompileNewNativeModule(
isolate, enabled_features, detected_features, std::move(compile_imports),
thrower, module, std::move(wire_bytes), compilation_id, context_id,
pgo_info);
if (!native_module) return {};
// Ensure that the code objects are logged before returning.
GetWasmEngine()->LogOutstandingCodesForIsolate(isolate);
// Now publish all detected features of this module in the current isolate.
PublishDetectedFeatures(
native_module->compilation_state()->detected_features(), isolate, true);
return native_module;
}
AsyncCompileJob::AsyncCompileJob(
Isolate* isolate, WasmEnabledFeatures enabled_features,
CompileTimeImports compile_imports, base::OwnedVector<const uint8_t> bytes,
DirectHandle<Context> context,
DirectHandle<NativeContext> incumbent_context, const char* api_method_name,
std::shared_ptr<CompilationResultResolver> resolver, int compilation_id)
: isolate_(isolate),
api_method_name_(api_method_name),
enabled_features_(enabled_features),
compile_imports_(std::move(compile_imports)),
dynamic_tiering_(DynamicTiering{v8_flags.wasm_dynamic_tiering.value()}),
start_time_(base::TimeTicks::Now()),
bytes_copy_(std::move(bytes)),
wire_bytes_(bytes_copy_.as_vector()),
resolver_(std::move(resolver)),
compilation_id_(compilation_id) {
TRACE_EVENT0(TRACE_DISABLED_BY_DEFAULT("v8.wasm.detailed"),
"wasm.AsyncCompileJob");
CHECK(v8_flags.wasm_async_compilation);
CHECK(!v8_flags.jitless || v8_flags.wasm_jitless);
v8::Isolate* v8_isolate = reinterpret_cast<v8::Isolate*>(isolate);
v8::Platform* platform = V8::GetCurrentPlatform();
foreground_task_runner_ = platform->GetForegroundTaskRunner(v8_isolate);
native_context_ =
isolate->global_handles()->Create(context->native_context());
incumbent_context_ = isolate->global_handles()->Create(*incumbent_context);
DCHECK(IsNativeContext(*native_context_));
context_id_ = isolate->GetOrRegisterRecorderContextId(native_context_);
metrics_event_.async = true;
}
void AsyncCompileJob::Start() {
DoAsync<DecodeModule>(isolate_->counters(),
isolate_->metrics_recorder()); // --
}
void AsyncCompileJob::Abort() {
// Removing this job will trigger the destructor, which will cancel all
// compilation.
GetWasmEngine()->RemoveCompileJob(this);
}
// {ValidateFunctionsStreamingJobData} holds information that is shared between
// the {AsyncStreamingProcessor} and the {ValidateFunctionsStreamingJob}. It
// lives in the {AsyncStreamingProcessor} and is updated from both classes.
struct ValidateFunctionsStreamingJobData {
struct Unit {
// {func_index == -1} represents an "invalid" unit.
int func_index = -1;
base::Vector<const uint8_t> code;
// Check whether the unit is valid.
operator bool() const { // NOLINT(google-explicit-constructor)
DCHECK_LE(-1, func_index);
return func_index >= 0;
}
};
void Initialize(int num_declared_functions) {
DCHECK_NULL(units);
units = base::OwnedVector<Unit>::NewForOverwrite(num_declared_functions);
// Initially {next == end}.
next_available_unit.store(units.begin(), std::memory_order_relaxed);
end_of_available_units.store(units.begin(), std::memory_order_relaxed);
}
void AddUnit(int declared_func_index, base::Vector<const uint8_t> code,
JobHandle* job_handle) {
DCHECK_NOT_NULL(units);
// Write new unit to {*end}, then increment {end}. There is only one thread
// adding new units, so no further synchronization needed.
Unit* ptr = end_of_available_units.load(std::memory_order_relaxed);
// Check invariant: {next <= end}.
DCHECK_LE(next_available_unit.load(std::memory_order_relaxed), ptr);
*ptr++ = {declared_func_index, code};
// Use release semantics, so whoever loads this pointer (using acquire
// semantics) sees all our previous stores.
end_of_available_units.store(ptr, std::memory_order_release);
size_t total_units_added = ptr - units.begin();
// Periodically notify concurrency increase. This has overhead, so avoid
// calling it too often. As long as threads are still running they will
// continue processing new units anyway, and if background threads validate
// faster than we can add units, then only notifying after increasingly long
// delays is the right thing to do to avoid too many small validation tasks.
// We notify on each power of two after 16 units, and every 16k units (just
// to have *some* upper limit and avoiding to pile up too many units).
// Additionally, notify after receiving the last unit of the module.
if ((total_units_added >= 16 &&
base::bits::IsPowerOfTwo(total_units_added)) ||
(total_units_added % (16 * 1024)) == 0 || ptr == units.end()) {
job_handle->NotifyConcurrencyIncrease();
}
}
size_t NumOutstandingUnits() const {
Unit* next = next_available_unit.load(std::memory_order_relaxed);
Unit* end = end_of_available_units.load(std::memory_order_relaxed);
DCHECK_LE(next, end);
return end - next;
}
// Retrieve one unit to validate; returns an "invalid" unit if nothing is in
// the queue.
Unit GetUnit() {
// Use an acquire load to synchronize with the store in {AddUnit}. All units
// before this {end} are fully initialized and ready to execute.
Unit* end = end_of_available_units.load(std::memory_order_acquire);
Unit* next = next_available_unit.load(std::memory_order_relaxed);
while (next < end) {
if (next_available_unit.compare_exchange_weak(
next, next + 1, std::memory_order_relaxed)) {
return *next;
}
// Otherwise retry with updated {next} pointer.
}
return {};
}
void UpdateDetectedFeatures(WasmDetectedFeatures new_detected_features) {
WasmDetectedFeatures old_features =
detected_features.load(std::memory_order_relaxed);
while (!detected_features.compare_exchange_weak(
old_features, old_features | new_detected_features,
std::memory_order_relaxed)) {
// Retry with updated {old_features}.
}
}
base::OwnedVector<Unit> units;
std::atomic<Unit*> next_available_unit;
std::atomic<Unit*> end_of_available_units;
std::atomic<bool> found_error{false};
std::atomic<WasmDetectedFeatures> detected_features;
};
class ValidateFunctionsStreamingJob final : public JobTask {
public:
ValidateFunctionsStreamingJob(const WasmModule* module,
WasmEnabledFeatures enabled_features,
ValidateFunctionsStreamingJobData* data)
: module_(module), enabled_features_(enabled_features), data_(data) {}
void Run(JobDelegate* delegate) override {
TRACE_EVENT0("v8.wasm", "wasm.ValidateFunctionsStreaming");
using Unit = ValidateFunctionsStreamingJobData::Unit;
Zone validation_zone{GetWasmEngine()->allocator(), ZONE_NAME};
WasmDetectedFeatures detected_features;
while (Unit unit = data_->GetUnit()) {
validation_zone.Reset();
DecodeResult result = ValidateSingleFunction(
&validation_zone, module_, unit.func_index, unit.code,
enabled_features_, &detected_features);
if (result.failed()) {
data_->found_error.store(true, std::memory_order_relaxed);
break;
}
// After validating one function, check if we should yield.
if (delegate->ShouldYield()) break;
}
data_->UpdateDetectedFeatures(detected_features);
}
size_t GetMaxConcurrency(size_t worker_count) const override {
return worker_count + data_->NumOutstandingUnits();
}
private:
const WasmModule* const module_;
const WasmEnabledFeatures enabled_features_;
ValidateFunctionsStreamingJobData* data_;
};
class AsyncStreamingProcessor final : public StreamingProcessor {
public:
explicit AsyncStreamingProcessor(AsyncCompileJob* job);
bool ProcessModuleHeader(base::Vector<const uint8_t> bytes) override;
bool ProcessSection(SectionCode section_code,
base::Vector<const uint8_t> bytes,
uint32_t offset) override;
bool ProcessCodeSectionHeader(int num_functions,
uint32_t functions_mismatch_error_offset,
std::shared_ptr<WireBytesStorage>,
int code_section_start,
int code_section_length) override;
bool ProcessFunctionBody(base::Vector<const uint8_t> bytes,
uint32_t offset) override;
void OnFinishedChunk() override;
void OnFinishedStream(base::OwnedVector<const uint8_t> bytes,
bool after_error) override;
void OnAbort() override;
bool Deserialize(base::Vector<const uint8_t> wire_bytes,
base::Vector<const uint8_t> module_bytes) override;
private:
void CommitCompilationUnits();
ModuleDecoder decoder_;
AsyncCompileJob* job_;
std::unique_ptr<CompilationUnitBuilder> compilation_unit_builder_;
int num_functions_ = 0;
bool prefix_cache_hit_ = false;
bool before_code_section_ = true;
ValidateFunctionsStreamingJobData validate_functions_job_data_;
std::unique_ptr<JobHandle> validate_functions_job_handle_;
// {prefix_hasher_} computes a running hash of the wire bytes up to code
// section size, but excludes the code section itself. Used by the
// {NativeModuleCache} to detect potential duplicate modules.
base::Hasher prefix_hasher_;
};
std::shared_ptr<StreamingDecoder> AsyncCompileJob::CreateStreamingDecoder() {
DCHECK_NULL(stream_);
stream_ = StreamingDecoder::CreateAsyncStreamingDecoder(
std::make_unique<AsyncStreamingProcessor>(this));
return stream_;
}
AsyncCompileJob::~AsyncCompileJob() {
// Note: This destructor always runs on the foreground thread of the isolate.
background_task_manager_.CancelAndWait();
// If initial compilation did not finish yet we can abort it.
if (native_module_) {
Impl(native_module_->compilation_state())
->CancelCompilation(CompilationStateImpl::kCancelInitialCompilation);
}
// Tell the streaming decoder that the AsyncCompileJob is not available
// anymore.
if (stream_) stream_->NotifyCompilationDiscarded();
CancelPendingForegroundTask();
isolate_->global_handles()->Destroy(native_context_.location());
isolate_->global_handles()->Destroy(incumbent_context_.location());
if (!module_object_.is_null()) {
isolate_->global_handles()->Destroy(module_object_.location());
}
}
void AsyncCompileJob::CreateNativeModule(
std::shared_ptr<const WasmModule> module, size_t code_size_estimate) {
// Embedder usage count for declared shared memories.
const bool has_shared_memory =
std::any_of(module->memories.begin(), module->memories.end(),
[](auto& memory) { return memory.is_shared; });
if (has_shared_memory) {
isolate_->CountUsage(v8::Isolate::UseCounterFeature::kWasmSharedMemory);
}
// Create the module object and populate with compiled functions and
// information needed at instantiation time.
native_module_ = GetWasmEngine()->NewNativeModule(
isolate_, enabled_features_, detected_features_,
std::move(compile_imports_), std::move(module), code_size_estimate);
native_module_->SetWireBytes(std::move(bytes_copy_));
native_module_->compilation_state()->set_compilation_id(compilation_id_);
}
bool AsyncCompileJob::GetOrCreateNativeModule(
std::shared_ptr<const WasmModule> module, size_t code_size_estimate) {
native_module_ = GetWasmEngine()->MaybeGetNativeModule(
module->origin, wire_bytes_.module_bytes(), compile_imports_, isolate_);
if (native_module_ == nullptr) {
CreateNativeModule(std::move(module), code_size_estimate);
return false;
}
return true;
}
void AsyncCompileJob::PrepareRuntimeObjects() {
// Create heap objects for script and module bytes to be stored in the
// module object. Asm.js is not compiled asynchronously.
DCHECK(module_object_.is_null());
auto source_url =
stream_ ? base::VectorOf(stream_->url()) : base::Vector<const char>();
auto script =
GetWasmEngine()->GetOrCreateScript(isolate_, native_module_, source_url);
DirectHandle<WasmModuleObject> module_object =
WasmModuleObject::New(isolate_, native_module_, script);
module_object_ = isolate_->global_handles()->Create(*module_object);
}
// This function assumes that it is executed in a HandleScope, and that a
// context is set on the isolate.
void AsyncCompileJob::FinishCompile(bool is_after_cache_hit) {
TRACE_EVENT0(TRACE_DISABLED_BY_DEFAULT("v8.wasm.detailed"),
"wasm.FinishAsyncCompile");
if (stream_) {
stream_->NotifyNativeModuleCreated(native_module_);
}
const WasmModule* module = native_module_->module();
auto compilation_state = Impl(native_module_->compilation_state());
// Update the compilation state with feature detected during module decoding
// and (potentially) validation. We will publish all features below, in the
// current isolate, so ignore the return value here.
USE(compilation_state->UpdateDetectedFeatures(detected_features_));
// If experimental PGO via files is enabled, load profile information now that
// we have all wire bytes and know that the module is valid.
if (V8_UNLIKELY(v8_flags.experimental_wasm_pgo_from_file)) {
std::unique_ptr<ProfileInformation> pgo_info =
LoadProfileFromFile(module, native_module_->wire_bytes());
if (pgo_info) {
compilation_state->ApplyPgoInfoLate(pgo_info.get());
}
}
bool is_after_deserialization = !module_object_.is_null();
if (!is_after_deserialization) {
PrepareRuntimeObjects();
}
// Measure duration of baseline compilation or deserialization from cache.
if (base::TimeTicks::IsHighResolution()) {
base::TimeDelta duration = base::TimeTicks::Now() - start_time_;
int duration_usecs = static_cast<int>(duration.InMicroseconds());
isolate_->counters()->wasm_streaming_finish_wasm_module_time()->AddSample(
duration_usecs);
if (is_after_cache_hit || is_after_deserialization) {
v8::metrics::WasmModuleCompiled event{
true, // async
true, // streamed
is_after_cache_hit, // cached
is_after_deserialization, // deserialized
v8_flags.wasm_lazy_compilation, // lazy
!compilation_state->failed(), // success
native_module_->turbofan_code_size(), // code_size_in_bytes
native_module_->liftoff_bailout_count(), // liftoff_bailout_count
duration.InMicroseconds()}; // wall_clock_duration_in_us
isolate_->metrics_recorder()->DelayMainThreadEvent(event, context_id_);
}
}
DCHECK(!isolate_->context().is_null());
// Finish the wasm script now and make it public to the debugger.
DirectHandle<Script> script(module_object_->script(), isolate_);
auto sourcemap_symbol =
module->debug_symbols[WasmDebugSymbols::Type::SourceMap];
if (script->type() == Script::Type::kWasm &&
sourcemap_symbol.type != WasmDebugSymbols::Type::None &&
!sourcemap_symbol.external_url.is_empty()) {
ModuleWireBytes wire_bytes(native_module_->wire_bytes());
MaybeDirectHandle<String> src_map_str =
isolate_->factory()->NewStringFromUtf8(
wire_bytes.GetNameOrNull(sourcemap_symbol.external_url),
AllocationType::kOld);
script->set_source_mapping_url(*src_map_str.ToHandleChecked());
}
{
TRACE_EVENT0(TRACE_DISABLED_BY_DEFAULT("v8.wasm.detailed"),
"wasm.Debug.OnAfterCompile");
isolate_->debug()->OnAfterCompile(script);
}
// Publish the detected features in this isolate, once initial compilation
// is done. Validate should have detected all features, unless lazy validation
// is enabled.
PublishDetectedFeatures(compilation_state->detected_features(), isolate_,
true);
// We might need debug code for the module, if the debugger was enabled while
// streaming compilation was running. Since handling this while compiling via
// streaming is tricky, we just remove all code which may have been generated,
// and compile debug code lazily.
if (native_module_->IsInDebugState()) {
WasmCodeRefScope ref_scope;
native_module_->RemoveCompiledCode(
NativeModule::RemoveFilter::kRemoveNonDebugCode);
}
// Finally, log all generated code (it does not matter if this happens
// repeatedly in case the script is shared).
native_module_->LogWasmCodes(isolate_, module_object_->script());
FinishSuccessfully();
}
void AsyncCompileJob::Failed() {
// {job} keeps the {this} pointer alive.
std::unique_ptr<AsyncCompileJob> job =
GetWasmEngine()->RemoveCompileJob(this);
// Revalidate the whole module to produce a deterministic error message.
constexpr bool kValidate = true;
WasmDetectedFeatures unused_detected_features;
ModuleResult result =
DecodeWasmModule(enabled_features_, wire_bytes_.module_bytes(), kValidate,
kWasmOrigin, &unused_detected_features);
ErrorThrower thrower(isolate_, api_method_name_);
if (result.failed()) {
thrower.CompileFailed(std::move(result).error());
} else {
// The only possible reason why {result} might be okay is if the failure
// was due to compile-time imports checking.
CHECK(!job->compile_imports_.empty());
WasmError error = ValidateAndSetBuiltinImports(
result.value().get(), wire_bytes_.module_bytes(), job->compile_imports_,
&unused_detected_features);
CHECK(error.has_error());
thrower.CompileError("%s", error.message().c_str());
}
resolver_->OnCompilationFailed(thrower.Reify());
}
class AsyncCompileJob::CompilationStateCallback
: public CompilationEventCallback {
public:
explicit CompilationStateCallback(AsyncCompileJob* job) : job_(job) {}
void call(CompilationEvent event) override {
// This callback is only being called from a foreground task.
switch (event) {
case CompilationEvent::kFinishedBaselineCompilation:
DCHECK(!last_event_.has_value());
if (job_->DecrementAndCheckFinisherCount()) {
// Install the native module in the cache, or reuse a conflicting one.
// If we get a conflicting module, wait until we are back in the
// main thread to update {job_->native_module_} to avoid a data race.
std::shared_ptr<NativeModule> cached_native_module =
GetWasmEngine()->UpdateNativeModuleCache(
false, job_->native_module_, job_->isolate_);
if (cached_native_module == job_->native_module_) {
// There was no cached module.
cached_native_module = nullptr;
}
job_->DoSync<FinishCompilation>(std::move(cached_native_module));
}
break;
case CompilationEvent::kFinishedCompilationChunk:
DCHECK(CompilationEvent::kFinishedBaselineCompilation == last_event_ ||
CompilationEvent::kFinishedCompilationChunk == last_event_);
break;
case CompilationEvent::kFailedCompilation:
DCHECK(!last_event_.has_value());
if (job_->DecrementAndCheckFinisherCount()) {
// Don't update {job_->native_module_} to avoid data races with other
// compilation threads. Use a copy of the shared pointer instead.
GetWasmEngine()->UpdateNativeModuleCache(true, job_->native_module_,
job_->isolate_);
job_->DoSync<Fail>();
}
break;
}
#ifdef DEBUG
last_event_ = event;
#endif
}
private:
AsyncCompileJob* job_;
#ifdef DEBUG
// This will be modified by different threads, but they externally
// synchronize, so no explicit synchronization (currently) needed here.
std::optional<CompilationEvent> last_event_;
#endif
};
// A closure to run a compilation step (either as foreground or background
// task) and schedule the next step(s), if any.
class AsyncCompileJob::CompileStep {
public:
virtual ~CompileStep() = default;
void Run(AsyncCompileJob* job, bool on_foreground) {
if (on_foreground) {
HandleScope scope(job->isolate_);
SaveAndSwitchContext saved_context(job->isolate_, *job->native_context_);
RunInForeground(job);
} else {
RunInBackground(job);
}
}
virtual void RunInForeground(AsyncCompileJob*) { UNREACHABLE(); }
virtual void RunInBackground(AsyncCompileJob*) { UNREACHABLE(); }
};
class AsyncCompileJob::CompileTask : public CancelableTask {
public:
CompileTask(AsyncCompileJob* job, bool on_foreground)
// We only manage the background tasks with the {CancelableTaskManager} of
// the {AsyncCompileJob}. Foreground tasks are managed by the system's
// {CancelableTaskManager}. Background tasks cannot spawn tasks managed by
// their own task manager.
: CancelableTask(on_foreground ? job->isolate_->cancelable_task_manager()
: &job->background_task_manager_),
job_(job),
on_foreground_(on_foreground) {}
~CompileTask() override {
if (job_ != nullptr && on_foreground_) ResetPendingForegroundTask();
}
void RunInternal() final {
if (!job_) return;
if (on_foreground_) ResetPendingForegroundTask();
job_->step_->Run(job_, on_foreground_);
// After execution, reset {job_} such that we don't try to reset the pending
// foreground task when the task is deleted.
job_ = nullptr;
}
void Cancel() {
DCHECK_NOT_NULL(job_);
job_ = nullptr;
}
private:
// {job_} will be cleared to cancel a pending task.
AsyncCompileJob* job_;
bool on_foreground_;
void ResetPendingForegroundTask() const {
DCHECK_EQ(this, job_->pending_foreground_task_);
job_->pending_foreground_task_ = nullptr;
}
};
void AsyncCompileJob::StartForegroundTask() {
DCHECK_NULL(pending_foreground_task_);
auto new_task = std::make_unique<CompileTask>(this, true);
pending_foreground_task_ = new_task.get();
foreground_task_runner_->PostTask(std::move(new_task));
}
void AsyncCompileJob::ExecuteForegroundTaskImmediately() {
DCHECK_NULL(pending_foreground_task_);
auto new_task = std::make_unique<CompileTask>(this, true);
pending_foreground_task_ = new_task.get();
new_task->Run();
}
void AsyncCompileJob::CancelPendingForegroundTask() {
if (!pending_foreground_task_) return;
pending_foreground_task_->Cancel();
pending_foreground_task_ = nullptr;
}
void AsyncCompileJob::StartBackgroundTask() {
auto task = std::make_unique<CompileTask>(this, false);
// If --wasm-num-compilation-tasks=0 is passed, do only spawn foreground
// tasks. This is used to make timing deterministic.
if (v8_flags.wasm_num_compilation_tasks > 0) {
V8::GetCurrentPlatform()->PostTaskOnWorkerThread(
TaskPriority::kUserBlocking, std::move(task));
} else {
foreground_task_runner_->PostTask(std::move(task));
}
}
template <typename Step,
AsyncCompileJob::UseExistingForegroundTask use_existing_fg_task,
typename... Args>
void AsyncCompileJob::DoSync(Args&&... args) {
NextStep<Step>(std::forward<Args>(args)...);
if (use_existing_fg_task && pending_foreground_task_ != nullptr) return;
StartForegroundTask();
}
template <typename Step, typename... Args>
void AsyncCompileJob::DoImmediately(Args&&... args) {
NextStep<Step>(std::forward<Args>(args)...);
ExecuteForegroundTaskImmediately();
}
template <typename Step, typename... Args>
void AsyncCompileJob::DoAsync(Args&&... args) {
NextStep<Step>(std::forward<Args>(args)...);
StartBackgroundTask();
}
template <typename Step, typename... Args>
void AsyncCompileJob::NextStep(Args&&... args) {
step_.reset(new Step(std::forward<Args>(args)...));
}
//==========================================================================
// Step 1: (async) Decode the module.
//==========================================================================
class AsyncCompileJob::DecodeModule : public AsyncCompileJob::CompileStep {
public:
explicit DecodeModule(Counters* counters,
std::shared_ptr<metrics::Recorder> metrics_recorder)
: counters_(counters), metrics_recorder_(std::move(metrics_recorder)) {}
void RunInBackground(AsyncCompileJob* job) override {
ModuleResult result;
{
DisallowHandleAllocation no_handle;
DisallowGarbageCollection no_gc;
// Decode the module bytes.
TRACE_COMPILE("(1) Decoding module...\n");
TRACE_EVENT0(TRACE_DISABLED_BY_DEFAULT("v8.wasm.detailed"),
"wasm.DecodeModule");
auto enabled_features = job->enabled_features_;
result = DecodeWasmModule(
enabled_features, job->wire_bytes_.module_bytes(), false, kWasmOrigin,
counters_, metrics_recorder_, job->context_id(),
DecodingMethod::kAsync, &job->detected_features_);
// Validate lazy functions here if requested.
if (result.ok() && !v8_flags.wasm_lazy_validation) {
const WasmModule* module = result.value().get();
if (WasmError validation_error = ValidateFunctions(
module, job->wire_bytes_.module_bytes(), job->enabled_features_,
kOnlyLazyFunctions, &job->detected_features_)) {
result = ModuleResult{std::move(validation_error)};
}
}
if (result.ok()) {
const WasmModule* module = result.value().get();
if (WasmError error = ValidateAndSetBuiltinImports(
module, job->wire_bytes_.module_bytes(), job->compile_imports_,
&job->detected_features_)) {
result = ModuleResult{std::move(error)};
}
}
}
if (result.failed()) {
// Decoding failure; reject the promise and clean up.
job->DoSync<Fail>();
} else {
// Decode passed.
std::shared_ptr<WasmModule> module = std::move(result).value();
const bool include_liftoff = v8_flags.liftoff;
size_t code_size_estimate =
wasm::WasmCodeManager::EstimateNativeModuleCodeSize(
module.get(), include_liftoff, job->dynamic_tiering_);
job->DoSync<PrepareAndStartCompile>(
std::move(module), true /* start_compilation */,
true /* lazy_functions_are_validated */, code_size_estimate);
}
}
private:
Counters* const counters_;
std::shared_ptr<metrics::Recorder> metrics_recorder_;
};
//==========================================================================
// Step 2 (sync): Create heap-allocated data and start compilation.
//==========================================================================
class AsyncCompileJob::PrepareAndStartCompile : public CompileStep {
public:
PrepareAndStartCompile(std::shared_ptr<const WasmModule> module,
bool start_compilation,
bool lazy_functions_are_validated,
size_t code_size_estimate)
: module_(std::move(module)),
start_compilation_(start_compilation),
lazy_functions_are_validated_(lazy_functions_are_validated),
code_size_estimate_(code_size_estimate) {}
private:
void RunInForeground(AsyncCompileJob* job) override {
TRACE_COMPILE("(2) Prepare and start compile...\n");
const bool streaming = job->wire_bytes_.length() == 0;
if (streaming) {
// Streaming compilation already checked for cache hits.
job->CreateNativeModule(module_, code_size_estimate_);
} else if (job->GetOrCreateNativeModule(std::move(module_),
code_size_estimate_)) {
job->FinishCompile(true);
return;
} else if (!lazy_functions_are_validated_) {
// If we are not streaming and did not get a cache hit, we might have hit
// the path where the streaming decoder got a prefix cache hit, but the
// module then turned out to be invalid, and we are running it through
// non-streaming decoding again. In this case, function bodies have not
// been validated yet (would have happened in the {DecodeModule} phase
// if we would not come via the non-streaming path). Thus do this now.
// Note that we only need to validate lazily compiled functions, others
// will be validated during eager compilation.
DCHECK(start_compilation_);
if (!v8_flags.wasm_lazy_validation &&
ValidateFunctions(*job->native_module_, kOnlyLazyFunctions)
.has_error()) {
job->Failed();
return;
}
}
// Make sure all compilation tasks stopped running. Decoding (async step)
// is done.
job->background_task_manager_.CancelAndWait();
CompilationStateImpl* compilation_state =
Impl(job->native_module_->compilation_state());
compilation_state->AddCallback(
std::make_unique<CompilationStateCallback>(job));
if (base::TimeTicks::IsHighResolution()) {
auto compile_mode = job->stream_ == nullptr
? CompilationTimeCallback::kAsync
: CompilationTimeCallback::kStreaming;
compilation_state->AddCallback(std::make_unique<CompilationTimeCallback>(
job->isolate_->async_counters(), job->isolate_->metrics_recorder(),
job->context_id_, job->native_module_, compile_mode));
}
if (start_compilation_) {
// TODO(13209): Use PGO for async compilation, if available.
constexpr ProfileInformation* kNoProfileInformation = nullptr;
std::unique_ptr<CompilationUnitBuilder> builder = InitializeCompilation(
job->isolate(), job->native_module_.get(), kNoProfileInformation);
compilation_state->InitializeCompilationUnits(std::move(builder));
// In single-threaded mode there are no worker tasks that will do the
// compilation. We call {WaitForCompilationEvent} here so that the main
// thread participates and finishes the compilation.
if (v8_flags.wasm_num_compilation_tasks == 0 || v8_flags.wasm_jitless) {
compilation_state->WaitForCompilationEvent(
CompilationEvent::kFinishedBaselineCompilation);
}
}
}
const std::shared_ptr<const WasmModule> module_;
const bool start_compilation_;
const bool lazy_functions_are_validated_;
const size_t code_size_estimate_;
};
//==========================================================================
// Step 3 (sync): Compilation finished.
//==========================================================================
class AsyncCompileJob::FinishCompilation : public CompileStep {
public:
explicit FinishCompilation(std::shared_ptr<NativeModule> cached_native_module)
: cached_native_module_(std::move(cached_native_module)) {}
private:
void RunInForeground(AsyncCompileJob* job) override {
TRACE_COMPILE("(3) Compilation finished\n");
if (cached_native_module_) {
job->native_module_ = cached_native_module_;
}
// Then finalize and publish the generated module.
job->FinishCompile(cached_native_module_ != nullptr);
}
std::shared_ptr<NativeModule> cached_native_module_;
};
//==========================================================================
// Step 4 (sync): Decoding or compilation failed.
//==========================================================================
class AsyncCompileJob::Fail : public CompileStep {
private:
void RunInForeground(AsyncCompileJob* job) override {
TRACE_COMPILE("(4) Async compilation failed.\n");
// {job_} is deleted in {Failed}, therefore the {return}.
return job->Failed();
}
};
void AsyncCompileJob::FinishSuccessfully() {
TRACE_COMPILE("(4) Finish module...\n");
{
TRACE_EVENT0(TRACE_DISABLED_BY_DEFAULT("v8.wasm.detailed"),
"wasm.OnCompilationSucceeded");
// We have to make sure that an "incumbent context" is available in case
// the module's start function calls out to Blink.
Local<v8::Context> backup_incumbent_context =
Utils::ToLocal(incumbent_context_);
v8::Context::BackupIncumbentScope incumbent(backup_incumbent_context);
resolver_->OnCompilationSucceeded(module_object_);
}
GetWasmEngine()->RemoveCompileJob(this);
}
AsyncStreamingProcessor::AsyncStreamingProcessor(AsyncCompileJob* job)
: decoder_(job->enabled_features_, &job->detected_features_),
job_(job),
compilation_unit_builder_(nullptr) {}
// Process the module header.
bool AsyncStreamingProcessor::ProcessModuleHeader(
base::Vector<const uint8_t> bytes) {
TRACE_STREAMING("Process module header...\n");
decoder_.DecodeModuleHeader(bytes);
if (!decoder_.ok()) return false;
// Note: We do not include the magic bytes in the hash; they are constant
// anyways.
return true;
}
// Process all sections except for the code section.
bool AsyncStreamingProcessor::ProcessSection(SectionCode section_code,
base::Vector<const uint8_t> bytes,
uint32_t offset) {
TRACE_STREAMING("Process section %d ...\n", section_code);
if (compilation_unit_builder_) {
// We reached a section after the code section, we do not need the
// compilation_unit_builder_ anymore.
CommitCompilationUnits();
compilation_unit_builder_.reset();
}
if (before_code_section_) {
// Combine section hashes until code section.
prefix_hasher_.AddRange(bytes);
}
if (section_code == SectionCode::kUnknownSectionCode) {
size_t bytes_consumed = ModuleDecoder::IdentifyUnknownSection(
&decoder_, bytes, offset, &section_code);
if (!decoder_.ok()) return false;
if (section_code == SectionCode::kUnknownSectionCode) {
// Skip unknown sections that we do not know how to handle.
return true;
}
// Remove the unknown section tag from the payload bytes.
offset += bytes_consumed;
bytes = bytes.SubVector(bytes_consumed, bytes.size());
}
decoder_.DecodeSection(section_code, bytes, offset);
return decoder_.ok();
}
// Start the code section.
bool AsyncStreamingProcessor::ProcessCodeSectionHeader(
int num_functions, uint32_t functions_mismatch_error_offset,
std::shared_ptr<WireBytesStorage> wire_bytes_storage,
int code_section_start, int code_section_length) {
DCHECK_LE(0, code_section_length);
before_code_section_ = false;
TRACE_STREAMING("Start the code section with %d functions...\n",
num_functions);
prefix_hasher_.Add(static_cast<uint32_t>(code_section_length));
if (!decoder_.CheckFunctionsCount(static_cast<uint32_t>(num_functions),
functions_mismatch_error_offset)) {
return false;
}
decoder_.StartCodeSection({static_cast<uint32_t>(code_section_start),
static_cast<uint32_t>(code_section_length)});
if (!GetWasmEngine()->GetStreamingCompilationOwnership(
prefix_hasher_.hash(), job_->compile_imports_)) {
// Known prefix, wait until the end of the stream and check the cache.
prefix_cache_hit_ = true;
return true;
}
// Execute the PrepareAndStartCompile step immediately and not in a separate
// task.
int num_imported_functions =
static_cast<int>(decoder_.module()->num_imported_functions);
DCHECK_EQ(kWasmOrigin, decoder_.module()->origin);
const bool include_liftoff = v8_flags.liftoff;
size_t code_size_estimate =
wasm::WasmCodeManager::EstimateNativeModuleCodeSize(
num_functions, num_imported_functions, code_section_length,
include_liftoff, job_->dynamic_tiering_);
job_->DoImmediately<AsyncCompileJob::PrepareAndStartCompile>(
decoder_.shared_module(),
// start_compilation: false; triggered when we receive the bodies.
false,
// lazy_functions_are_validated: false (bodies not received yet).
false, code_size_estimate);
auto* compilation_state = Impl(job_->native_module_->compilation_state());
compilation_state->SetWireBytesStorage(std::move(wire_bytes_storage));
DCHECK_EQ(job_->native_module_->module()->origin, kWasmOrigin);
// Set outstanding_finishers_ to 2, because both the AsyncCompileJob and the
// AsyncStreamingProcessor have to finish.
job_->outstanding_finishers_.store(2);
// TODO(13209): Use PGO for streaming compilation, if available.
constexpr ProfileInformation* kNoProfileInformation = nullptr;
compilation_unit_builder_ = InitializeCompilation(
job_->isolate(), job_->native_module_.get(), kNoProfileInformation);
return true;
}
// Process a function body.
bool AsyncStreamingProcessor::ProcessFunctionBody(
base::Vector<const uint8_t> bytes, uint32_t offset) {
TRACE_STREAMING("Process function body %d ...\n", num_functions_);
uint32_t func_index =
decoder_.module()->num_imported_functions + num_functions_;
++num_functions_;
// In case of {prefix_cache_hit} we still need the function body to be
// decoded. Otherwise a later cache miss cannot be handled.
decoder_.DecodeFunctionBody(func_index, static_cast<uint32_t>(bytes.length()),
offset);
if (prefix_cache_hit_) {
// Don't compile yet if we might have a cache hit.
return true;
}
const WasmModule* module = decoder_.module();
auto enabled_features = job_->enabled_features_;
DCHECK_EQ(module->origin, kWasmOrigin);
const bool lazy_module = v8_flags.wasm_lazy_compilation;
CompileStrategy strategy =
GetCompileStrategy(module, enabled_features, func_index, lazy_module);
CHECK_IMPLIES(v8_flags.wasm_jitless, !v8_flags.wasm_lazy_validation);
bool validate_lazily_compiled_function =
v8_flags.wasm_jitless ||
(!v8_flags.wasm_lazy_validation &&
(strategy == CompileStrategy::kLazy ||
strategy == CompileStrategy::kLazyBaselineEagerTopTier));
if (validate_lazily_compiled_function) {
// {bytes} is part of a section buffer owned by the streaming decoder. The
// streaming decoder is held alive by the {AsyncCompileJob}, so we can just
// use the {bytes} vector as long as the {AsyncCompileJob} is still running.
if (!validate_functions_job_handle_) {
validate_functions_job_data_.Initialize(module->num_declared_functions);
validate_functions_job_handle_ = V8::GetCurrentPlatform()->CreateJob(
TaskPriority::kUserVisible,
std::make_unique<ValidateFunctionsStreamingJob>(
module, enabled_features, &validate_functions_job_data_));
}
validate_functions_job_data_.AddUnit(func_index, bytes,
validate_functions_job_handle_.get());
}
auto* compilation_state = Impl(job_->native_module_->compilation_state());
compilation_state->AddCompilationUnit(compilation_unit_builder_.get(),
func_index);
return true;
}
void AsyncStreamingProcessor::CommitCompilationUnits() {
DCHECK(compilation_unit_builder_);
compilation_unit_builder_->Commit();
}
void AsyncStreamingProcessor::OnFinishedChunk() {
TRACE_STREAMING("FinishChunk...\n");
if (compilation_unit_builder_) CommitCompilationUnits();
}
// Finish the processing of the stream.
void AsyncStreamingProcessor::OnFinishedStream(
base::OwnedVector<const uint8_t> bytes, bool after_error) {
TRACE_STREAMING("Finish stream...\n");
ModuleResult module_result = decoder_.FinishDecoding();
if (module_result.failed()) after_error = true;
if (validate_functions_job_handle_) {
// Wait for background validation to finish, then check if a validation
// error was found.
// TODO(13447): Do not block here; register validation as another finisher
// instead.
validate_functions_job_handle_->Join();
validate_functions_job_handle_.reset();
if (validate_functions_job_data_.found_error) after_error = true;
job_->detected_features_ |=
validate_functions_job_data_.detected_features.load(
std::memory_order_relaxed);
}
job_->wire_bytes_ = ModuleWireBytes(bytes.as_vector());
job_->bytes_copy_ = std::move(bytes);
if (!after_error) {
WasmDetectedFeatures detected_imports_features;
if (WasmError error = ValidateAndSetBuiltinImports(
module_result.value().get(), job_->wire_bytes_.module_bytes(),
job_->compile_imports_, &detected_imports_features)) {
after_error = true;
} else {
job_->detected_features_ |= detected_imports_features;
}
}
// Record event metrics.
auto duration = base::TimeTicks::Now() - job_->start_time_;
job_->metrics_event_.success = !after_error;
job_->metrics_event_.streamed = true;
job_->metrics_event_.module_size_in_bytes = job_->wire_bytes_.length();
job_->metrics_event_.function_count = num_functions_;
job_->metrics_event_.wall_clock_duration_in_us = duration.InMicroseconds();
job_->isolate_->metrics_recorder()->DelayMainThreadEvent(job_->metrics_event_,
job_->context_id_);
if (after_error) {
if (job_->native_module_ && job_->native_module_->wire_bytes().empty()) {
// Clean up the temporary cache entry.
GetWasmEngine()->StreamingCompilationFailed(prefix_hasher_.hash(),
job_->compile_imports_);
}
// Calling {Failed} will invalidate the {AsyncCompileJob} and delete {this}.
job_->Failed();
return;
}
std::shared_ptr<WasmModule> module = std::move(module_result).value();
// At this point we identified the module as valid (except maybe for function
// bodies, if lazy validation is enabled).
// This DCHECK could be considered slow, but it only happens once per async
// module compilation, and we only re-decode the module structure, without
// validating function bodies. Overall this does not add a lot of overhead.
#ifdef DEBUG
WasmDetectedFeatures detected_module_features;
DCHECK(DecodeWasmModule(job_->enabled_features_,
job_->bytes_copy_.as_vector(),
/* validate functions */ false, kWasmOrigin,
&detected_module_features)
.ok());
// Module decoding should not detect any new features.
DCHECK(job_->detected_features_.contains_all(detected_module_features));
#endif
DCHECK_EQ(NativeModuleCache::PrefixHash(job_->wire_bytes_.module_bytes()),
prefix_hasher_.hash());
if (prefix_cache_hit_) {
// Restart as an asynchronous, non-streaming compilation. Most likely
// {PrepareAndStartCompile} will get the native module from the cache.
const bool include_liftoff = v8_flags.liftoff;
size_t code_size_estimate =
wasm::WasmCodeManager::EstimateNativeModuleCodeSize(
module.get(), include_liftoff, job_->dynamic_tiering_);
job_->DoSync<AsyncCompileJob::PrepareAndStartCompile>(
std::move(module), true /* start_compilation */,
false /* lazy_functions_are_validated_ */, code_size_estimate);
return;
}
// We have to open a HandleScope and prepare the Context for
// CreateNativeModule, PrepareRuntimeObjects and FinishCompile as this is a
// callback from the embedder.
HandleScope scope(job_->isolate_);
SaveAndSwitchContext saved_context(job_->isolate_, *job_->native_context_);
// Record the size of the wire bytes and the number of functions. In
// synchronous and asynchronous (non-streaming) compilation, this happens in
// {DecodeWasmModule}.
auto* module_size_histogram =
job_->isolate_->counters()->wasm_wasm_module_size_bytes();
module_size_histogram->AddSample(job_->wire_bytes_.module_bytes().length());
auto* num_functions_histogram =
job_->isolate_->counters()->wasm_functions_per_wasm_module();
num_functions_histogram->AddSample(static_cast<int>(num_functions_));
const bool has_code_section = job_->native_module_ != nullptr;
bool cache_hit = false;
if (!has_code_section) {
// We are processing a WebAssembly module without code section. Create the
// native module now (would otherwise happen in {PrepareAndStartCompile} or
// {ProcessCodeSectionHeader}).
constexpr size_t kCodeSizeEstimate = 0;
cache_hit =
job_->GetOrCreateNativeModule(std::move(module), kCodeSizeEstimate);
} else {
job_->native_module_->SetWireBytes(std::move(job_->bytes_copy_));
}
const bool needs_finish = job_->DecrementAndCheckFinisherCount();
DCHECK_IMPLIES(!has_code_section, needs_finish);
if (needs_finish) {
const bool failed = job_->native_module_->compilation_state()->failed();
if (!cache_hit) {
auto* prev_native_module = job_->native_module_.get();
job_->native_module_ = GetWasmEngine()->UpdateNativeModuleCache(
failed, std::move(job_->native_module_), job_->isolate_);
cache_hit = prev_native_module != job_->native_module_.get();
}
// We finally call {Failed} or {FinishCompile}, which will invalidate the
// {AsyncCompileJob} and delete {this}.
if (failed) {
job_->Failed();
} else {
job_->FinishCompile(cache_hit);
}
}
}
void AsyncStreamingProcessor::OnAbort() {
TRACE_STREAMING("Abort stream...\n");
if (validate_functions_job_handle_) {
validate_functions_job_handle_->Cancel();
validate_functions_job_handle_.reset();
}
if (job_->native_module_ && job_->native_module_->wire_bytes().empty()) {
// Clean up the temporary cache entry.
GetWasmEngine()->StreamingCompilationFailed(prefix_hasher_.hash(),
job_->compile_imports_);
}
// {Abort} invalidates the {AsyncCompileJob}, which in turn deletes {this}.
job_->Abort();
}
bool AsyncStreamingProcessor::Deserialize(
base::Vector<const uint8_t> module_bytes,
base::Vector<const uint8_t> wire_bytes) {
TRACE_EVENT0("v8.wasm", "wasm.Deserialize");
std::optional<TimedHistogramScope> time_scope;
if (base::TimeTicks::IsHighResolution()) {
time_scope.emplace(job_->isolate()->counters()->wasm_deserialization_time(),
job_->isolate());
}
// DeserializeNativeModule and FinishCompile assume that they are executed in
// a HandleScope, and that a context is set on the isolate.
HandleScope scope(job_->isolate_);
SaveAndSwitchContext saved_context(job_->isolate_, *job_->native_context_);
MaybeDirectHandle<WasmModuleObject> result = DeserializeNativeModule(
job_->isolate_, module_bytes, wire_bytes, job_->compile_imports_,
base::VectorOf(job_->stream_->url()));
if (result.is_null()) return false;
job_->module_object_ =
job_->isolate_->global_handles()->Create(*result.ToHandleChecked());
job_->native_module_ = job_->module_object_->shared_native_module();
job_->wire_bytes_ = ModuleWireBytes(job_->native_module_->wire_bytes());
// Calling {FinishCompile} deletes the {AsyncCompileJob} and {this}.
job_->FinishCompile(false);
return true;
}
CompilationStateImpl::CompilationStateImpl(
const std::shared_ptr<NativeModule>& native_module,
std::shared_ptr<Counters> async_counters, DynamicTiering dynamic_tiering,
WasmDetectedFeatures detected_features)
: native_module_(native_module.get()),
native_module_weak_(std::move(native_module)),
async_counters_(std::move(async_counters)),
compilation_unit_queues_(native_module->num_imported_functions(),
native_module->num_declared_functions()),
dynamic_tiering_(dynamic_tiering),
detected_features_(detected_features) {}
void CompilationStateImpl::InitCompileJob() {
DCHECK_NULL(baseline_compile_job_);
DCHECK_NULL(top_tier_compile_job_);
// Create the job, but don't spawn workers yet. This will happen on
// {NotifyConcurrencyIncrease}.
baseline_compile_job_ = V8::GetCurrentPlatform()->CreateJob(
TaskPriority::kUserVisible,
std::make_unique<BackgroundCompileJob>(
native_module_weak_, async_counters_, CompilationTier::kBaseline));
top_tier_compile_job_ = V8::GetCurrentPlatform()->CreateJob(
TaskPriority::kUserVisible,
std::make_unique<BackgroundCompileJob>(
native_module_weak_, async_counters_, CompilationTier::kTopTier));
}
void CompilationStateImpl::CancelCompilation(
CompilationStateImpl::CancellationPolicy cancellation_policy) {
base::MutexGuard callbacks_guard(&callbacks_mutex_);
if (cancellation_policy == kCancelInitialCompilation &&
finished_events_.contains(
CompilationEvent::kFinishedBaselineCompilation)) {
// Initial compilation already finished; cannot be cancelled.
return;
}
// std::memory_order_relaxed is sufficient because no other state is
// synchronized with |compile_cancelled_|.
compile_cancelled_.store(true, std::memory_order_relaxed);
// No more callbacks after abort.
callbacks_.clear();
}
bool CompilationStateImpl::cancelled() const {
return compile_cancelled_.load(std::memory_order_relaxed);
}
void CompilationStateImpl::ApplyCompilationHintToInitialProgress(
const WasmCompilationHint& hint, size_t hint_idx) {
// Get old information.
uint8_t& progress = compilation_progress_[hint_idx];
ExecutionTier old_baseline_tier = RequiredBaselineTierField::decode(progress);
ExecutionTier old_top_tier = RequiredTopTierField::decode(progress);
// Compute new information.
ExecutionTier new_baseline_tier =
ApplyHintToExecutionTier(hint.baseline_tier, old_baseline_tier);
ExecutionTier new_top_tier =
ApplyHintToExecutionTier(hint.top_tier, old_top_tier);
switch (hint.strategy) {
case WasmCompilationHintStrategy::kDefault:
// Be careful not to switch from lazy to non-lazy.
if (old_baseline_tier == ExecutionTier::kNone) {
new_baseline_tier = ExecutionTier::kNone;
}
if (old_top_tier == ExecutionTier::kNone) {
new_top_tier = ExecutionTier::kNone;
}
break;
case WasmCompilationHintStrategy::kLazy:
new_baseline_tier = ExecutionTier::kNone;
new_top_tier = ExecutionTier::kNone;
break;
case WasmCompilationHintStrategy::kEager:
// Nothing to do, use the encoded (new) tiers.
break;
case WasmCompilationHintStrategy::kLazyBaselineEagerTopTier:
new_baseline_tier = ExecutionTier::kNone;
break;
}
progress = RequiredBaselineTierField::update(progress, new_baseline_tier);
progress = RequiredTopTierField::update(progress, new_top_tier);
// Update counter for outstanding baseline units.
outstanding_baseline_units_ += (new_baseline_tier != ExecutionTier::kNone) -
(old_baseline_tier != ExecutionTier::kNone);
}
void CompilationStateImpl::ApplyPgoInfoToInitialProgress(
ProfileInformation* pgo_info) {
// Functions that were executed in the profiling run are eagerly compiled to
// Liftoff.
const WasmModule* module = native_module_->module();
for (int func_index : pgo_info->executed_functions()) {
uint8_t& progress =
compilation_progress_[declared_function_index(module, func_index)];
ExecutionTier old_baseline_tier =
RequiredBaselineTierField::decode(progress);
// If the function is already marked for eager compilation, we are good.
if (old_baseline_tier != ExecutionTier::kNone) continue;
// Set the baseline tier to Liftoff, so we eagerly compile to Liftoff.
// TODO(13288): Compile Liftoff code in the background, if lazy compilation
// is enabled.
progress =
RequiredBaselineTierField::update(progress, ExecutionTier::kLiftoff);
++outstanding_baseline_units_;
}
// Functions that were tiered up during PGO generation are eagerly compiled to
// TurboFan (in the background, not blocking instantiation).
for (int func_index : pgo_info->tiered_up_functions()) {
uint8_t& progress =
compilation_progress_[declared_function_index(module, func_index)];
ExecutionTier old_baseline_tier =
RequiredBaselineTierField::decode(progress);
ExecutionTier old_top_tier = RequiredTopTierField::decode(progress);
// If the function is already marked for eager or background compilation to
// TurboFan, we are good.
if (old_baseline_tier == ExecutionTier::kTurbofan) continue;
if (old_top_tier == ExecutionTier::kTurbofan) continue;
// Set top tier to TurboFan, so we eagerly trigger compilation in the
// background.
progress = RequiredTopTierField::update(progress, ExecutionTier::kTurbofan);
}
}
void CompilationStateImpl::ApplyPgoInfoLate(ProfileInformation* pgo_info) {
TRACE_EVENT0("v8.wasm", "wasm.ApplyPgoInfo");
const WasmModule* module = native_module_->module();
CompilationUnitBuilder builder{native_module_};
base::MutexGuard guard(&callbacks_mutex_);
// Functions that were executed in the profiling run are eagerly compiled to
// Liftoff (in the background).
for (int func_index : pgo_info->executed_functions()) {
uint8_t& progress =
compilation_progress_[declared_function_index(module, func_index)];
ExecutionTier old_baseline_tier =
RequiredBaselineTierField::decode(progress);
// If the function is already marked for eager compilation, we are good.
if (old_baseline_tier != ExecutionTier::kNone) continue;
// If we already compiled Liftoff or TurboFan code, we are also good.
ExecutionTier reached_tier = ReachedTierField::decode(progress);
if (reached_tier >= ExecutionTier::kLiftoff) continue;
// Set the baseline tier to Liftoff and schedule a compilation unit.
progress =
RequiredBaselineTierField::update(progress, ExecutionTier::kLiftoff);
// Add this as a "top tier unit" since it does not contribute to initial
// compilation ("baseline finished" might already be triggered).
// TODO(clemensb): Rename "baseline finished" to "initial compile finished".
// TODO(clemensb): Avoid scheduling both a Liftoff and a TurboFan unit, or
// prioritize Liftoff when executing the units.
builder.AddTopTierUnit(func_index, ExecutionTier::kLiftoff);
}
// Functions that were tiered up during PGO generation are eagerly compiled to
// TurboFan in the background.
for (int func_index : pgo_info->tiered_up_functions()) {
uint8_t& progress =
compilation_progress_[declared_function_index(module, func_index)];
ExecutionTier old_baseline_tier =
RequiredBaselineTierField::decode(progress);
ExecutionTier old_top_tier = RequiredTopTierField::decode(progress);
// If the function is already marked for eager or background compilation to
// TurboFan, we are good.
if (old_baseline_tier == ExecutionTier::kTurbofan) continue;
if (old_top_tier == ExecutionTier::kTurbofan) continue;
// If we already compiled TurboFan code, we are also good.
ExecutionTier reached_tier = ReachedTierField::decode(progress);
if (reached_tier == ExecutionTier::kTurbofan) continue;
// Set top tier to TurboFan and schedule a compilation unit.
progress = RequiredTopTierField::update(progress, ExecutionTier::kTurbofan);
builder.AddTopTierUnit(func_index, ExecutionTier::kTurbofan);
}
builder.Commit();
}
void CompilationStateImpl::InitializeCompilationProgress(
ProfileInformation* pgo_info) {
DCHECK(!failed());
base::MutexGuard guard(&callbacks_mutex_);
if (!v8_flags.wasm_jitless) {
auto* module = native_module_->module();
DCHECK_EQ(0, outstanding_baseline_units_);
// Compute the default compilation progress for all functions, and set it.
const ExecutionTierPair default_tiers = GetDefaultTiersPerModule(
native_module_, dynamic_tiering_, native_module_->IsInDebugState(),
IsLazyModule(module));
const uint8_t default_progress =
RequiredBaselineTierField::encode(default_tiers.baseline_tier) |
RequiredTopTierField::encode(default_tiers.top_tier) |
ReachedTierField::encode(ExecutionTier::kNone);
compilation_progress_.assign(module->num_declared_functions,
default_progress);
if (default_tiers.baseline_tier != ExecutionTier::kNone) {
outstanding_baseline_units_ += module->num_declared_functions;
}
// Apply compilation hints, if enabled.
if (native_module_->enabled_features().has_compilation_hints()) {
size_t num_hints = std::min(module->compilation_hints.size(),
size_t{module->num_declared_functions});
for (size_t hint_idx = 0; hint_idx < num_hints; ++hint_idx) {
const auto& hint = module->compilation_hints[hint_idx];
ApplyCompilationHintToInitialProgress(hint, hint_idx);
}
}
// Transform --wasm-eager-tier-up-function, if given, into a fake
// compilation hint.
if (V8_UNLIKELY(
v8_flags.wasm_eager_tier_up_function >= 0 &&
static_cast<uint32_t>(v8_flags.wasm_eager_tier_up_function) >=
module->num_imported_functions &&
static_cast<uint32_t>(v8_flags.wasm_eager_tier_up_function) <
module->functions.size())) {
uint32_t func_idx =
v8_flags.wasm_eager_tier_up_function - module->num_imported_functions;
WasmCompilationHint hint{WasmCompilationHintStrategy::kEager,
WasmCompilationHintTier::kOptimized,
WasmCompilationHintTier::kOptimized};
ApplyCompilationHintToInitialProgress(hint, func_idx);
}
}
// Apply PGO information, if available.
if (pgo_info) ApplyPgoInfoToInitialProgress(pgo_info);
// Trigger callbacks if module needs no baseline or top tier compilation. This
// can be the case for an empty or fully lazy module.
TriggerOutstandingCallbacks();
}
void CompilationStateImpl::AddCompilationUnitInternal(
CompilationUnitBuilder* builder, int function_index,
uint8_t function_progress) {
ExecutionTier required_baseline_tier =
CompilationStateImpl::RequiredBaselineTierField::decode(
function_progress);
ExecutionTier required_top_tier =
CompilationStateImpl::RequiredTopTierField::decode(function_progress);
ExecutionTier reached_tier =
CompilationStateImpl::ReachedTierField::decode(function_progress);
if (reached_tier < required_baseline_tier) {
builder->AddBaselineUnit(function_index, required_baseline_tier);
}
if (reached_tier < required_top_tier &&
required_baseline_tier != required_top_tier) {
builder->AddTopTierUnit(function_index, required_top_tier);
}
}
void CompilationStateImpl::InitializeCompilationUnits(
std::unique_ptr<CompilationUnitBuilder> builder) {
if (!v8_flags.wasm_jitless) {
int offset = native_module_->module()->num_imported_functions;
{
base::MutexGuard guard(&callbacks_mutex_);
for (size_t i = 0, e = compilation_progress_.size(); i < e; ++i) {
uint8_t function_progress = compilation_progress_[i];
int func_index = offset + static_cast<int>(i);
AddCompilationUnitInternal(builder.get(), func_index,
function_progress);
}
}
}
builder->Commit();
}
void CompilationStateImpl::AddCompilationUnit(CompilationUnitBuilder* builder,
int func_index) {
int offset = native_module_->module()->num_imported_functions;
int progress_index = func_index - offset;
uint8_t function_progress = 0;
if (!v8_flags.wasm_jitless) {
// TODO(ahaas): This lock may cause overhead. If so, we could get rid of the
// lock as follows:
// 1) Make compilation_progress_ an array of atomic<uint8_t>, and access it
// lock-free.
// 2) Have a copy of compilation_progress_ that we use for initialization.
// 3) Just re-calculate the content of compilation_progress_.
base::MutexGuard guard(&callbacks_mutex_);
function_progress = compilation_progress_[progress_index];
}
AddCompilationUnitInternal(builder, func_index, function_progress);
}
void CompilationStateImpl::InitializeCompilationProgressAfterDeserialization(
base::Vector<const int> lazy_functions,
base::Vector<const int> eager_functions) {
TRACE_EVENT2("v8.wasm", "wasm.CompilationAfterDeserialization",
"num_lazy_functions", lazy_functions.size(),
"num_eager_functions", eager_functions.size());
std::optional<TimedHistogramScope> lazy_compile_time_scope;
if (base::TimeTicks::IsHighResolution()) {
lazy_compile_time_scope.emplace(
counters()->wasm_compile_after_deserialize());
}
auto* module = native_module_->module();
{
base::MutexGuard guard(&callbacks_mutex_);
DCHECK(compilation_progress_.empty());
// Initialize the compilation progress as if everything was
// TurboFan-compiled.
constexpr uint8_t kProgressAfterTurbofanDeserialization =
RequiredBaselineTierField::encode(ExecutionTier::kLiftoff) |
RequiredTopTierField::encode(ExecutionTier::kTurbofan) |
ReachedTierField::encode(ExecutionTier::kTurbofan);
compilation_progress_.assign(module->num_declared_functions,
kProgressAfterTurbofanDeserialization);
// Update compilation state for lazy functions.
constexpr uint8_t kProgressForLazyFunctions =
RequiredBaselineTierField::encode(ExecutionTier::kNone) |
RequiredTopTierField::encode(ExecutionTier::kNone) |
ReachedTierField::encode(ExecutionTier::kNone);
for (auto func_index : lazy_functions) {
compilation_progress_[declared_function_index(module, func_index)] =
kProgressForLazyFunctions;
}
// Update compilation state for eagerly compiled functions.
constexpr bool kNotLazy = false;
ExecutionTierPair default_tiers =
GetDefaultTiersPerModule(native_module_, dynamic_tiering_,
native_module_->IsInDebugState(), kNotLazy);
uint8_t progress_for_eager_functions =
RequiredBaselineTierField::encode(default_tiers.baseline_tier) |
RequiredTopTierField::encode(default_tiers.top_tier) |
ReachedTierField::encode(ExecutionTier::kNone);
for (auto func_index : eager_functions) {
// Check that {func_index} is not contained in {lazy_functions}.
DCHECK_EQ(
compilation_progress_[declared_function_index(module, func_index)],
kProgressAfterTurbofanDeserialization);
compilation_progress_[declared_function_index(module, func_index)] =
progress_for_eager_functions;
}
DCHECK_NE(ExecutionTier::kNone, default_tiers.baseline_tier);
outstanding_baseline_units_ += eager_functions.size();
// Baseline compilation is done if we do not have any Liftoff functions to
// compile.
if (eager_functions.empty() || v8_flags.wasm_lazy_compilation) {
finished_events_.Add(CompilationEvent::kFinishedBaselineCompilation);
}
}
auto builder = std::make_unique<CompilationUnitBuilder>(native_module_);
InitializeCompilationUnits(std::move(builder));
if (!v8_flags.wasm_lazy_compilation) {
WaitForCompilationEvent(CompilationEvent::kFinishedBaselineCompilation);
}
}
void CompilationStateImpl::AddCallback(
std::unique_ptr<CompilationEventCallback> callback) {
base::MutexGuard callbacks_guard(&callbacks_mutex_);
// Immediately trigger events that already happened.
for (auto event : {CompilationEvent::kFinishedBaselineCompilation,
CompilationEvent::kFailedCompilation}) {
if (finished_events_.contains(event)) {
callback->call(event);
}
}
constexpr base::EnumSet<CompilationEvent> kFinalEvents{
CompilationEvent::kFailedCompilation};
if (!finished_events_.contains_any(kFinalEvents)) {
callbacks_.emplace_back(std::move(callback));
}
}
void CompilationStateImpl::CommitCompilationUnits(
base::Vector<WasmCompilationUnit> baseline_units,
base::Vector<WasmCompilationUnit> top_tier_units) {
base::MutexGuard guard{&mutex_};
if (!baseline_units.empty() || !top_tier_units.empty()) {
compilation_unit_queues_.AddUnits(baseline_units, top_tier_units,
native_module_->module());
}
if (!baseline_units.empty()) {
DCHECK(baseline_compile_job_->IsValid());
baseline_compile_job_->NotifyConcurrencyIncrease();
}
if (!top_tier_units.empty()) {
DCHECK(top_tier_compile_job_->IsValid());
top_tier_compile_job_->NotifyConcurrencyIncrease();
}
}
void CompilationStateImpl::CommitTopTierCompilationUnit(
WasmCompilationUnit unit) {
CommitCompilationUnits({}, {&unit, 1});
}
void CompilationStateImpl::AddTopTierPriorityCompilationUnit(
WasmCompilationUnit unit, size_t priority) {
compilation_unit_queues_.AddTopTierPriorityUnit(unit, priority);
// We should not have a {CodeSpaceWriteScope} open at this point, as
// {NotifyConcurrencyIncrease} can spawn new threads which could inherit PKU
// permissions (which would be a security issue).
top_tier_compile_job_->NotifyConcurrencyIncrease();
}
CompilationUnitQueues::Queue* CompilationStateImpl::GetQueueForCompileTask(
int task_id) {
return compilation_unit_queues_.GetQueueForTask(task_id);
}
std::optional<WasmCompilationUnit> CompilationStateImpl::GetNextCompilationUnit(
CompilationUnitQueues::Queue* queue, CompilationTier tier) {
return compilation_unit_queues_.GetNextUnit(queue, tier);
}
void CompilationStateImpl::OnFinishedUnits(
base::Vector<WasmCode*> code_vector) {
TRACE_EVENT1(TRACE_DISABLED_BY_DEFAULT("v8.wasm.detailed"),
"wasm.OnFinishedUnits", "units", code_vector.size());
base::MutexGuard guard(&callbacks_mutex_);
// Assume an order of execution tiers that represents the quality of their
// generated code.
static_assert(ExecutionTier::kNone < ExecutionTier::kLiftoff &&
ExecutionTier::kLiftoff < ExecutionTier::kTurbofan,
"Assume an order on execution tiers");
if (!v8_flags.wasm_jitless) {
DCHECK_EQ(compilation_progress_.size(),
native_module_->module()->num_declared_functions);
}
bool has_top_tier_code = false;
for (size_t i = 0; i < code_vector.size(); i++) {
WasmCode* code = code_vector[i];
DCHECK_NOT_NULL(code);
DCHECK_LT(code->index(), native_module_->num_functions());
has_top_tier_code |= code->tier() == ExecutionTier::kTurbofan;
if (code->index() <
static_cast<int>(native_module_->num_imported_functions())) {
// Import wrapper.
DCHECK_EQ(code->tier(), ExecutionTier::kTurbofan);
outstanding_baseline_units_--;
} else {
// Function.
DCHECK_NE(code->tier(), ExecutionTier::kNone);
// Read function's compilation progress.
// This view on the compilation progress may differ from the actually
// compiled code. Any lazily compiled function does not contribute to the
// compilation progress but may publish code to the code manager.
int slot_index =
declared_function_index(native_module_->module(), code->index());
uint8_t function_progress = compilation_progress_[slot_index];
ExecutionTier required_baseline_tier =
RequiredBaselineTierField::decode(function_progress);
ExecutionTier reached_tier = ReachedTierField::decode(function_progress);
// Check whether required baseline or top tier are reached.
if (reached_tier < required_baseline_tier &&
required_baseline_tier <= code->tier()) {
DCHECK_GT(outstanding_baseline_units_, 0);
outstanding_baseline_units_--;
}
if (code->tier() == ExecutionTier::kTurbofan) {
bytes_since_last_chunk_ += code->instructions().size();
}
// Update function's compilation progress.
if (code->tier() > reached_tier) {
compilation_progress_[slot_index] = ReachedTierField::update(
compilation_progress_[slot_index], code->tier());
}
// Allow another top tier compilation if deopts are enabled and the
// currently installed code object is a liftoff object.
// Ideally, this would be done only if the code->tier() ==
// ExecutionTier::Liftoff as the code object for which we run this
// function should be the same as the one installed on the native_module.
// This is unfortunately not the case as installing a code object on the
// native module and updating the compilation_progress_ and the
// CompilationUnitQueues::top_tier_compiled_ are not synchronized.
// Note: GetCode() acquires the NativeModule::allocation_mutex_, so this
// could cause deadlocks if any other place acquires
// NativeModule::allocation_mutex_ first and then
// CompilationStateImpl::callbacks_mutex_!
const bool is_liftoff = code->tier() == ExecutionTier::kLiftoff;
auto published_code_is_liftoff = [this](int index) {
WasmCode* code = native_module_->GetCode(index);
if (code == nullptr) return false;
return code->is_liftoff();
};
if (v8_flags.wasm_deopt &&
(is_liftoff || published_code_is_liftoff(code->index()))) {
// Setting the reached tier below the baseline tier would create an
// inconsistent state and has actually led to crashes before (see
// https://crbug.com/379086474).
DCHECK_LE(required_baseline_tier, ExecutionTier::kLiftoff);
compilation_progress_[slot_index] = ReachedTierField::update(
compilation_progress_[slot_index], ExecutionTier::kLiftoff);
compilation_unit_queues_.AllowAnotherTopTierJob(code->index());
}
DCHECK_LE(0, outstanding_baseline_units_);
}
}
// Update the {last_top_tier_compilation_timestamp_} if it is set (i.e. a
// delayed task has already been spawned).
if (has_top_tier_code && !last_top_tier_compilation_timestamp_.IsNull()) {
last_top_tier_compilation_timestamp_ = base::TimeTicks::Now();
}
TriggerOutstandingCallbacks();
}
namespace {
class TriggerCodeCachingAfterTimeoutTask : public v8::Task {
public:
explicit TriggerCodeCachingAfterTimeoutTask(
std::weak_ptr<NativeModule> native_module)
: native_module_(std::move(native_module)) {}
void Run() override {
if (std::shared_ptr<NativeModule> native_module = native_module_.lock()) {
Impl(native_module->compilation_state())->TriggerCachingAfterTimeout();
}
}
private:
const std::weak_ptr<NativeModule> native_module_;
};
} // namespace
void CompilationStateImpl::TriggerOutstandingCallbacks() {
callbacks_mutex_.AssertHeld();
base::EnumSet<CompilationEvent> triggered_events;
if (outstanding_baseline_units_ == 0) {
triggered_events.Add(CompilationEvent::kFinishedBaselineCompilation);
}
// For dynamic tiering, trigger "compilation chunk finished" after a new chunk
// of size {v8_flags.wasm_caching_threshold}.
if (dynamic_tiering_ &&
static_cast<size_t>(v8_flags.wasm_caching_threshold) <=
bytes_since_last_chunk_) {
// Trigger caching immediately if
// - there is no timeout,
// - the hard threshold was reached, or
// - we are running single-threaded.
if (v8_flags.single_threaded || v8_flags.wasm_caching_timeout_ms <= 0 ||
static_cast<size_t>(v8_flags.wasm_caching_hard_threshold) <=
bytes_since_last_chunk_) {
triggered_events.Add(CompilationEvent::kFinishedCompilationChunk);
bytes_since_last_chunk_ = 0;
} else if (last_top_tier_compilation_timestamp_.IsNull()) {
// Trigger a task after the given timeout; that task will only trigger
// caching if no new code was added until then. Otherwise, it will
// re-schedule itself.
V8::GetCurrentPlatform()->PostDelayedTaskOnWorkerThread(
TaskPriority::kUserVisible,
std::make_unique<TriggerCodeCachingAfterTimeoutTask>(
native_module_weak_),
1e-3 * v8_flags.wasm_caching_timeout_ms);
// Set the timestamp (will be updated by {OnFinishedUnits} if more
// top-tier compilation finished before the delayed task is being run).
last_top_tier_compilation_timestamp_ = base::TimeTicks::Now();
}
}
if (compile_failed_.load(std::memory_order_relaxed)) {
// *Only* trigger the "failed" event.
triggered_events =
base::EnumSet<CompilationEvent>({CompilationEvent::kFailedCompilation});
}
TriggerCallbacks(triggered_events);
}
void CompilationStateImpl::TriggerCallbacks(
base::EnumSet<CompilationEvent> events) {
if (events.empty()) return;
// Don't trigger past events again.
events -= finished_events_;
// There can be multiple compilation chunks, thus do not store this.
finished_events_ |= events - CompilationEvent::kFinishedCompilationChunk;
for (auto event :
{std::make_pair(CompilationEvent::kFailedCompilation,
"wasm.CompilationFailed"),
std::make_pair(CompilationEvent::kFinishedBaselineCompilation,
"wasm.BaselineFinished"),
std::make_pair(CompilationEvent::kFinishedCompilationChunk,
"wasm.CompilationChunkFinished")}) {
if (!events.contains(event.first)) continue;
DCHECK_NE(compilation_id_, kInvalidCompilationID);
TRACE_EVENT1("v8.wasm", event.second, "id", compilation_id_);
for (auto& callback : callbacks_) {
callback->call(event.first);
}
}
if (outstanding_baseline_units_ == 0) {
auto new_end = std::remove_if(
callbacks_.begin(), callbacks_.end(), [](const auto& callback) {
return callback->release_after_final_event();
});
callbacks_.erase(new_end, callbacks_.end());
}
}
void CompilationStateImpl::TriggerCachingAfterTimeout() {
base::MutexGuard guard{&callbacks_mutex_};
// It can happen that we reached the hard threshold while waiting for the
// timeout to expire. In that case, {bytes_since_last_chunk_} might be zero
// and there is nothing new to cache.
if (bytes_since_last_chunk_ == 0) return;
DCHECK(!last_top_tier_compilation_timestamp_.IsNull());
base::TimeTicks caching_time =
last_top_tier_compilation_timestamp_ +
base::TimeDelta::FromMilliseconds(v8_flags.wasm_caching_timeout_ms);
base::TimeDelta time_until_caching = caching_time - base::TimeTicks::Now();
// If we are still half a millisecond or more away from the timeout,
// reschedule the task. Otherwise, call the caching callback.
if (time_until_caching >= base::TimeDelta::FromMicroseconds(500)) {
int ms_remaining =
static_cast<int>(time_until_caching.InMillisecondsRoundedUp());
DCHECK_LE(1, ms_remaining);
V8::GetCurrentPlatform()->PostDelayedTaskOnWorkerThread(
TaskPriority::kUserVisible,
std::make_unique<TriggerCodeCachingAfterTimeoutTask>(
native_module_weak_),
ms_remaining);
return;
}
TriggerCallbacks({CompilationEvent::kFinishedCompilationChunk});
last_top_tier_compilation_timestamp_ = {};
bytes_since_last_chunk_ = 0;
}
void CompilationStateImpl::OnCompilationStopped(
WasmDetectedFeatures detected_features) {
WasmDetectedFeatures new_detected_features =
UpdateDetectedFeatures(detected_features);
if (new_detected_features.empty()) return;
// New detected features can only happen during eager compilation or if lazy
// validation is enabled.
// The exceptions are currently stringref and imported strings, which are only
// detected on top-tier compilation.
DCHECK(!v8_flags.wasm_lazy_compilation || v8_flags.wasm_lazy_validation ||
(new_detected_features -
WasmDetectedFeatures{{WasmDetectedFeature::stringref,
WasmDetectedFeature::imported_strings_utf8,
WasmDetectedFeature::imported_strings}})
.empty());
// TODO(clemensb): Fix reporting of late detected features (relevant for lazy
// validation and for stringref).
}
WasmDetectedFeatures CompilationStateImpl::UpdateDetectedFeatures(
WasmDetectedFeatures detected_features) {
WasmDetectedFeatures old_features =
detected_features_.load(std::memory_order_relaxed);
while (!detected_features_.compare_exchange_weak(
old_features, old_features | detected_features,
std::memory_order_relaxed)) {
// Retry with updated {old_features}.
}
return detected_features - old_features;
}
void CompilationStateImpl::PublishCompilationResults(
std::vector<UnpublishedWasmCode> unpublished_code) {
if (unpublished_code.empty()) return;
#if DEBUG
// We don't compile import wrappers eagerly.
for (const auto& [code, assumptions] : unpublished_code) {
int func_index = code->index();
DCHECK_LE(native_module_->num_imported_functions(), func_index);
DCHECK_LT(func_index, native_module_->num_functions());
}
#endif
PublishCode(base::VectorOf(unpublished_code));
}
std::vector<WasmCode*> CompilationStateImpl::PublishCode(
base::Vector<UnpublishedWasmCode> code) {
WasmCodeRefScope code_ref_scope;
std::vector<WasmCode*> published_code =
native_module_->PublishCode(std::move(code));
// Defer logging code in case wire bytes were not fully received yet.
if (native_module_->log_code() && native_module_->HasWireBytes()) {
GetWasmEngine()->LogCode(base::VectorOf(published_code));
}
OnFinishedUnits(base::VectorOf(published_code));
return published_code;
}
void CompilationStateImpl::SchedulePublishCompilationResults(
std::vector<UnpublishedWasmCode> unpublished_code, CompilationTier tier) {
PublishState& state = publish_state_[tier];
{
base::MutexGuard guard(&state.mutex_);
if (state.publisher_running_) {
// Add new code to the queue and return.
state.publish_queue_.reserve(state.publish_queue_.size() +
unpublished_code.size());
for (auto& c : unpublished_code) {
state.publish_queue_.emplace_back(std::move(c));
}
return;
}
state.publisher_running_ = true;
}
while (true) {
PublishCompilationResults(std::move(unpublished_code));
unpublished_code.clear();
// Keep publishing new code that came in.
base::MutexGuard guard(&state.mutex_);
DCHECK(state.publisher_running_);
if (state.publish_queue_.empty()) {
state.publisher_running_ = false;
return;
}
unpublished_code.swap(state.publish_queue_);
}
}
size_t CompilationStateImpl::NumOutstandingCompilations(
CompilationTier tier) const {
return compilation_unit_queues_.GetSizeForTier(tier);
}
void CompilationStateImpl::SetError() {
compile_cancelled_.store(true, std::memory_order_relaxed);
if (compile_failed_.exchange(true, std::memory_order_relaxed)) {
return; // Already failed before.
}
base::MutexGuard callbacks_guard(&callbacks_mutex_);
TriggerOutstandingCallbacks();
callbacks_.clear();
}
void CompilationStateImpl::WaitForCompilationEvent(
CompilationEvent expect_event) {
switch (expect_event) {
case CompilationEvent::kFinishedBaselineCompilation:
if (baseline_compile_job_->IsValid()) baseline_compile_job_->Join();
break;
default:
// Waiting on other CompilationEvent doesn't make sense.
UNREACHABLE();
}
#ifdef DEBUG
base::EnumSet<CompilationEvent> events{expect_event,
CompilationEvent::kFailedCompilation};
base::MutexGuard guard(&callbacks_mutex_);
DCHECK(finished_events_.contains_any(events));
#endif
}
void CompilationStateImpl::TierUpAllFunctions() {
const WasmModule* module = native_module_->module();
uint32_t num_wasm_functions = module->num_declared_functions;
WasmCodeRefScope code_ref_scope;
CompilationUnitBuilder builder(native_module_);
for (uint32_t i = 0; i < num_wasm_functions; ++i) {
int func_index = module->num_imported_functions + i;
WasmCode* code = native_module_->GetCode(func_index);
if (!code || !code->is_turbofan()) {
builder.AddTopTierUnit(func_index, ExecutionTier::kTurbofan);
}
}
builder.Commit();
// Join the compilation, until no compilation units are left anymore.
class DummyDelegate final : public JobDelegate {
bool ShouldYield() override { return false; }
bool IsJoiningThread() const override { return true; }
void NotifyConcurrencyIncrease() override { UNIMPLEMENTED(); }
uint8_t GetTaskId() override { return kMainTaskId; }
};
DummyDelegate delegate;
ExecuteCompilationUnits(native_module_weak_, async_counters_.get(), &delegate,
CompilationTier::kTopTier);
// We cannot wait for other compilation threads to finish, so we explicitly
// compile all functions which are not yet available as TurboFan code.
for (uint32_t i = 0; i < num_wasm_functions; ++i) {
uint32_t func_index = module->num_imported_functions + i;
WasmCode* code = native_module_->GetCode(func_index);
if (!code || !code->is_turbofan()) {
wasm::GetWasmEngine()->CompileFunction(async_counters_.get(),
native_module_, func_index,
wasm::ExecutionTier::kTurbofan);
}
}
}
WasmCode* CompileImportWrapperForTest(Isolate* isolate,
NativeModule* native_module,
ImportCallKind kind,
const CanonicalSig* sig,
CanonicalTypeIndex type_index,
int expected_arity, Suspend suspend) {
bool source_positions = is_asmjs_module(native_module->module());
if (v8_flags.wasm_jitless) {
WasmImportWrapperCache::ModificationScope cache_scope(
GetWasmImportWrapperCache());
WasmImportWrapperCache::CacheKey key(kind, type_index, expected_arity,
suspend);
DCHECK_NULL(cache_scope[key]);
return nullptr;
}
return GetWasmImportWrapperCache()->CompileWasmImportCallWrapper(
isolate, kind, sig, type_index, source_positions, expected_arity,
suspend);
}
} // namespace v8::internal::wasm
#undef TRACE_COMPILE
#undef TRACE_STREAMING
#undef TRACE_LAZY