[heap] Concurrently free empty typed slot set chunks.
BUG=chromium:648568
Review-Url: https://codereview.chromium.org/2352423002
Cr-Commit-Position: refs/heads/master@{#39605}
diff --git a/src/heap/mark-compact.cc b/src/heap/mark-compact.cc
index 08472e5..aae6593 100644
--- a/src/heap/mark-compact.cc
+++ b/src/heap/mark-compact.cc
@@ -3865,6 +3865,12 @@
} else {
max_freed = RawSweep(page, REBUILD_FREE_LIST, free_space_mode);
}
+
+ // After finishing sweeping of a page we clean up its remembered set.
+ if (page->typed_old_to_new_slots()) {
+ page->typed_old_to_new_slots()->FreeToBeFreedChunks();
+ }
+
{
base::LockGuard<base::Mutex> guard(&mutex_);
swept_list_[identity].Add(page);
diff --git a/src/heap/remembered-set.cc b/src/heap/remembered-set.cc
index 6575d55..467f725 100644
--- a/src/heap/remembered-set.cc
+++ b/src/heap/remembered-set.cc
@@ -36,7 +36,8 @@
} else {
return REMOVE_SLOT;
}
- });
+ },
+ TypedSlotSet::PREFREE_EMPTY_CHUNKS);
}
}
for (MemoryChunk* chunk : *heap->map_space()) {
diff --git a/src/heap/remembered-set.h b/src/heap/remembered-set.h
index bdfae03..75aa524 100644
--- a/src/heap/remembered-set.h
+++ b/src/heap/remembered-set.h
@@ -149,10 +149,13 @@
static void RemoveRangeTyped(MemoryChunk* page, Address start, Address end) {
TypedSlotSet* slots = GetTypedSlotSet(page);
if (slots != nullptr) {
- slots->Iterate([start, end](SlotType slot_type, Address host_addr,
- Address slot_addr) {
- return start <= slot_addr && slot_addr < end ? REMOVE_SLOT : KEEP_SLOT;
- });
+ slots->Iterate(
+ [start, end](SlotType slot_type, Address host_addr,
+ Address slot_addr) {
+ return start <= slot_addr && slot_addr < end ? REMOVE_SLOT
+ : KEEP_SLOT;
+ },
+ TypedSlotSet::PREFREE_EMPTY_CHUNKS);
}
}
@@ -173,7 +176,7 @@
static void IterateTyped(MemoryChunk* chunk, Callback callback) {
TypedSlotSet* slots = GetTypedSlotSet(chunk);
if (slots != nullptr) {
- int new_count = slots->Iterate(callback);
+ int new_count = slots->Iterate(callback, TypedSlotSet::KEEP_EMPTY_CHUNKS);
if (new_count == 0) {
ReleaseTypedSlotSet(chunk);
}
diff --git a/src/heap/slot-set.h b/src/heap/slot-set.h
index 6ba3174..3690099 100644
--- a/src/heap/slot-set.h
+++ b/src/heap/slot-set.h
@@ -5,6 +5,8 @@
#ifndef V8_SLOT_SET_H
#define V8_SLOT_SET_H
+#include <stack>
+
#include "src/allocation.h"
#include "src/base/atomic-utils.h"
#include "src/base/bits.h"
@@ -259,6 +261,8 @@
// typed slots contain V8 internal pointers that are not directly exposed to JS.
class TypedSlotSet {
public:
+ enum IterationMode { PREFREE_EMPTY_CHUNKS, KEEP_EMPTY_CHUNKS };
+
typedef std::pair<SlotType, uint32_t> TypeAndOffset;
struct TypedSlot {
@@ -326,6 +330,10 @@
void Insert(SlotType type, uint32_t host_offset, uint32_t offset) {
TypedSlot slot(type, host_offset, offset);
Chunk* top_chunk = chunk_.Value();
+ if (!top_chunk) {
+ top_chunk = new Chunk(nullptr, kInitialBufferSize);
+ chunk_.SetValue(top_chunk);
+ }
if (!top_chunk->AddSlot(slot)) {
Chunk* new_top_chunk =
new Chunk(top_chunk, NextCapacity(top_chunk->capacity.Value()));
@@ -346,13 +354,15 @@
// else return REMOVE_SLOT;
// });
template <typename Callback>
- int Iterate(Callback callback) {
+ int Iterate(Callback callback, IterationMode mode) {
STATIC_ASSERT(CLEARED_SLOT < 8);
Chunk* chunk = chunk_.Value();
+ Chunk* previous = nullptr;
int new_count = 0;
while (chunk != nullptr) {
TypedSlot* buffer = chunk->buffer.Value();
int count = chunk->count.Value();
+ bool empty = true;
for (int i = 0; i < count; i++) {
// Order is important here. We have to read out the slot type last to
// observe the concurrent removal case consistently.
@@ -363,16 +373,40 @@
Address addr = page_start_ + type_and_offset.second;
if (callback(type, host_addr, addr) == KEEP_SLOT) {
new_count++;
+ empty = false;
} else {
buffer[i].Clear();
}
}
}
+
+ if (mode == PREFREE_EMPTY_CHUNKS && empty) {
+ // We remove the chunk from the list but let it still point its next
+ // chunk to allow concurrent iteration.
+ if (previous) {
+ previous->next.SetValue(chunk->next.Value());
+ } else {
+ chunk_.SetValue(chunk->next.Value());
+ }
+ base::LockGuard<base::Mutex> guard(&to_be_freed_chunks_mutex_);
+ to_be_freed_chunks_.push(chunk);
+ } else {
+ previous = chunk;
+ }
chunk = chunk->next.Value();
}
return new_count;
}
+ void FreeToBeFreedChunks() {
+ base::LockGuard<base::Mutex> guard(&to_be_freed_chunks_mutex_);
+ while (!to_be_freed_chunks_.empty()) {
+ Chunk* top = to_be_freed_chunks_.top();
+ to_be_freed_chunks_.pop();
+ delete top;
+ }
+ }
+
private:
static const int kInitialBufferSize = 100;
static const int kMaxBufferSize = 16 * KB;
@@ -411,6 +445,8 @@
Address page_start_;
base::AtomicValue<Chunk*> chunk_;
+ base::Mutex to_be_freed_chunks_mutex_;
+ std::stack<Chunk*> to_be_freed_chunks_;
};
} // namespace internal
diff --git a/test/unittests/heap/slot-set-unittest.cc b/test/unittests/heap/slot-set-unittest.cc
index 9bf8507..8db5b1a8 100644
--- a/test/unittests/heap/slot-set-unittest.cc
+++ b/test/unittests/heap/slot-set-unittest.cc
@@ -152,24 +152,29 @@
++added;
}
int iterated = 0;
- set.Iterate([&iterated, kDelta, kHostDelta](SlotType type, Address host_addr,
- Address addr) {
- uint32_t i = static_cast<uint32_t>(reinterpret_cast<uintptr_t>(addr));
- uint32_t j = static_cast<uint32_t>(reinterpret_cast<uintptr_t>(host_addr));
- EXPECT_EQ(i % CLEARED_SLOT, static_cast<uint32_t>(type));
- EXPECT_EQ(0, i % kDelta);
- EXPECT_EQ(0, j % kHostDelta);
- ++iterated;
- return i % 2 == 0 ? KEEP_SLOT : REMOVE_SLOT;
- });
+ set.Iterate(
+ [&iterated, kDelta, kHostDelta](SlotType type, Address host_addr,
+ Address addr) {
+ uint32_t i = static_cast<uint32_t>(reinterpret_cast<uintptr_t>(addr));
+ uint32_t j =
+ static_cast<uint32_t>(reinterpret_cast<uintptr_t>(host_addr));
+ EXPECT_EQ(i % CLEARED_SLOT, static_cast<uint32_t>(type));
+ EXPECT_EQ(0, i % kDelta);
+ EXPECT_EQ(0, j % kHostDelta);
+ ++iterated;
+ return i % 2 == 0 ? KEEP_SLOT : REMOVE_SLOT;
+ },
+ TypedSlotSet::KEEP_EMPTY_CHUNKS);
EXPECT_EQ(added, iterated);
iterated = 0;
- set.Iterate([&iterated](SlotType type, Address host_addr, Address addr) {
- uint32_t i = static_cast<uint32_t>(reinterpret_cast<uintptr_t>(addr));
- EXPECT_EQ(0, i % 2);
- ++iterated;
- return KEEP_SLOT;
- });
+ set.Iterate(
+ [&iterated](SlotType type, Address host_addr, Address addr) {
+ uint32_t i = static_cast<uint32_t>(reinterpret_cast<uintptr_t>(addr));
+ EXPECT_EQ(0, i % 2);
+ ++iterated;
+ return KEEP_SLOT;
+ },
+ TypedSlotSet::KEEP_EMPTY_CHUNKS);
EXPECT_EQ(added / 2, iterated);
}
