lib/Common/Memory/HeapBucket.cpp - external/github.com/Microsoft/ChakraCore - Git at Google

 //-------------------------------------------------------------------------------------------------------
 // Copyright (C) Microsoft. All rights reserved.
 // Licensed under the MIT license. See LICENSE.txt file in the project root for full license information.
 //-------------------------------------------------------------------------------------------------------
 #include "CommonMemoryPch.h"


 HeapBucket::HeapBucket() :
     heapInfo(nullptr),
     sizeCat(0)
 {
 #if defined(RECYCLER_SLOW_CHECK_ENABLED) || ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP
     heapBlockCount = 0;
     newHeapBlockCount = 0;
 #endif

 #if defined(RECYCLER_SLOW_CHECK_ENABLED)
     emptyHeapBlockCount = 0;
 #endif

 #if ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP
     this->allocationsStartedDuringConcurrentSweep = false;
     this->concurrentSweepAllocationsThresholdExceeded = false;
 #endif

 #ifdef RECYCLER_PAGE_HEAP
     isPageHeapEnabled = false;
 #endif
 }

 HeapInfo *
 HeapBucket::GetHeapInfo() const
 {
     return this->heapInfo;
 }

 uint
 HeapBucket::GetSizeCat() const
 {
     return this->sizeCat;
 }

 uint
 HeapBucket::GetBucketIndex() const
 {
     return HeapInfo::GetBucketIndex(this->sizeCat);
 }

 uint
 HeapBucket::GetMediumBucketIndex() const
 {
     return HeapInfo::GetMediumBucketIndex(this->sizeCat);
 }

 namespace Memory
 {

 template <typename TBlockType>
 HeapBucketT<TBlockType>::HeapBucketT() :
     nextAllocableBlockHead(nullptr),
     emptyBlockList(nullptr),
     fullBlockList(nullptr),
     heapBlockList(nullptr),
 #if ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP
 #if SUPPORT_WIN32_SLIST
     lastKnownNextAllocableBlockHead(nullptr),
     allocableHeapBlockListHead(nullptr),
     sweepableHeapBlockList(nullptr),
 #endif
 #endif
     explicitFreeList(nullptr),
     lastExplicitFreeListAllocator(nullptr)
 {
 #ifdef RECYCLER_PAGE_HEAP
     explicitFreeLockBlockList = nullptr;
 #endif

     isAllocationStopped = false;
 }


 template <typename TBlockType>
 HeapBucketT<TBlockType>::~HeapBucketT()
 {
     DeleteHeapBlockList(this->heapBlockList);
     DeleteHeapBlockList(this->fullBlockList);

 #if ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP
 #if SUPPORT_WIN32_SLIST
     if (allocableHeapBlockListHead != nullptr)
     {
         if (CONFIG_FLAG_RELEASE(EnableConcurrentSweepAlloc))
         {
             FlushInterlockedSList(this->allocableHeapBlockListHead);
         }

         _aligned_free(this->allocableHeapBlockListHead);
     }

     DeleteHeapBlockList(this->sweepableHeapBlockList);
 #endif
 #endif

 #if defined(RECYCLER_SLOW_CHECK_ENABLED) || ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP
     Assert(this->heapBlockCount + this->newHeapBlockCount == 0);
 #endif
     RECYCLER_SLOW_CHECK(Assert(this->emptyHeapBlockCount == HeapBlockList::Count(this->emptyBlockList)));
     DeleteEmptyHeapBlockList(this->emptyBlockList);
 #if defined(RECYCLER_SLOW_CHECK_ENABLED)
     Assert(this->heapBlockCount + this->newHeapBlockCount + this->emptyHeapBlockCount == 0);
 #endif
 }
 };

 template <typename TBlockType>
 void
 HeapBucketT<TBlockType>::DeleteHeapBlockList(TBlockType * list, Recycler * recycler)
 {
     HeapBlockList::ForEachEditing(list, [recycler](TBlockType * heapBlock)
     {
 #if DBG
         heapBlock->ReleasePagesShutdown(recycler);
 #endif
         TBlockType::Delete(heapBlock);
     });
 }

 template <typename TBlockType>
 void
 HeapBucketT<TBlockType>::DeleteEmptyHeapBlockList(TBlockType * list)
 {
     HeapBlockList::ForEachEditing(list, [](TBlockType * heapBlock)
     {
         TBlockType::Delete(heapBlock);
     });
 }

 template <typename TBlockType>
 void
 HeapBucketT<TBlockType>::DeleteHeapBlockList(TBlockType * list)
 {
     DeleteHeapBlockList(list, this->heapInfo->recycler);
 }

 #if ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP && SUPPORT_WIN32_SLIST
 template<typename TBlockType>
 bool
 HeapBucketT<TBlockType>::PushHeapBlockToSList(PSLIST_HEADER list, TBlockType * heapBlock)
 {
     Assert(list != nullptr);
     HeapBlockSListItem<TBlockType> * currentBlock = (HeapBlockSListItem<TBlockType> *) _aligned_malloc(sizeof(HeapBlockSListItem<TBlockType>), MEMORY_ALLOCATION_ALIGNMENT);
     if (currentBlock == nullptr)
     {
         return false;
     }

     // While in the SLIST the blocks live as standalone, when they come out they
     // will go into appropriate list and the Next block will be set accordingly.
     heapBlock->SetNextBlock(nullptr);
     currentBlock->itemHeapBlock = heapBlock;

     ::InterlockedPushEntrySList(list, &(currentBlock->itemEntry));
     return true;
 }

 template<typename TBlockType>
 TBlockType *
 HeapBucketT<TBlockType>::PopHeapBlockFromSList(PSLIST_HEADER list)
 {
     Assert(list != nullptr);
     TBlockType * heapBlock = nullptr;

     PSLIST_ENTRY top = ::InterlockedPopEntrySList(list);
     if (top != nullptr)
     {
         HeapBlockSListItem<TBlockType> * topItem = (HeapBlockSListItem<TBlockType> *) top;
         heapBlock = topItem->itemHeapBlock;
         Assert(heapBlock != nullptr);
         _aligned_free(top);
     }

     return heapBlock;
 }

 template<typename TBlockType>
 ushort
 HeapBucketT<TBlockType>::QueryDepthInterlockedSList(PSLIST_HEADER list)
 {
     Assert(list != nullptr);
     return ::QueryDepthSList(list);
 }

 template<typename TBlockType>
 void
 HeapBucketT<TBlockType>::FlushInterlockedSList(PSLIST_HEADER list)
 {
     Assert(list != nullptr);
     if (::QueryDepthSList(list) > 0)
     {
         PSLIST_ENTRY listEntry = ::InterlockedPopEntrySList(list);
         while (listEntry != nullptr)
         {
             _aligned_free(listEntry);
             listEntry = ::InterlockedPopEntrySList(list);
         }
     }

     ::InterlockedFlushSList(list);
 }
 #endif

 template <typename TBlockType>
 void
 HeapBucketT<TBlockType>::Initialize(HeapInfo * heapInfo, uint sizeCat)
 {
     this->heapInfo = heapInfo;
 #ifdef RECYCLER_PAGE_HEAP
     this->isPageHeapEnabled = heapInfo->IsPageHeapEnabledForBlock<typename TBlockType::HeapBlockAttributes>(sizeCat);
 #endif
     this->sizeCat = sizeCat;
     allocatorHead.Initialize();
 #if defined(PROFILE_RECYCLER_ALLOC) || defined(RECYCLER_MEMORY_VERIFY)
     allocatorHead.bucket = this;
 #endif
     this->lastExplicitFreeListAllocator = &allocatorHead;
 }

 template <typename TBlockType>
 template <class Fn>
 void
 HeapBucketT<TBlockType>::ForEachAllocator(Fn fn)
 {
     TBlockAllocatorType * current = &allocatorHead;
     do
     {
         fn(current);
         current = current->GetNext();
     }
     while (current != &allocatorHead);
 }

 template <typename TBlockType>
 void
 HeapBucketT<TBlockType>::UpdateAllocators()
 {
     ForEachAllocator([](TBlockAllocatorType * allocator) { allocator->UpdateHeapBlock(); });
 }

 template <typename TBlockType>
 void
 HeapBucketT<TBlockType>::ClearAllocators()
 {
     ForEachAllocator([](TBlockAllocatorType * allocator) { ClearAllocator(allocator); });

 #ifdef RECYCLER_PAGE_HEAP

 #endif

 #ifdef RECYCLER_MEMORY_VERIFY
     FreeObject* freeObject = this->explicitFreeList;

     while (freeObject)
     {
         HeapBlock* heapBlock = this->GetRecycler()->FindHeapBlock((void*)freeObject);
         Assert(heapBlock != nullptr);
         Assert(!heapBlock->IsLargeHeapBlock());
         TBlockType* smallBlock = (TBlockType*)heapBlock;

         smallBlock->ClearExplicitFreeBitForObject((void*)freeObject);
         freeObject = freeObject->GetNext();
     }
 #endif

     this->explicitFreeList = nullptr;
 }

 #if ENABLE_CONCURRENT_GC
 template <typename TBlockType>
 void
 HeapBucketT<TBlockType>::PrepareSweep()
 {
     // CONCURRENT-TODO: Technically, We don't really need to invalidate allocators here,
     // but currently invalidating may update the unallocateCount which is
     // used to calculate the partial heuristics, so it needs to be done
     // before sweep. When the partial heuristic changes, we can remove this
     // (And remove rescan from leaf bucket, so this function doesn't need to exist)
     ClearAllocators();
 }
 #endif

 template <typename TBlockType>
 void
 HeapBucketT<TBlockType>::AddAllocator(TBlockAllocatorType * allocator)
 {
     Assert(allocator != &this->allocatorHead);
     allocator->Initialize();
     allocator->next = this->allocatorHead.next;
     allocator->prev = &this->allocatorHead;
     allocator->next->prev = allocator;
     this->allocatorHead.next = allocator;
 #if defined(PROFILE_RECYCLER_ALLOC) || defined(RECYCLER_MEMORY_VERIFY)
     allocator->bucket = this;
 #endif
 }

 template <typename TBlockType>
 void
 HeapBucketT<TBlockType>::RemoveAllocator(TBlockAllocatorType * allocator)
 {
     Assert(allocator != &this->allocatorHead);
     ClearAllocator(allocator);

     allocator->next->prev = allocator->prev;
     allocator->prev->next = allocator->next;

     if (allocator == this->lastExplicitFreeListAllocator)
     {
         this->lastExplicitFreeListAllocator = &allocatorHead;
     }
 }

 template <typename TBlockType>
 void
 HeapBucketT<TBlockType>::ClearAllocator(TBlockAllocatorType * allocator)
 {
     allocator->Clear();
 }

 template <typename TBlockType>
 bool
 HeapBucketT<TBlockType>::IntegrateBlock(char * blockAddress, PageSegment * segment, Recycler * recycler)
 {
     // Add a new heap block
     TBlockType * heapBlock = GetUnusedHeapBlock();
     if (heapBlock == nullptr)
     {
         return false;
     }

     // TODO: Consider supporting guard pages for this codepath
     if (!heapBlock->SetPage(blockAddress, segment, recycler))
     {
         FreeHeapBlock(heapBlock);
         return false;
     }

     heapBlock->SetNextBlock(this->fullBlockList);
     this->fullBlockList = heapBlock;

 #if defined(RECYCLER_SLOW_CHECK_ENABLED) || ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP
     this->heapBlockCount++;
 #endif

     recycler->autoHeap.uncollectedAllocBytes += heapBlock->GetAndClearLastFreeCount() * heapBlock->GetObjectSize();
     RecyclerMemoryTracking::ReportAllocation(recycler, blockAddress, heapBlock->GetObjectSize() * heapBlock->GetObjectCount());
     RECYCLER_PERF_COUNTER_ADD(LiveObject,heapBlock->GetObjectCount());
     RECYCLER_PERF_COUNTER_ADD(LiveObjectSize, heapBlock->GetObjectSize() * heapBlock->GetObjectCount());

     if (heapBlock->IsLargeHeapBlock())
     {
         RECYCLER_PERF_COUNTER_ADD(LargeHeapBlockLiveObject,heapBlock->GetObjectCount());
         RECYCLER_PERF_COUNTER_ADD(LargeHeapBlockLiveObjectSize, heapBlock->GetObjectSize() * heapBlock->GetObjectCount());
     }
     else
     {
         RECYCLER_PERF_COUNTER_ADD(SmallHeapBlockLiveObject,heapBlock->GetObjectCount());
         RECYCLER_PERF_COUNTER_ADD(SmallHeapBlockLiveObjectSize, heapBlock->GetObjectSize() * heapBlock->GetObjectCount());
     }

 #if DBG
     heapBlock->SetIsIntegratedBlock();
 #endif

     return true;
 }

 #if DBG
 template <typename TBlockType>
 bool
 HeapBucketT<TBlockType>::AllocatorsAreEmpty() const
 {
     TBlockAllocatorType const * current = &allocatorHead;
     do
     {
         if (current->GetHeapBlock() != nullptr || current->GetExplicitFreeList() != nullptr)
         {
             return false;
         }
         current = current->GetNext();
     }
     while (current != &allocatorHead);
     return true;
 }

 template <typename TBlockType>
 bool
 HeapBucketT<TBlockType>::HasPendingDisposeHeapBlocks() const
 {
 #ifdef RECYCLER_WRITE_BARRIER
     return (IsFinalizableBucket || IsFinalizableWriteBarrierBucket) &&
     ((SmallFinalizableHeapBucketT<typename TBlockType::HeapBlockAttributes> *)this)->pendingDisposeList != nullptr;
 #else
     return IsFinalizableBucket && ((SmallFinalizableHeapBucketT<typename TBlockType::HeapBlockAttributes> *)this)->pendingDisposeList != nullptr;
 #endif
 }

 #endif

 template <typename TBlockType>
 void
 HeapBucketT<TBlockType>::AssertCheckHeapBlockNotInAnyList(TBlockType * heapBlock)
 {
 #if DBG
     AssertMsg(!HeapBlockList::Contains(heapBlock, heapBlockList), "The heap block already exists in the heapBlockList.");
     AssertMsg(!HeapBlockList::Contains(heapBlock, fullBlockList), "The heap block already exists in the fullBlockList.");
     AssertMsg(!HeapBlockList::Contains(heapBlock, emptyBlockList), "The heap block already exists in the emptyBlockList.");
 #if ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP
     AssertMsg(!HeapBlockList::Contains(heapBlock, sweepableHeapBlockList), "The heap block already exists in the sweepableHeapBlockList.");
 #endif
 #endif
 }

 #if DBG || defined(RECYCLER_SLOW_CHECK_ENABLED)
 template <typename TBlockType>
 size_t
 HeapBucketT<TBlockType>::GetNonEmptyHeapBlockCount(bool checkCount) const
 {
     size_t currentHeapBlockCount = HeapBlockList::Count(fullBlockList);
     currentHeapBlockCount += HeapBlockList::Count(heapBlockList);
     bool allocatingDuringConcurrentSweep = false;

 #if ENABLE_CONCURRENT_GC
 #if ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP
 #if SUPPORT_WIN32_SLIST
     if (CONFIG_FLAG_RELEASE(EnableConcurrentSweepAlloc) && !this->IsAnyFinalizableBucket())
     {
         allocatingDuringConcurrentSweep = true;
         // This lock is needed only in the debug mode while we verify block counts. Not needed otherwise, as this list is never accessed concurrently.
         // Items are added to it by the allocator when allocations are allowed during concurrent sweep. The list is drained during the next sweep while
         // allocation are stopped.
         debugSweepableHeapBlockListLock.Enter();
         if (allocableHeapBlockListHead != nullptr)
         {
             currentHeapBlockCount += QueryDepthInterlockedSList(allocableHeapBlockListHead);
         }
         currentHeapBlockCount += HeapBlockList::Count(sweepableHeapBlockList);
         debugSweepableHeapBlockListLock.Leave();
     }
 #endif
 #endif

     // Recycler can be null if we have OOM in the ctor
     if (this->GetRecycler() && this->GetRecycler()->recyclerSweepManager != nullptr)
     {
         currentHeapBlockCount += this->GetRecycler()->recyclerSweepManager->GetHeapBlockCount(this);
     }
 #endif

     // There is no way to determine the number of item in an SLIST if there are >= 65535 items in the list.
     RECYCLER_SLOW_CHECK(Assert(!checkCount || heapBlockCount == currentHeapBlockCount || (heapBlockCount >= 65535 && allocatingDuringConcurrentSweep)));

     return currentHeapBlockCount;
 }

 template <typename TBlockType>
 size_t
 HeapBucketT<TBlockType>::GetEmptyHeapBlockCount() const
 {
     size_t count = HeapBlockList::Count(this->emptyBlockList);
     RECYCLER_SLOW_CHECK(Assert(count == this->emptyHeapBlockCount));
     return count;
 }
 #endif

 template <typename TBlockType>
 char *
 HeapBucketT<TBlockType>::TryAlloc(Recycler * recycler, TBlockAllocatorType * allocator, size_t sizeCat, ObjectInfoBits attributes)
 {
     AUTO_NO_EXCEPTION_REGION;

     Assert((attributes & InternalObjectInfoBitMask) == attributes);

     ClearAllocator(allocator);

     TBlockType * heapBlock = this->nextAllocableBlockHead;
 #if ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP
 #if SUPPORT_WIN32_SLIST
     bool heapBlockFromAllocableHeapBlockList = false;
     DebugOnly(bool heapBlockInPendingSweepPrepList = false);

     if (heapBlock == nullptr && this->allocationsStartedDuringConcurrentSweep)
     {
 #if DBG || defined(RECYCLER_SLOW_CHECK_ENABLED)
         // This lock is needed only in the debug mode while we verify block counts. Not needed otherwise, as this list is never accessed concurrently.
         // Items are added to it by the allocator when allocations are allowed during concurrent sweep. The list is drained during the next sweep while
         // allocation are stopped.
         debugSweepableHeapBlockListLock.Enter();
 #endif
         heapBlock = PopHeapBlockFromSList(this->allocableHeapBlockListHead);
         if (heapBlock != nullptr)
         {
             Assert(!this->IsAnyFinalizableBucket());
             DebugOnly(this->AssertCheckHeapBlockNotInAnyList(heapBlock));
 #if DBG || defined(RECYCLER_SLOW_CHECK_ENABLED)
             heapBlock->wasAllocatedFromDuringSweep = true;
 #endif
 #if DBG || defined(RECYCLER_TRACE)
             if (heapBlock->isPendingConcurrentSweepPrep)
             {
                 AssertMsg(heapBlock->objectsAllocatedDuringConcurrentSweepCount == 0, "We just picked up this block for allocations during concurrent sweep, we haven't allocated from it yet.");

 #ifdef RECYCLER_TRACE
                 recycler->PrintBlockStatus(this, heapBlock, _u("[**31**] pending Pass1 prep, picked up for allocations during concurrent sweep."));
 #endif
                 DebugOnly(heapBlockInPendingSweepPrepList = true);
             }
             else
             {
                 // Put the block in the sweepable heap block list so we don't lose track of it. The block will eventually be moved to the
                 // heapBlockList or fullBlockList as appropriate during the next sweep.
                 AssertMsg(!HeapBlockList::Contains(heapBlock, sweepableHeapBlockList), "The heap block already exists in this list.");

 #ifdef RECYCLER_TRACE
                 recycler->PrintBlockStatus(this, heapBlock, _u("[**32**] picked up for allocations during concurrent sweep."));
 #endif
             }
 #endif
             heapBlock->SetNextBlock(sweepableHeapBlockList);
             sweepableHeapBlockList = heapBlock;
             heapBlockFromAllocableHeapBlockList = true;
         }
 #if DBG|| defined(RECYCLER_SLOW_CHECK_ENABLED)
         debugSweepableHeapBlockListLock.Leave();
 #endif
     }
 #endif
 #endif

    if (heapBlock != nullptr)
    {
         Assert(!this->IsAllocationStopped());

 #if ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP && SUPPORT_WIN32_SLIST
         // When allocations are allowed during concurrent sweep we set nextAllocableBlockHead to NULL as the allocator will pick heap blocks from the
         // interlocked SLIST. During that time, the heap block at the top of the SLIST is always the nextAllocableBlockHead.
         // If the heapBlock was just picked from the SLIST and nextAllocableBlockHead is not NULL then we just resumed normal allocations on the background thread
         // while finishing the concurrent sweep, and the nextAllocableBlockHead is already set properly.
         if (this->nextAllocableBlockHead != nullptr && !heapBlockFromAllocableHeapBlockList)
 #endif
         {
             this->nextAllocableBlockHead = heapBlock->GetNextBlock();
         }

         allocator->Set(heapBlock);
     }
     else if (this->explicitFreeList != nullptr)
     {
         allocator->SetExplicitFreeList(this->explicitFreeList);
         this->lastExplicitFreeListAllocator = allocator;
         this->explicitFreeList = nullptr;
     }
     else
     {
         return nullptr;
     }
     // We just found a block we can allocate on
     char * memBlock = allocator->template SlowAlloc<false /* disallow fault injection */>(recycler, sizeCat, attributes);
     Assert(memBlock != nullptr);
     return memBlock;
 }

 template <typename TBlockType>
 char *
 HeapBucketT<TBlockType>::TryAllocFromNewHeapBlock(Recycler * recycler, TBlockAllocatorType * allocator, size_t sizeCat, size_t size, ObjectInfoBits attributes)
 {
     AUTO_NO_EXCEPTION_REGION;

     Assert((attributes & InternalObjectInfoBitMask) == attributes);

 #ifdef RECYCLER_PAGE_HEAP
     if (IsPageHeapEnabled(attributes))
     {
         return this->PageHeapAlloc(recycler, sizeCat, size, attributes, this->heapInfo->pageHeapMode, true);
     }
 #endif

     TBlockType * heapBlock = CreateHeapBlock(recycler);
     if (heapBlock == nullptr)
     {
         return nullptr;
     }

     // new heap block added, allocate from that.
     allocator->SetNew(heapBlock);
     // We just created a block we can allocate on
     char * memBlock = allocator->template SlowAlloc<false /* disallow fault injection */>(recycler, sizeCat, attributes);
     Assert(memBlock != nullptr || IS_FAULTINJECT_NO_THROW_ON);
     return memBlock;
 }

 Recycler *
 HeapBucket::GetRecycler() const
 {
     return this->heapInfo->recycler;
 }

 bool
 HeapBucket::AllocationsStartedDuringConcurrentSweep() const
 {
 #if ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP
     return this->allocationsStartedDuringConcurrentSweep;
 #else
     return false;
 #endif
 }

 #if ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP
 bool
 HeapBucket::ConcurrentSweepAllocationsThresholdExceeded() const
 {
     return this->concurrentSweepAllocationsThresholdExceeded;
 }

 bool
 HeapBucket::DoTwoPassConcurrentSweepPreCheck()
 {
     this->concurrentSweepAllocationsThresholdExceeded = ((this->heapBlockCount + this->newHeapBlockCount) > RecyclerHeuristic::AllocDuringConcurrentSweepHeapBlockThreshold);

 #ifdef RECYCLER_TRACE
     if (this->GetRecycler()->GetRecyclerFlagsTable().Trace.IsEnabled(Js::ConcurrentSweepPhase) && CONFIG_FLAG_RELEASE(Verbose))
     {
         if (this->concurrentSweepAllocationsThresholdExceeded)
         {
             Output::Print(_u("[HeapBucket 0x%p] exceeded concurrent sweep allocations threshold (%d). Total heap block count: %d \n"), this, RecyclerHeuristic::AllocDuringConcurrentSweepHeapBlockThreshold, this->heapBlockCount + this->newHeapBlockCount);
         }
     }
 #endif

     return this->concurrentSweepAllocationsThresholdExceeded;
 }
 #endif

 #ifdef RECYCLER_PAGE_HEAP
 template <typename TBlockType>
 char *
 HeapBucketT<TBlockType>::PageHeapAlloc(Recycler * recycler, DECLSPEC_GUARD_OVERFLOW size_t sizeCat, size_t size, ObjectInfoBits attributes, PageHeapMode mode, bool nothrow)
 {
     AllocationVerboseTrace(recycler->GetRecyclerFlagsTable(), _u("In PageHeapAlloc [Size: 0x%x, Attributes: 0x%x]\n"), size, attributes);
     char* addr =  heapInfo->largeObjectBucket.PageHeapAlloc(recycler, sizeCat, size, attributes, mode, nothrow);

     if (addr)
     {
         this->GetRecycler()->autoHeap.uncollectedAllocBytes += sizeCat;
     }

     return addr;
 }
 #endif

 template <typename TBlockType>
 char *
 HeapBucketT<TBlockType>::SnailAlloc(Recycler * recycler, TBlockAllocatorType * allocator, DECLSPEC_GUARD_OVERFLOW size_t sizeCat, size_t size, ObjectInfoBits attributes, bool nothrow)
 {
     AllocationVerboseTrace(recycler->GetRecyclerFlagsTable(), _u("In SnailAlloc [Size: 0x%x, Attributes: 0x%x]\n"), sizeCat, attributes);

     Assert(sizeCat == this->sizeCat);
     Assert((attributes & InternalObjectInfoBitMask) == attributes);

     char * memBlock = this->TryAlloc(recycler, allocator, sizeCat, attributes);
     if (memBlock != nullptr)
     {
         return memBlock;
     }

 #if ENABLE_CONCURRENT_GC
     // No free memory, try to collect with allocated bytes and time heuristic, and concurrently
     BOOL collected = recycler->disableCollectOnAllocationHeuristics ? recycler->FinishConcurrent<FinishConcurrentOnAllocation>() :
         recycler->CollectNow<CollectOnAllocation>();
 #else
     BOOL collected = recycler->disableCollectOnAllocationHeuristics ? FALSE : recycler->CollectNow<CollectOnAllocation>();
 #endif

     AllocationVerboseTrace(recycler->GetRecyclerFlagsTable(), _u("TryAlloc failed, forced collection on allocation [Collected: %d]\n"), collected);

     if (!collected)
     {
 #if ENABLE_CONCURRENT_GC
 #if ENABLE_PARTIAL_GC
         // wait for background sweeping finish if there are too many pages allocated during background sweeping
         if (recycler->IsConcurrentSweepExecutingState() && recycler->autoHeap.uncollectedNewPageCount > (uint)CONFIG_FLAG(NewPagesCapDuringBGSweeping))
 #else
         if (recycler->IsConcurrentSweepExecutingState())
 #endif
         {
             recycler->FinishConcurrent<ForceFinishCollection>();
             memBlock = this->TryAlloc(recycler, allocator, sizeCat, attributes);
             if (memBlock != nullptr)
             {
                 return memBlock;
             }
         }
 #endif

         // We didn't collect, try to add a new heap block
         memBlock = TryAllocFromNewHeapBlock(recycler, allocator, sizeCat, size, attributes);
         if (memBlock != nullptr)
         {
             return memBlock;
         }

         // Can't even allocate a new block, we need force a collection and
         //allocate some free memory, add a new heap block again, or throw out of memory
         AllocationVerboseTrace(recycler->GetRecyclerFlagsTable(), _u("TryAllocFromNewHeapBlock failed, forcing in-thread collection\n"));
         recycler->CollectNow<CollectNowForceInThread>();
     }

     // Collection might trigger finalizer, which might allocate memory. So the allocator
     // might have a heap block already, try to allocate from that first
     memBlock = allocator->template SlowAlloc<true /* allow fault injection */>(recycler, sizeCat, attributes);
     if (memBlock != nullptr)
     {
         return memBlock;
     }

     AllocationVerboseTrace(recycler->GetRecyclerFlagsTable(), _u("SlowAlloc failed\n"));

     // do the allocation
     memBlock = this->TryAlloc(recycler, allocator, sizeCat, attributes);
     if (memBlock != nullptr)
     {
         return memBlock;
     }

     AllocationVerboseTrace(recycler->GetRecyclerFlagsTable(), _u("TryAlloc failed\n"));
     // add a heap block if there are no preallocated memory left.
     memBlock = TryAllocFromNewHeapBlock(recycler, allocator, sizeCat, size, attributes);
     if (memBlock != nullptr)
     {
         return memBlock;
     }

     AllocationVerboseTrace(recycler->GetRecyclerFlagsTable(), _u("TryAllocFromNewHeapBlock failed- triggering OOM handler"));

     if (nothrow == false)
     {
         // Can't add a heap block, we are out of memory
         // Since we're allowed to throw, throw right here
         recycler->OutOfMemory();
     }

     return nullptr;
 }

 template <typename TBlockType>
 TBlockType*
 HeapBucketT<TBlockType>::GetUnusedHeapBlock()
 {
     // Add a new heap block
     TBlockType * heapBlock = emptyBlockList;
     if (heapBlock == nullptr)
     {
         // We couldn't find a reusable heap block
         heapBlock = TBlockType::New(this);
 #if defined(RECYCLER_SLOW_CHECK_ENABLED)
         Assert(this->emptyHeapBlockCount == 0);
 #endif
     }
     else
     {
         emptyBlockList = heapBlock->GetNextBlock();
 #if defined(RECYCLER_SLOW_CHECK_ENABLED)
         this->emptyHeapBlockCount--;
 #endif
     }
     return heapBlock;
 }

 template <typename TBlockType>
 TBlockType *
 HeapBucketT<TBlockType>::CreateHeapBlock(Recycler * recycler)
 {
     FAULTINJECT_MEMORY_NOTHROW(_u("HeapBlock"), sizeof(TBlockType));

     // Add a new heap block
     TBlockType * heapBlock = GetUnusedHeapBlock();
     if (heapBlock == nullptr)
     {
         return nullptr;
     }

     if (!heapBlock->ReassignPages(recycler))
     {
         FreeHeapBlock(heapBlock);
         return nullptr;
     }

     // Add it to head of heap block list so we will keep track of the block
     this->heapInfo->AppendNewHeapBlock(heapBlock, this);
 #if defined(RECYCLER_SLOW_CHECK_ENABLED) || ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP
 #if ENABLE_CONCURRENT_GC
     ::InterlockedIncrement(&this->newHeapBlockCount);
 #else
     this->heapBlockCount++;
 #endif
 #endif
     return heapBlock;
 }

 template <typename TBlockType>
 void
 HeapBucketT<TBlockType>::FreeHeapBlock(TBlockType * heapBlock)
 {
     heapBlock->Reset();
     heapBlock->SetNextBlock(emptyBlockList);
     emptyBlockList = heapBlock;
 #if defined(RECYCLER_SLOW_CHECK_ENABLED)
     this->emptyHeapBlockCount++;
 #endif
 }

 template <typename TBlockType>
 void
 HeapBucketT<TBlockType>::ResetMarks(ResetMarkFlags flags)
 {
     RECYCLER_SLOW_CHECK(this->VerifyHeapBlockCount((flags & ResetMarkFlags_Background) != 0));

 #if !ENABLE_CONCURRENT_GC
     Assert((flags & ResetMarkFlags_Background) == 0);
 #endif

     if ((flags & ResetMarkFlags_Background) == 0)
     {
         // The is equivalent to the ClearAllocators in Rescan
         // But since we are not doing concurrent, we need to do it here.
         ClearAllocators();
     }

     // Note, mark bits are now cleared in HeapBlockMap32::ResetMarks, so we don't need to clear them here.

     if ((flags & ResetMarkFlags_ScanImplicitRoot) != 0)
     {
         HeapBlockList::ForEach(fullBlockList, [flags](TBlockType * heapBlock)
         {
             heapBlock->MarkImplicitRoots();
             Assert(!heapBlock->HasFreeObject());
         });

 #if ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP && SUPPORT_WIN32_SLIST
         if (CONFIG_FLAG_RELEASE(EnableConcurrentSweepAlloc) && !this->IsAnyFinalizableBucket())
         {
             HeapBlockList::ForEach(sweepableHeapBlockList, [flags](TBlockType * heapBlock)
             {
                 heapBlock->MarkImplicitRoots();
             });
         }
 #endif

         HeapBlockList::ForEach(heapBlockList, [flags](TBlockType * heapBlock)
         {
             heapBlock->MarkImplicitRoots();
         });
     }

 #if DBG
     if ((flags & ResetMarkFlags_Background) == 0)
     {
         // When allocations are enabled for buckets during oncurrent sweep we don't keep track of the nextAllocableBlockHead as it directly
         // comes out of the SLIST. As a result, the below validations can't be performed reliably on a heap block.
 #if ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP
         if (!CONFIG_FLAG_RELEASE(EnableConcurrentSweepAlloc) || this->IsAnyFinalizableBucket())
 #endif
         {
             // Verify that if you are in the heapBlockList, before the nextAllocableBlockHead, we have fully allocated from
             // the block already, except if we have cleared from the allocator, or it is still in the allocator
             HeapBlockList::ForEach(heapBlockList, nextAllocableBlockHead, [](TBlockType * heapBlock)
             {
                 // If the heap block is in the allocator, then the heap block may or may not have free object still
                 // So we can't assert.  Otherwise, we have free object iff we were cleared from allocator
                 Assert(heapBlock->IsInAllocator() || heapBlock->HasFreeObject() == heapBlock->IsClearedFromAllocator());
             });

             // We should still have allocable free object after nextAllocableBlockHead
             HeapBlockList::ForEach(nextAllocableBlockHead, [](TBlockType * heapBlock)
             {
                 Assert(heapBlock->HasFreeObject());
             });
         }
     }
 #endif
 }

 template <typename TBlockType>
 void
 HeapBucketT<TBlockType>::ScanNewImplicitRoots(Recycler * recycler)
 {
     HeapBlockList::ForEach(fullBlockList, [recycler](TBlockType * heapBlock)
     {
         heapBlock->ScanNewImplicitRoots(recycler);
     });

 #if ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP && SUPPORT_WIN32_SLIST
     if (CONFIG_FLAG_RELEASE(EnableConcurrentSweepAlloc) && !this->IsAnyFinalizableBucket())
     {
         HeapBlockList::ForEach(sweepableHeapBlockList, [recycler](TBlockType * heapBlock)
         {
             heapBlock->ScanNewImplicitRoots(recycler);
         });
     }
 #endif

     HeapBlockList::ForEach(heapBlockList, [recycler](TBlockType * heapBlock)
     {
         heapBlock->ScanNewImplicitRoots(recycler);
     });
 }

 #if DBG
 template <typename TBlockType>
 void
 HeapBucketT<TBlockType>::VerifyBlockConsistencyInList(TBlockType * heapBlock, RecyclerVerifyListConsistencyData& recyclerSweep)
 {
     bool* expectFull = nullptr;
     bool* expectDispose = nullptr;
     HeapBlock* nextAllocableBlockHead = nullptr;

     if (TBlockType::HeapBlockAttributes::IsSmallBlock)
     {
         expectFull = &recyclerSweep.smallBlockVerifyListConsistencyData.expectFull;
         expectDispose = &recyclerSweep.smallBlockVerifyListConsistencyData.expectDispose;
         nextAllocableBlockHead = recyclerSweep.smallBlockVerifyListConsistencyData.nextAllocableBlockHead;
     }
     else if (TBlockType::HeapBlockAttributes::IsMediumBlock)
     {
         expectFull = &recyclerSweep.mediumBlockVerifyListConsistencyData.expectFull;
         expectDispose = &recyclerSweep.mediumBlockVerifyListConsistencyData.expectDispose;
         nextAllocableBlockHead = recyclerSweep.mediumBlockVerifyListConsistencyData.nextAllocableBlockHead;
     }
     else
     {
         Assert(false);
     }

     if (heapBlock == nextAllocableBlockHead)
     {
         (*expectFull) = false;
     }
     if (heapBlock->IsClearedFromAllocator())
     {
 #if ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP
         // As the blocks are added to a SLIST and used from there during concurrent sweep, the expectFull assertion doesn't hold anymore.
         // We could do some work to make this work again but there may be perf hit and it may be fragile.
         if (!CONFIG_FLAG_RELEASE(EnableConcurrentSweepAlloc))
 #endif
         {
             Assert(*expectFull && !*expectDispose);
             Assert(heapBlock->HasFreeObject());
             Assert(!heapBlock->HasAnyDisposeObjects());
         }
     }
     else if (*expectDispose)
     {
         Assert(heapBlock->IsAnyFinalizableBlock() && heapBlock->template AsFinalizableBlock<typename TBlockType::HeapBlockAttributes>()->IsPendingDispose());
         Assert(heapBlock->HasAnyDisposeObjects());
     }
     else
     {
         Assert(!heapBlock->HasAnyDisposeObjects());

         // ExpectFull is a bit of a misnomer if the list in question is the heap block list. It's there to check
         // of the heap block in question is before the nextAllocableBlockHead or not. This is to ensure that
         // blocks before nextAllocableBlockHead that are not being bump allocated from must be considered "full".
         // However, the exception is if this is the only heap block in this bucket, in which case nextAllocableBlockHead
         // would be null
 #if ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP
         // As the blocks are added to the SLIST and used from there during concurrent sweep, the expectFull assertion doesn't hold anymore.
         if (!CONFIG_FLAG_RELEASE(EnableConcurrentSweepAlloc))
 #endif
         {
             Assert(*expectFull == (!heapBlock->HasFreeObject() || heapBlock->IsInAllocator()) || nextAllocableBlockHead == nullptr);
         }
     }
 }

 template <typename TBlockType>
 void
 HeapBucketT<TBlockType>::VerifyBlockConsistencyInList(TBlockType * heapBlock, RecyclerVerifyListConsistencyData const& recyclerSweep, SweepState state)
 {
     bool expectFull = false;
     bool expectDispose = false;

     if (TBlockType::HeapBlockAttributes::IsSmallBlock)
     {
         expectFull = recyclerSweep.smallBlockVerifyListConsistencyData.expectFull;
         expectDispose = recyclerSweep.smallBlockVerifyListConsistencyData.expectDispose;
     }
     else if (TBlockType::HeapBlockAttributes::IsMediumBlock)
     {
         expectFull = recyclerSweep.mediumBlockVerifyListConsistencyData.expectFull;
         expectDispose = recyclerSweep.mediumBlockVerifyListConsistencyData.expectDispose;
     }
     else
     {
         Assert(false);
     }

     if (heapBlock->IsClearedFromAllocator())
     {
         // this function is called during sweep and we are recreating the heap block list
         // which would make all the block to be in it's rightful place
         heapBlock->SetIsClearedFromAllocator(false);

         Assert(SweepStateFull != state);
     }
     else
     {
         // You can still be full only if you are full before.
         Assert(expectFull || SweepStateFull != state);
     }

     // If you were pending dispose before, you can only be pending dispose after
     Assert(!expectDispose || SweepStatePendingDispose == state);
 }
 #endif // DBG

 #if ENABLE_PARTIAL_GC
 template <typename TBlockType>
 bool
 HeapBucketT<TBlockType>::DoQueuePendingSweep(Recycler * recycler)
 {
     return IsNormalBucket && recycler->inPartialCollectMode;
 }

 template <typename TBlockType>
 bool
 HeapBucketT<TBlockType>::DoPartialReuseSweep(Recycler * recycler)
 {
     // With leaf, we don't need to do a partial sweep
     // WriteBarrier-TODO: We shouldn't need to do this for write barrier heap buckets either
     return !IsLeafBucket && recycler->inPartialCollectMode;
 }
 #endif

 template <typename TBlockType>
 void
 HeapBucketT<TBlockType>::SweepHeapBlockList(RecyclerSweep& recyclerSweep, TBlockType * heapBlockList, bool allocable)
 {
 #if DBG
     if (TBlockType::HeapBlockAttributes::IsSmallBlock)
     {
         Assert(recyclerSweep.smallBlockVerifyListConsistencyData.hasSetupVerifyListConsistencyData);
         recyclerSweep.smallBlockVerifyListConsistencyData.hasSetupVerifyListConsistencyData = false;
     }
     else if (TBlockType::HeapBlockAttributes::IsMediumBlock)
     {
         Assert(recyclerSweep.mediumBlockVerifyListConsistencyData.hasSetupVerifyListConsistencyData);
         recyclerSweep.mediumBlockVerifyListConsistencyData.hasSetupVerifyListConsistencyData = false;
     }
     else
     {
         Assert(false);
     }
 #endif

     Recycler * recycler = recyclerSweep.GetRecycler();

     // Whether we run in thread or background thread, we want to queue up pending sweep
     // only if we are doing partial GC so we can calculate the heuristics before
     // determinate we want to fully sweep the block or partially sweep the block

 #if ENABLE_PARTIAL_GC
     // CONCURRENT-TODO: Add a mode where we can do in thread sweep, and concurrent partial sweep?
     bool const queuePendingSweep = this->DoQueuePendingSweep(recycler);
 #else
     bool const queuePendingSweep = false;
 #endif

 #if ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP
     Assert(this->IsAllocationStopped() || this->AllocationsStartedDuringConcurrentSweep());
 #else
     Assert(this->IsAllocationStopped());
 #endif

     HeapBlockList::ForEachEditing(heapBlockList, [=, &recyclerSweep](TBlockType * heapBlock)
     {
         // The whole list need to be consistent
         DebugOnly(VerifyBlockConsistencyInList(heapBlock, recyclerSweep));

 #ifdef RECYCLER_TRACE
         recyclerSweep.GetRecycler()->PrintBlockStatus(this, heapBlock, _u("[**1**] starting Sweep Pass1."));
 #endif
         SweepState state = heapBlock->Sweep(recyclerSweep, queuePendingSweep, allocable);

         DebugOnly(VerifyBlockConsistencyInList(heapBlock, recyclerSweep, state));

         switch (state)
         {
 #if ENABLE_CONCURRENT_GC
         case SweepStatePendingSweep:
         {
             Assert(IsNormalBucket);
             // blocks that have swept object. Queue up the block for concurrent sweep.
             Assert(queuePendingSweep);
             TBlockType *& pendingSweepList = recyclerSweep.GetPendingSweepBlockList(this);
             DebugOnly(this->AssertCheckHeapBlockNotInAnyList(heapBlock));
             AssertMsg(!HeapBlockList::Contains(heapBlock, pendingSweepList), "The heap block already exists in the pendingSweepList.");

             heapBlock->SetNextBlock(pendingSweepList);
             pendingSweepList = heapBlock;
 #ifdef RECYCLER_TRACE
             recyclerSweep.GetRecycler()->PrintBlockStatus(this, heapBlock, _u("[**2**] finished Sweep Pass1, heapblock added to pendingSweepList."));
 #endif
 #if ENABLE_PARTIAL_GC
             recyclerSweep.GetManager()->NotifyAllocableObjects(heapBlock);
 #endif
             break;
         }
 #endif
         case SweepStatePendingDispose:
         {
             Assert(!recyclerSweep.IsBackground());
 #ifdef RECYCLER_WRITE_BARRIER
             Assert(IsFinalizableBucket || IsFinalizableWriteBarrierBucket);
 #else
             Assert(IsFinalizableBucket);
 #endif
 #if ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP
             if (CONFIG_FLAG_RELEASE(EnableConcurrentSweepAlloc))
             {
                 AssertMsg(!heapBlock->isPendingConcurrentSweepPrep, "Finalizable blocks don't support allocations during concurrent sweep.");
             }
 #endif

             DebugOnly(heapBlock->template AsFinalizableBlock<typename TBlockType::HeapBlockAttributes>()->SetIsPendingDispose());

             // These are the blocks that have swept finalizable object

             // We already transferred the non finalizable swept objects when we are not doing
             // concurrent collection, so we only need to queue up the blocks that have
             // finalizable objects, so that we can go through and call the dispose, and then
             // transfer the finalizable object back to the free list.
             SmallFinalizableHeapBucketT<typename TBlockType::HeapBlockAttributes> * finalizableHeapBucket = (SmallFinalizableHeapBucketT<typename TBlockType::HeapBlockAttributes>*)this;
             DebugOnly(this->AssertCheckHeapBlockNotInAnyList(heapBlock));
             //AssertMsg(!HeapBlockList::Contains(heapBlock, finalizableHeapBucket->pendingDisposeList), "The heap block already exists in the pendingDisposeList.");
             heapBlock->template AsFinalizableBlock<typename TBlockType::HeapBlockAttributes>()->SetNextBlock(finalizableHeapBucket->pendingDisposeList);
             finalizableHeapBucket->pendingDisposeList = heapBlock->template AsFinalizableBlock<typename TBlockType::HeapBlockAttributes>();
             Assert(!this->heapInfo->hasPendingTransferDisposedObjects);
             recycler->hasDisposableObject = true;
 #ifdef RECYCLER_TRACE
             recyclerSweep.GetRecycler()->PrintBlockStatus(this, heapBlock, _u("[**3**] finished Sweep Pass1, heapblock added to pendingDisposeList."));
 #endif
             break;
         }
         case SweepStateSwept:
         {
             Assert(this->nextAllocableBlockHead == nullptr);
             Assert(heapBlock->HasFreeObject());
             DebugOnly(this->AssertCheckHeapBlockNotInAnyList(heapBlock));

 #if ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP && SUPPORT_WIN32_SLIST
             if (this->AllocationsStartedDuringConcurrentSweep())
             {
                 Assert(!this->IsAnyFinalizableBucket());
                 Assert(!heapBlock->isPendingConcurrentSweepPrep);
                 bool blockAddedToSList = HeapBucketT<TBlockType>::PushHeapBlockToSList(this->allocableHeapBlockListHead, heapBlock);

                 // If we encountered OOM while pushing the heapBlock to the SLIST we must add it to the heapBlockList so we don't lose track of it.
                 if (!blockAddedToSList)
                 {
                     //TODO: akatti: We should handle this gracefully and try to recover from this state.
                     AssertOrFailFastMsg(false, "OOM while adding a heap block to the SLIST during concurrent sweep.");
                 }
 #ifdef RECYCLER_TRACE
                 else
                 {
                     this->GetRecycler()->PrintBlockStatus(this, heapBlock, _u("[**4**] swept and added to SLIST allocableHeapBlockListHead during Pass1."));
                 }
 #endif
             }
             else
 #endif
             {
 #if ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP
                 AssertMsg(!this->AllowAllocationsDuringConcurrentSweep(), "Why are allocations not started during concurrent sweep?");
 #endif
                 heapBlock->SetNextBlock(this->heapBlockList);
                 this->heapBlockList = heapBlock;
             }

 #ifdef RECYCLER_TRACE
             recyclerSweep.GetRecycler()->PrintBlockStatus(this, heapBlock, _u("[**6**] finished Sweep Pass1, heapblock added to heapBlockList."));
 #endif
 #if ENABLE_PARTIAL_GC
             recyclerSweep.GetManager()->NotifyAllocableObjects(heapBlock);
 #endif
             break;
         }
         case SweepStateFull:
         {
             Assert(!heapBlock->HasFreeObject());
             DebugOnly(this->AssertCheckHeapBlockNotInAnyList(heapBlock));
             heapBlock->SetNextBlock(this->fullBlockList);
             this->fullBlockList = heapBlock;
 #ifdef RECYCLER_TRACE
             recyclerSweep.GetRecycler()->PrintBlockStatus(this, heapBlock, _u("[**7**] finished Sweep Pass1, heapblock FULL added to fullBlockList."));
 #endif
             break;
         }
         case SweepStateEmpty:
         {
             // the block is empty, just free them
 #ifdef RECYCLER_MEMORY_VERIFY
             // Let's verify it before we free it
             if (recycler->VerifyEnabled())
             {
                 heapBlock->Verify();
             }
 #endif

             RECYCLER_STATS_INC(recycler, numEmptySmallBlocks[heapBlock->GetHeapBlockType()]);

 #if ENABLE_CONCURRENT_GC
             // CONCURRENT-TODO: Finalizable block never have background == true and always be processed
             // in thread, so it will not queue up the pages even if we are doing concurrent GC
             DebugOnly(this->AssertCheckHeapBlockNotInAnyList(heapBlock));
 #if ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP
             if (CONFIG_FLAG_RELEASE(EnableConcurrentSweepAlloc))
             {
                 AssertMsg(heapBlock->objectsAllocatedDuringConcurrentSweepCount == 0, "We allocated to this block during concurrent sweep; it's not EMPTY anymore, it should NOT be freed or queued as EMPTY.");
             }
 #endif

             if (recyclerSweep.IsBackground())
             {
 #ifdef RECYCLER_WRITE_BARRIER
                 Assert(!(IsFinalizableBucket || IsFinalizableWriteBarrierBucket));
 #else
                 Assert(!IsFinalizableBucket);
 #endif
                 // CONCURRENT-TODO: We will zero heap block even if the number free page pool exceed
                 // the maximum and will get decommitted anyway
                 recyclerSweep.template QueueEmptyHeapBlock<TBlockType>(this, heapBlock);
                 RECYCLER_STATS_INC(recycler, numZeroedOutSmallBlocks);
 #ifdef RECYCLER_TRACE
                 recyclerSweep.GetRecycler()->PrintBlockStatus(this, heapBlock, _u("[**8**] finished Sweep Pass1, heapblock EMPTY added to pendingEmptyBlockList."));
 #endif
             }
             else
 #endif
             {
                 // Just free the page in thread (and zero the page)
                 heapBlock->ReleasePagesSweep(recycler);
                 FreeHeapBlock(heapBlock);
 #if defined(RECYCLER_SLOW_CHECK_ENABLED) || ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP
                 this->heapBlockCount--;
 #endif
 #ifdef RECYCLER_TRACE
                 recyclerSweep.GetRecycler()->PrintBlockStatus(this, heapBlock, _u("[**9**] finished Sweep Pass1, heapblock EMPTY, was FREED in-thread."));
 #endif
             }

             break;
         }
         }
     });
 }

 template <typename TBlockType>
 void
 HeapBucketT<TBlockType>::SweepBucket(RecyclerSweep& recyclerSweep)
 {
     DebugOnly(TBlockType * savedNextAllocableBlockHead);
     RECYCLER_SLOW_CHECK(this->VerifyHeapBlockCount(recyclerSweep.IsBackground()));
 #if ENABLE_CONCURRENT_GC
     if (recyclerSweep.GetManager()->HasSetupBackgroundSweep())
     {
         // SetupBackgroundSweep set nextAllocableBlockHead to null already
         Assert(IsAllocationStopped());
         DebugOnly(savedNextAllocableBlockHead = recyclerSweep.GetSavedNextAllocableBlockHead(this));
     }
     else
 #endif
     {
         Assert(AllocatorsAreEmpty());
         DebugOnly(savedNextAllocableBlockHead = this->nextAllocableBlockHead);
         this->StopAllocationBeforeSweep();
     }

     // We just started sweeping.  These pending lists should be empty
 #if ENABLE_CONCURRENT_GC
     Assert(recyclerSweep.GetPendingSweepBlockList(this) == nullptr);
 #else
     Assert(!recyclerSweep.IsBackground());
 #endif

 #if ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP && SUPPORT_WIN32_SLIST
     if (CONFIG_FLAG_RELEASE(EnableConcurrentSweepAlloc) && this->sweepableHeapBlockList != nullptr)
     {
         Assert(!this->IsAnyFinalizableBucket());

 #if DBG || defined(RECYCLER_SLOW_CHECK_ENABLED)
         // This lock is needed only in the debug mode while we verify block counts. Not needed otherwise, as this list is never accessed concurrently.
         // Items are added to it by the allocator when allocations are allowed during concurrent sweep. The list is drained during the next sweep while
         // allocation are stopped.
         debugSweepableHeapBlockListLock.Enter();
 #endif
         // Return the blocks we may have allocated from during the previous concurrent sweep back to the fullBlockList.
         // We need to rebuild the free bit vectors for these blocks.
         HeapBlockList::ForEachEditing(this->sweepableHeapBlockList, [this](TBlockType * heapBlock)
         {
             heapBlock->BuildFreeBitVector();

             AssertMsg(!HeapBlockList::Contains(heapBlock, heapBlockList), "The heap block already exists in the heapBlockList.");
             AssertMsg(!HeapBlockList::Contains(heapBlock, fullBlockList), "The heap block already exists in the fullBlockList.");
             AssertMsg(!HeapBlockList::Contains(heapBlock, emptyBlockList), "The heap block already exists in the emptyBlockList.");

             heapBlock->SetNextBlock(this->fullBlockList);
             this->fullBlockList = heapBlock;
         });
         this->sweepableHeapBlockList = nullptr;
 #if DBG || defined(RECYCLER_SLOW_CHECK_ENABLED)
         debugSweepableHeapBlockListLock.Leave();
 #endif
     }
 #endif

 #if DBG
     if (TBlockType::HeapBlockAttributes::IsSmallBlock)
     {
         recyclerSweep.SetupVerifyListConsistencyDataForSmallBlock((SmallHeapBlock*) savedNextAllocableBlockHead, true, false);
     }
     else if (TBlockType::HeapBlockAttributes::IsMediumBlock)
     {
         recyclerSweep.SetupVerifyListConsistencyDataForMediumBlock((MediumHeapBlock*) savedNextAllocableBlockHead, true, false);
     }
     else
     {
         Assert(false);
     }
 #endif

     // Move the list locally.  We will relink them during sweep
     TBlockType * currentFullBlockList = fullBlockList;
     TBlockType * currentHeapBlockList = heapBlockList;
     this->heapBlockList = nullptr;
     this->fullBlockList = nullptr;

 #if ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP
     // In order to allow allocations during sweep (Pass-1) we will set aside blocks after nextAllocableBlockHead (excluding) and allow
     // allocations to these blocks as we know that these blocks are not full yet. These will need to be swept later though before starting Pass-2
     // of the sweep.
     this->PrepareForAllocationsDuringConcurrentSweep(currentHeapBlockList);
 #endif

     this->SweepHeapBlockList(recyclerSweep, currentHeapBlockList, true);

 #if DBG
     if (TBlockType::HeapBlockAttributes::IsSmallBlock)
     {
         recyclerSweep.SetupVerifyListConsistencyDataForSmallBlock(nullptr, true, false);
     }
     else if (TBlockType::HeapBlockAttributes::IsMediumBlock)
     {
         recyclerSweep.SetupVerifyListConsistencyDataForMediumBlock(nullptr, true, false);
     }
     else
     {
         Assert(false);
     }
 #endif

     this->SweepHeapBlockList(recyclerSweep, currentFullBlockList, false);

     // We shouldn't have allocate from any block yet
     Assert(this->nextAllocableBlockHead == nullptr);
 }

 template <typename TBlockType>
 bool
 HeapBucketT<TBlockType>::AllowAllocationsDuringConcurrentSweep()
 {
 #if ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP
     Recycler * recycler = this->GetRecycler();
     if (!CONFIG_FLAG_RELEASE(EnableConcurrentSweepAlloc) || !recycler->AllowAllocationsDuringConcurrentSweep() || !this->concurrentSweepAllocationsThresholdExceeded)
     {
         return false;
     }

 #if ENABLE_PARTIAL_GC
     bool isPartialGC = (recycler->recyclerSweepManager != nullptr) && recycler->recyclerSweepManager->InPartialCollect();
 #else
     bool isPartialGC = false;
 #endif

     // Allocations are allowed during concurrent sweep for small non-finalizable buckets while not doing a Partial GC.
     return (recycler->IsConcurrentSweepSetupState() || recycler->InConcurrentSweep()) && !this->IsAnyFinalizableBucket() && !isPartialGC;
 #else
     return false;
 #endif
 }

 template <typename TBlockType>
 void
 HeapBucketT<TBlockType>::StopAllocationBeforeSweep()
 {
 #if ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP
     this->allocationsStartedDuringConcurrentSweep = false;
 #if SUPPORT_WIN32_SLIST
     this->lastKnownNextAllocableBlockHead = this->nextAllocableBlockHead;
 #endif
 #endif

     Assert(!this->IsAllocationStopped());
     this->isAllocationStopped = true;
     this->nextAllocableBlockHead = nullptr;
 }

 template <typename TBlockType>
 void
 HeapBucketT<TBlockType>::StartAllocationAfterSweep()
 {
     Assert(this->IsAllocationStopped());
     this->isAllocationStopped = false;
     this->nextAllocableBlockHead = this->heapBlockList;
 }

 #if ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP
 template <typename TBlockType>
 void
 HeapBucketT<TBlockType>::StartAllocationDuringConcurrentSweep()
 {
     Recycler * recycler = this->GetRecycler();
     Assert(!recycler->recyclerSweepManager->InPartialCollect());
     Assert(!this->IsAnyFinalizableBucket());

     Assert(this->IsAllocationStopped());
     this->isAllocationStopped = false;
     Assert(!this->allocationsStartedDuringConcurrentSweep);
     this->allocationsStartedDuringConcurrentSweep = true;

 #if SUPPORT_WIN32_SLIST
     // When allocations are allowed during concurrent sweep we set nextAllocableBlockHead to NULL as the allocator will pick heap blocks from the
     // interlocked SLIST. During that time, the heap block at the top of the SLIST is always the nextAllocableBlockHead.
     this->nextAllocableBlockHead = nullptr;
     this->lastKnownNextAllocableBlockHead = nullptr;
 #endif
 }

 template <typename TBlockType>
 void
 HeapBucketT<TBlockType>::ResumeNormalAllocationAfterConcurrentSweep(TBlockType * newNextAllocableBlockHead)
 {
     this->allocationsStartedDuringConcurrentSweep = false;
     this->isAllocationStopped = false;
     // If the newNextAllocableBlockHead is NULL at this point that means we have exhausted usable blocks and will have to allocate a new block the next time.
     this->nextAllocableBlockHead = newNextAllocableBlockHead;
 }

 /*//////////////////////////////////////////////////////////////////////////////////////////////////////
 If allocations are to be allowed to existing heap blocks during concurrent sweep, we set aside a few
 heap blocks from the heapBlockList prior to beginning sweep. However, we eed to then go back and make
 sure these blocks also swept before this sweep finishes. In order to do this we clearly define concurrent
 sweep having 2 passes now. These passes existed before but were not distiguished as they would always start
 and finish in one go on the background thread. However, whenever allocations are allowed during concurrent
 sweep; the concurrent sweep will start Pass1 on the background thread, wait to finish Pass1 of the blocks
 we set aside to allocate from on the main thread and then go back to finish Pass2 for all heap blocks on
 the background thread. Note that, due to this need to finish Pass1 on the foreground thread the overall
 background sweep will now appear to take longer whenever we chose to do such a two-pass sweep.

 The sequence of things we do to allow allocations during concurrent sweep is described below:
 1. At the beginning of concurrrent sweep we decide if we will benefit from allowing allocations during concurrent
 sweep for any of the buckets. If there is at-least one bucket for which we think we will benefit we will turn on
 allocations during concurrent sweep. Once turned on we will attempt to enable allocations during concurrent sweep
 for all supported buckets (i.e. small/medium, normal/leaf, non-finalizable buckets.write barrrier bickets are supported
 as well.).
 2. If allocations are turned on during concurrent sweep, we will see if there are any allocable blocks in the
 heapBlockList after the nextAllocableBlockHead. If we find any such blocks, we move them to a SLIST that the
 allocator can pick these blocks from during sweep.
 3. CollectionStateConcurrentSweepPass1: We will finish Pass1 of the sweep for all the remaining blocks (other than the
 ones we put in the SLIST in step 2 above) This will generally happen on the background thread unless we are forcing
 in-thread sweep. This state is now specifically identified as CollectionStateConcurrentSweepPass1.
 4. CollectionStateConcurrentSweepPass1Wait: At this point we need to wait for all the blocks that we put in the SLIST
 to also finish the Pass1 of the sweep. This needs to happen on the foreground thread so we prevent the allocator from
 picking up the blocks from SLIST while we do this. This state is now identified as CollectionStateConcurrentSweepPass1Wait.
 5. CollectionStateConcurrentSweepPass2: At this point we will do the actual sweeping of all the blocks that are not yet swept,
 for eaxample, any blocks that were put onto the pendingSweepList. As these blocks get swept we keep adding them to the
 SLIST again to allow allocators to allocate from them as soon as they are swept.
 6. Before the Pass2 can finish we can call this concurrent sweep done we need to move all the blocks off of the SLIST so
 that normal allocations can begin after the sweep. This is the last step of the concurrent sweep.
 //////////////////////////////////////////////////////////////////////////////////////////////////////*/
 template<typename TBlockType>
 void
 HeapBucketT<TBlockType>::PrepareForAllocationsDuringConcurrentSweep(TBlockType * &currentHeapBlockList)
 {
 #if SUPPORT_WIN32_SLIST
     if (this->AllowAllocationsDuringConcurrentSweep())
     {
         this->EnsureAllocableHeapBlockList();

         Assert(!this->IsAnyFinalizableBucket());
         Assert(HeapBucketT<TBlockType>::QueryDepthInterlockedSList(this->allocableHeapBlockListHead) == 0);
         Assert(HeapBlockList::Count(this->sweepableHeapBlockList) == 0);

         TBlockType* startingNextAllocableBlockHead = this->lastKnownNextAllocableBlockHead;
         bool allocationsStarted = false;
         if (startingNextAllocableBlockHead != nullptr)
         {
             // To avoid a race condition between the allocator attempting to allocate from the lastKnownNextAllocableBlockHead and this code
             // where we are adding it to the SLIST we skip the lastKnownNextAllocableBlockHead and pick up the next block to start with.
             // Allocations should have stopped by then; so allocator shouldn't pick up the lastKnownNextAllocableBlockHead->Next block.
             TBlockType* savedNextAllocableBlockHead = startingNextAllocableBlockHead->GetNextBlock();
             startingNextAllocableBlockHead->SetNextBlock(nullptr);
             startingNextAllocableBlockHead = savedNextAllocableBlockHead;

             if (startingNextAllocableBlockHead != nullptr)
             {
                 // The allocable blocks, if any are available, will now be added to the allocable blocks SLIST at this time; start allocations now.
                 this->StartAllocationDuringConcurrentSweep();
                 allocationsStarted = true;

                 HeapBlockList::ForEachEditing(startingNextAllocableBlockHead, [this, &allocationsStarted](TBlockType * heapBlock)
                 {
                     // This heap block is NOT ready to be swept concurrently as it hasn't yet been through sweep prep (i.e. Pass1 of sweep).
                     heapBlock->isPendingConcurrentSweepPrep = true;
                     DebugOnly(this->AssertCheckHeapBlockNotInAnyList(heapBlock));
                     bool blockAddedToSList = HeapBucketT<TBlockType>::PushHeapBlockToSList(this->allocableHeapBlockListHead, heapBlock);

                     // If we encountered OOM while pushing the heapBlock to the SLIST we must add it to the heapBlockList so we don't lose track of it.
                     if (!blockAddedToSList)
                     {
                         //TODO: akatti: We should handle this gracefully and try to recover from this state.
                         AssertOrFailFastMsg(false, "OOM while adding a heap block to the SLIST during concurrent sweep.");
                     }
                     else
                     {
 #ifdef RECYCLER_TRACE
                         this->GetRecycler()->PrintBlockStatus(this, heapBlock, _u("[**5**] added to SLIST before Pass1."));
 #endif
                     }
                 });
 #ifdef RECYCLER_TRACE
                 if (this->GetRecycler()->GetRecyclerFlagsTable().Trace.IsEnabled(Js::ConcurrentSweepPhase) && CONFIG_FLAG_RELEASE(Verbose))
                 {
                     size_t currentHeapBlockCount = QueryDepthInterlockedSList(allocableHeapBlockListHead);
                     CollectionState collectionState = this->GetRecycler()->collectionState;
                     Output::Print(_u("[GC #%d] [HeapBucket 0x%p] Starting allocations during  concurrent sweep with %d blocks. [CollectionState: %d] \n"), this->GetRecycler()->collectionCount, this, currentHeapBlockCount, collectionState);
                     Output::Print(_u("[GC #%d] [HeapBucket 0x%p] The heapBlockList has %d blocks. Total heapBlockCount is %d.\n\n"), this->GetRecycler()->collectionCount, this, HeapBlockList::Count(this->heapBlockList), this->heapBlockCount);
                 }
 #endif
             }
         }

         if (!allocationsStarted)
         {
             // If we didn't start allocations yet, start them now in anticipation of blocks becoming available later as blocks complete sweep.
             this->StartAllocationDuringConcurrentSweep();
             allocationsStarted = true;
         }

         Assert(!this->IsAllocationStopped());
     }
 #endif
 }
 #endif

 template <typename TBlockType>
 bool
 HeapBucketT<TBlockType>::IsAllocationStopped() const
 {
     if (this->isAllocationStopped)
     {
         Assert(this->nextAllocableBlockHead == nullptr);
         return true;
     }
     return false;
 }

 template <typename TBlockType>
 uint
 HeapBucketT<TBlockType>::Rescan(Recycler * recycler, RescanFlags flags)
 {
 #if ENABLE_CONCURRENT_GC
     RECYCLER_SLOW_CHECK(this->VerifyHeapBlockCount(!!recycler->IsConcurrentMarkState()));
 #else
     RECYCLER_SLOW_CHECK(this->VerifyHeapBlockCount(false /* background */));
 #endif

 #if ENABLE_CONCURRENT_GC
     // If we do the final rescan concurrently, the main thread will prepare for sweep concurrently
     // If we do rescan in thread, we will need to prepare sweep here.
     // However, if we are in the rescan for OOM, we have already done it, so no need to do it again
     if (!recycler->IsConcurrentMarkState() && !recycler->inEndMarkOnLowMemory)
     {
         this->PrepareSweep();
     }
 #endif

     // By default heap bucket doesn't rescan anything
     return 0;
 }

 #if ENABLE_CONCURRENT_GC
 template <typename TBlockType>
 void
 HeapBucketT<TBlockType>::MergeNewHeapBlock(TBlockType * heapBlock)
 {
     Assert(heapBlock->GetObjectSize() == this->sizeCat);
     heapBlock->SetNextBlock(this->heapBlockList);
     this->heapBlockList = heapBlock;
 #if defined(RECYCLER_SLOW_CHECK_ENABLED) || ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP
     ::InterlockedDecrement(&this->newHeapBlockCount);
     this->heapBlockCount++;
 #endif
 }

 template <typename TBlockType>
 void
 HeapBucketT<TBlockType>::SetupBackgroundSweep(RecyclerSweep& recyclerSweep)
 {
     // Don't allocate from existing block temporary when concurrent sweeping

     // Currently Rescan clear allocators, if we remove the uncollectedAllocBytes there, we can
     // avoid it there and do it here.
     Assert(this->AllocatorsAreEmpty());

     DebugOnly(recyclerSweep.SaveNextAllocableBlockHead(this));
     Assert(recyclerSweep.GetPendingSweepBlockList(this) == nullptr);

     this->StopAllocationBeforeSweep();
 }
 #endif

 #if ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP
 template <typename TBlockType>
 void
 HeapBucketT<TBlockType>::FinishConcurrentSweepPass1(RecyclerSweep& recyclerSweep)
 {
     if (this->concurrentSweepAllocationsThresholdExceeded)
     {
         AssertMsg(this->AllowAllocationsDuringConcurrentSweep(), "Why are we in two pass concurrent sweep?");
         Assert(!this->IsAnyFinalizableBucket());

         // Rebuild the free bit vectors for the blocks we allocated from during concurrent sweep.
         TBlockType * currentPendingSweepPrepHeapBlockList = nullptr;
         TBlockType * currentSweepableHeapBlockList = this->sweepableHeapBlockList;
         this->sweepableHeapBlockList = nullptr;

         HeapBlockList::ForEachEditing(currentSweepableHeapBlockList, [this, &currentPendingSweepPrepHeapBlockList](TBlockType * heapBlock)
         {
             if (heapBlock->isPendingConcurrentSweepPrep)
             {
                 ushort previousFreeCount = heapBlock->freeCount;
                 heapBlock->BuildFreeBitVector();

 #if ENABLE_PARTIAL_GC
                 heapBlock->oldFreeCount = heapBlock->lastFreeCount = heapBlock->freeCount;
 #else
                 heapBlock->lastFreeCount = heapBlock->freeCount;
 #endif
                 ushort newAllocatedObjects = previousFreeCount - heapBlock->freeCount;
                 AssertMsg(newAllocatedObjects == heapBlock->objectsMarkedDuringSweep, "The counts of objects allocated during sweep should match the objects marked during sweep.");
 #if DBG || defined(RECYCLER_SLOW_CHECK_ENABLED)
                 heapBlock->objectsAllocatedDuringConcurrentSweepCount = newAllocatedObjects;
 #endif

                 ushort currentMarkCount = (ushort)heapBlock->GetMarkCountForSweep();
                 heapBlock->markCount = currentMarkCount;
 #if DBG
                 heapBlock->GetRecycler()->heapBlockMap.SetPageMarkCount(heapBlock->GetAddress(), currentMarkCount);
 #endif
 #ifdef RECYCLER_TRACE
                 heapBlock->GetRecycler()->PrintBlockStatus(this, heapBlock, _u("[**13**] ending sweep Pass1, rebuilt free bit vector and set page mark count to match."));
 #endif

                 heapBlock->SetNextBlock(currentPendingSweepPrepHeapBlockList);
                 currentPendingSweepPrepHeapBlockList = heapBlock;
             }
             else
             {
                 heapBlock->SetNextBlock(this->sweepableHeapBlockList);
                 this->sweepableHeapBlockList = heapBlock;
             }
         });

 #if DBG || defined(RECYCLER_SLOW_CHECK_ENABLED)
         // This lock is needed only in the debug mode while we verify block counts. Not needed otherwise, as this list is never accessed concurrently.
         // Items are added to it by the allocator when allocations are allowed during concurrent sweep. The list is drained during the next sweep while
         // allocation are stopped.
         debugSweepableHeapBlockListLock.Enter();
 #endif

         // Pull the blocks from the allocable SLIST that we didn't use. We need to finish the Pass-1 sweep of these blocks too.
         TBlockType * heapBlock = PopHeapBlockFromSList(this->allocableHeapBlockListHead);
         while (heapBlock != nullptr)
         {
             DebugOnly(this->AssertCheckHeapBlockNotInAnyList(heapBlock));
             if (heapBlock->isPendingConcurrentSweepPrep)
             {
 #ifdef RECYCLER_TRACE
                 heapBlock->GetRecycler()->PrintBlockStatus(this, heapBlock, _u("[**19**] ending sweep Pass1, removed from SLIST."));
 #endif
                 heapBlock->SetNextBlock(currentPendingSweepPrepHeapBlockList);
                 currentPendingSweepPrepHeapBlockList = heapBlock;
             }
             else
             {
 #ifdef RECYCLER_TRACE
                 heapBlock->GetRecycler()->PrintBlockStatus(this, heapBlock, _u("[**23**] ending sweep Pass1, removed from SLIST and added to sweepableHeapBlockList."));
 #endif
                 // Already swept, put it back to the sweepableHeapBlockList list; so it can be processed later.
                 heapBlock->SetNextBlock(this->sweepableHeapBlockList);
                 this->sweepableHeapBlockList = heapBlock;
             }

             heapBlock = PopHeapBlockFromSList(this->allocableHeapBlockListHead);
         }
         Assert(QueryDepthInterlockedSList(this->allocableHeapBlockListHead) == 0);

 #if DBG || defined(RECYCLER_SLOW_CHECK_ENABLED)
         debugSweepableHeapBlockListLock.Leave();
 #endif

 #if DBG
         if (TBlockType::HeapBlockAttributes::IsSmallBlock)
         {
             recyclerSweep.SetupVerifyListConsistencyDataForSmallBlock(nullptr, true, false);
         }
         else if (TBlockType::HeapBlockAttributes::IsMediumBlock)
         {
             recyclerSweep.SetupVerifyListConsistencyDataForMediumBlock(nullptr, true, false);
         }
         else
         {
             Assert(false);
         }
 #endif

         // Start allocations now as we may start adding blocks to the SLIST during Pass1 sweep below.
         this->StartAllocationDuringConcurrentSweep();
         this->SweepHeapBlockList(recyclerSweep, currentPendingSweepPrepHeapBlockList, true /*allocable*/);
     }
 }

 template <typename TBlockType>
 void
 HeapBucketT<TBlockType>::EnsureAllocableHeapBlockList()
 {
 #if ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP && SUPPORT_WIN32_SLIST
     if (CONFIG_FLAG_RELEASE(EnableConcurrentSweepAlloc))
     {
         if (allocableHeapBlockListHead == nullptr)
         {
             allocableHeapBlockListHead = ((PSLIST_HEADER)_aligned_malloc(sizeof(SLIST_HEADER), MEMORY_ALLOCATION_ALIGNMENT));

             if (allocableHeapBlockListHead == nullptr)
             {
                 this->heapInfo->recycler->OutOfMemory();
             }
             else
             {
                 ::InitializeSListHead(allocableHeapBlockListHead);
             }
         }
     }
 #endif
 }

 template <typename TBlockType>
 void
 HeapBucketT<TBlockType>::FinishSweepPrep(RecyclerSweep& recyclerSweep)
 {
     if (this->AllocationsStartedDuringConcurrentSweep())
     {
         AssertMsg(this->AllowAllocationsDuringConcurrentSweep(), "Why are allocations started during concurrent sweep, if not allowed?");
         Assert(!this->IsAnyFinalizableBucket());

         this->StopAllocationBeforeSweep();
         this->ClearAllocators();
     }
 }

 template <typename TBlockType>
 void
 HeapBucketT<TBlockType>::FinishConcurrentSweep()
 {
     if (this->AllocationsStartedDuringConcurrentSweep())
     {
 #if SUPPORT_WIN32_SLIST
         Assert(!this->IsAnyFinalizableBucket());
         Assert(this->allocableHeapBlockListHead != nullptr);

 #ifdef RECYCLER_TRACE
         if (this->GetRecycler()->GetRecyclerFlagsTable().Trace.IsEnabled(Js::ConcurrentSweepPhase) && CONFIG_FLAG_RELEASE(Verbose))
         {
             CollectionState collectionState = this->GetRecycler()->collectionState;
             Output::Print(_u("[GC #%d] [HeapBucket 0x%p] starting FinishConcurrentSweep [CollectionState: %d] \n"), this->GetRecycler()->collectionCount, this, collectionState);
         }
 #endif

         TBlockType * newNextAllocableBlockHead = nullptr;
         // Put the blocks from the allocable SLIST into the heapBlockList.
         TBlockType * heapBlock = PopHeapBlockFromSList(this->allocableHeapBlockListHead);
         while (heapBlock != nullptr)
         {
             DebugOnly(this->AssertCheckHeapBlockNotInAnyList(heapBlock));
             AssertMsg(!heapBlock->isPendingConcurrentSweepPrep, "The blocks in the SLIST at this time should NOT have sweep prep i.e. sweep-Pass1 pending.");
             newNextAllocableBlockHead = heapBlock;
             heapBlock->SetNextBlock(this->heapBlockList);
             this->heapBlockList = heapBlock;
 #ifdef RECYCLER_TRACE
             this->GetRecycler()->PrintBlockStatus(this, heapBlock, _u("[**40**] finished FinishConcurrentSweep, heapblock removed from SLIST and added to heapBlockList."));
 #endif
             heapBlock = PopHeapBlockFromSList(this->allocableHeapBlockListHead);
         }

         Assert(QueryDepthInterlockedSList(this->allocableHeapBlockListHead) == 0);

         this->ResumeNormalAllocationAfterConcurrentSweep(newNextAllocableBlockHead);
 #endif

         Assert(!this->IsAllocationStopped());
     }
 }
 #endif

 template <typename TBlockType>
 void
 HeapBucketT<TBlockType>::AppendAllocableHeapBlockList(TBlockType * list)
 {
 #ifdef RECYCLER_TRACE
     if (this->GetRecycler()->GetRecyclerFlagsTable().Trace.IsEnabled(Js::ConcurrentSweepPhase) && CONFIG_FLAG_RELEASE(Verbose))
     {
         CollectionState collectionState = this->GetRecycler()->collectionState;
         Output::Print(_u("[GC #%d] [HeapBucket 0x%p] in AppendAllocableHeapBlockList [CollectionState: %d] \n"), this->GetRecycler()->collectionCount, this, collectionState);
     }
 #endif
     // Add the list to the end of the current list
     TBlockType * currentHeapBlockList = this->heapBlockList;
     if (currentHeapBlockList == nullptr)
     {
         // There weren't any heap block list before, just move the list over and start allocate from it
         this->heapBlockList = list;
         this->nextAllocableBlockHead = list;
     }
     else
     {
         // Find the last block and append the list
         TBlockType * tail = HeapBlockList::Tail(currentHeapBlockList);
         Assert(tail != nullptr);
         tail->SetNextBlock(list);

         // If we are not currently allocating from the existing heapBlockList,
         // that means fill all the exiting one already, we should start with what we just appended.
         if (this->nextAllocableBlockHead == nullptr)
         {
             this->nextAllocableBlockHead = list;
         }
     }
 }

 template <typename TBlockType>
 void
 HeapBucketT<TBlockType>::EnumerateObjects(ObjectInfoBits infoBits, void (*CallBackFunction)(void * address, size_t size))
 {
     UpdateAllocators();
     HeapBucket::EnumerateObjects(fullBlockList, infoBits, CallBackFunction);
 #if ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP && SUPPORT_WIN32_SLIST
     if (CONFIG_FLAG_RELEASE(EnableConcurrentSweepAlloc) && !this->IsAnyFinalizableBucket())
     {
         HeapBucket::EnumerateObjects(sweepableHeapBlockList, infoBits, CallBackFunction);
     }
 #endif
     HeapBucket::EnumerateObjects(heapBlockList, infoBits, CallBackFunction);
 }

 #ifdef RECYCLER_SLOW_CHECK_ENABLED
 template <typename TBlockType>
 void
 HeapBucketT<TBlockType>::VerifyHeapBlockCount(bool background)
 {
     // TODO-REFACTOR: GetNonEmptyHeapBlockCount really should be virtual
     static_cast<typename SmallHeapBlockType<TBlockType::RequiredAttributes, typename TBlockType::HeapBlockAttributes>::BucketType *>(this)->GetNonEmptyHeapBlockCount(true);
     if (!background)
     {
         this->GetEmptyHeapBlockCount();
     }
 }

 template <typename TBlockType>
 size_t
 HeapBucketT<TBlockType>::Check(bool checkCount)
 {
     Assert(this->GetRecycler()->recyclerSweepManager == nullptr);
     UpdateAllocators();
     size_t smallHeapBlockCount = HeapInfo::Check(true, false, this->fullBlockList);
     bool allocatingDuringConcurrentSweep = false;
 #if ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP && SUPPORT_WIN32_SLIST
     if (CONFIG_FLAG_RELEASE(EnableConcurrentSweepAlloc) && !this->IsAnyFinalizableBucket())
     {
         allocatingDuringConcurrentSweep = true;
         // This lock is needed only in the debug mode while we verify block counts. Not needed otherwise, as this list is never accessed concurrently.
         // Items are added to it by the allocator when allocations are allowed during concurrent sweep. The list is drained during the next sweep while
         // allocation are stopped.
         debugSweepableHeapBlockListLock.Enter();
         smallHeapBlockCount += HeapInfo::Check(true, false, this->sweepableHeapBlockList);
         debugSweepableHeapBlockListLock.Leave();
     }
 #endif
     smallHeapBlockCount += HeapInfo::Check(true, false, this->heapBlockList, this->nextAllocableBlockHead);
     smallHeapBlockCount += HeapInfo::Check(false, false, this->nextAllocableBlockHead);
     Assert(!checkCount || this->heapBlockCount == smallHeapBlockCount || (this->heapBlockCount >= 65535 && allocatingDuringConcurrentSweep));
     return smallHeapBlockCount;
 }
 #endif

 #if ENABLE_MEM_STATS
 template <typename TBlockType>
 void
 HeapBucketT<TBlockType>::AggregateBucketStats()
 {
     HeapBucket::AggregateBucketStats();  // call super

     auto allocatorHead = &this->allocatorHead;
     auto allocatorCurr = allocatorHead;

     do
     {
         TBlockType* allocatorHeapBlock = allocatorCurr->GetHeapBlock();
         if (allocatorHeapBlock)
         {
             allocatorHeapBlock->AggregateBlockStats(this->memStats, true, allocatorCurr->freeObjectList, allocatorCurr->endAddress != 0);
         }
         allocatorCurr = allocatorCurr->GetNext();
     } while (allocatorCurr != allocatorHead);

     auto blockStatsAggregator = [this](TBlockType* heapBlock) {
         heapBlock->AggregateBlockStats(this->memStats);
     };

     HeapBlockList::ForEach(fullBlockList, blockStatsAggregator);
 #if ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP && SUPPORT_WIN32_SLIST
     if (CONFIG_FLAG_RELEASE(EnableConcurrentSweepAlloc) && !this->IsAnyFinalizableBucket())
     {
         HeapBlockList::ForEach(sweepableHeapBlockList, blockStatsAggregator);
     }
 #endif
     HeapBlockList::ForEach(heapBlockList, blockStatsAggregator);
 }
 #endif

 #ifdef RECYCLER_MEMORY_VERIFY
 template <typename TBlockType>
 void
 HeapBucketT<TBlockType>::Verify()
 {
     UpdateAllocators();
 #if DBG
     RecyclerVerifyListConsistencyData recyclerVerifyListConsistencyData;

     if (TBlockType::HeapBlockAttributes::IsSmallBlock)
     {
         recyclerVerifyListConsistencyData.smallBlockVerifyListConsistencyData.SetupVerifyListConsistencyData((SmallHeapBlock*) nullptr, true, false);
     }
     else if (TBlockType::HeapBlockAttributes::IsMediumBlock)
     {
         recyclerVerifyListConsistencyData.mediumBlockVerifyListConsistencyData.SetupVerifyListConsistencyData((MediumHeapBlock*) nullptr, true, false);
     }
     else
     {
         Assert(false);
     }
 #endif

     HeapBlockList::ForEach(fullBlockList, [DebugOnly(&recyclerVerifyListConsistencyData)](TBlockType * heapBlock)
     {
         DebugOnly(VerifyBlockConsistencyInList(heapBlock, recyclerVerifyListConsistencyData));
         heapBlock->Verify();
     });

 #if ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP && SUPPORT_WIN32_SLIST
     if (CONFIG_FLAG_RELEASE(EnableConcurrentSweepAlloc) && !this->IsAnyFinalizableBucket())
     {
 #if DBG
         if (TBlockType::HeapBlockAttributes::IsSmallBlock)
         {
             recyclerVerifyListConsistencyData.smallBlockVerifyListConsistencyData.SetupVerifyListConsistencyData((SmallHeapBlock*) nullptr, true, false);
         }
         else if (TBlockType::HeapBlockAttributes::IsMediumBlock)
         {
             recyclerVerifyListConsistencyData.mediumBlockVerifyListConsistencyData.SetupVerifyListConsistencyData((MediumHeapBlock*) nullptr, true, false);
         }
         else
         {
             Assert(false);
         }
 #endif

         HeapBlockList::ForEach(sweepableHeapBlockList, [DebugOnly(&recyclerVerifyListConsistencyData)](TBlockType * heapBlock)
         {
             DebugOnly(VerifyBlockConsistencyInList(heapBlock, recyclerVerifyListConsistencyData));
             heapBlock->Verify();
         });
     }
 #endif

 #if DBG
     if (TBlockType::HeapBlockAttributes::IsSmallBlock)
     {
         recyclerVerifyListConsistencyData.smallBlockVerifyListConsistencyData.SetupVerifyListConsistencyData((SmallHeapBlock*) this->nextAllocableBlockHead, true, false);
     }
     else if (TBlockType::HeapBlockAttributes::IsMediumBlock)
     {
         recyclerVerifyListConsistencyData.mediumBlockVerifyListConsistencyData.SetupVerifyListConsistencyData((MediumHeapBlock*) this->nextAllocableBlockHead, true, false);
     }
     else
     {
         Assert(false);
     }
 #endif

     HeapBlockList::ForEach(heapBlockList, [this, DebugOnly(&recyclerVerifyListConsistencyData)](TBlockType * heapBlock)
     {
         DebugOnly(VerifyBlockConsistencyInList(heapBlock, recyclerVerifyListConsistencyData));
         char * bumpAllocateAddress = nullptr;
         this->ForEachAllocator([heapBlock, &bumpAllocateAddress](TBlockAllocatorType * allocator)
         {
             if (allocator->GetHeapBlock() == heapBlock && allocator->GetEndAddress() != nullptr)
             {
                 Assert(bumpAllocateAddress == nullptr);
                 bumpAllocateAddress = (char *)allocator->GetFreeObjectList();
             }
         });
         if (bumpAllocateAddress != nullptr)
         {
             heapBlock->VerifyBumpAllocated(bumpAllocateAddress);
         }
         else
         {
             heapBlock->Verify(false);
         }
     });
 }
 #endif

 #ifdef RECYCLER_VERIFY_MARK
 template <typename TBlockType>
 void
 HeapBucketT<TBlockType>::VerifyMark()
 {
     HeapBlockList::ForEach(this->fullBlockList, [](TBlockType * heapBlock)
     {
         heapBlock->VerifyMark();
     });

 #if ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP && SUPPORT_WIN32_SLIST
     if (CONFIG_FLAG_RELEASE(EnableConcurrentSweepAlloc) && !this->IsAnyFinalizableBucket())
     {
         HeapBlockList::ForEach(this->sweepableHeapBlockList, [](TBlockType * heapBlock)
         {
             heapBlock->VerifyMark();
         });
     }
 #endif

     HeapBlockList::ForEach(this->heapBlockList, [](TBlockType * heapBlock)
     {
         heapBlock->VerifyMark();
     });
 }
 #endif

 template <class TBlockAttributes>
 void
 HeapBucketGroup<TBlockAttributes>::Initialize(HeapInfo * heapInfo, uint sizeCat)
 {
     heapBucket.Initialize(heapInfo, sizeCat);
     leafHeapBucket.Initialize(heapInfo, sizeCat);
 #ifdef RECYCLER_WRITE_BARRIER
     smallNormalWithBarrierHeapBucket.Initialize(heapInfo, sizeCat);
     smallFinalizableWithBarrierHeapBucket.Initialize(heapInfo, sizeCat);
 #endif
     finalizableHeapBucket.Initialize(heapInfo, sizeCat);
 #ifdef RECYCLER_VISITED_HOST
     recyclerVisitedHostHeapBucket.Initialize(heapInfo, sizeCat);
 #endif
 }

 template <class TBlockAttributes>
 void
 HeapBucketGroup<TBlockAttributes>::ResetMarks(ResetMarkFlags flags)
 {
     heapBucket.ResetMarks(flags);
     leafHeapBucket.ResetMarks(flags);
 #ifdef RECYCLER_WRITE_BARRIER
     smallNormalWithBarrierHeapBucket.ResetMarks(flags);
     smallFinalizableWithBarrierHeapBucket.ResetMarks(flags);
 #endif

     // Although we pass in premarkFreeObjects, the finalizable heap bucket ignores
     // this parameter and never pre-marks free objects
     finalizableHeapBucket.ResetMarks(flags);
 #ifdef RECYCLER_VISITED_HOST
     recyclerVisitedHostHeapBucket.ResetMarks(flags);
 #endif
 }

 template <class TBlockAttributes>
 void
 HeapBucketGroup<TBlockAttributes>::ScanInitialImplicitRoots(Recycler * recycler)
 {
     heapBucket.ScanInitialImplicitRoots(recycler);
     // Don't need to scan implicit roots on leaf heap bucket
 #ifdef RECYCLER_WRITE_BARRIER
     smallNormalWithBarrierHeapBucket.ScanInitialImplicitRoots(recycler);
     smallFinalizableWithBarrierHeapBucket.ScanInitialImplicitRoots(recycler);
 #endif
     finalizableHeapBucket.ScanInitialImplicitRoots(recycler);
 }

 template <class TBlockAttributes>
 void
 HeapBucketGroup<TBlockAttributes>::ScanNewImplicitRoots(Recycler * recycler)
 {
     heapBucket.ScanNewImplicitRoots(recycler);
     // Need to scan new implicit roots on leaf heap bucket
     leafHeapBucket.ScanNewImplicitRoots(recycler);
 #ifdef RECYCLER_WRITE_BARRIER
     smallNormalWithBarrierHeapBucket.ScanNewImplicitRoots(recycler);
     smallFinalizableWithBarrierHeapBucket.ScanNewImplicitRoots(recycler);
 #endif
     finalizableHeapBucket.ScanNewImplicitRoots(recycler);
 }

 template <class TBlockAttributes>
 void
 HeapBucketGroup<TBlockAttributes>::Sweep(RecyclerSweep& recyclerSweep)
 {
     heapBucket.Sweep(recyclerSweep);
     leafHeapBucket.Sweep(recyclerSweep);
 #ifdef RECYCLER_WRITE_BARRIER
     smallNormalWithBarrierHeapBucket.Sweep(recyclerSweep);
 #endif
 }

 // Sweep finalizable objects first to ensure that if they reference any other
 // objects in the finalizer - they are valid
 template <class TBlockAttributes>
 void
 HeapBucketGroup<TBlockAttributes>::SweepFinalizableObjects(RecyclerSweep& recyclerSweep)
 {
     finalizableHeapBucket.Sweep(recyclerSweep);
 #ifdef RECYCLER_VISITED_HOST
     recyclerVisitedHostHeapBucket.Sweep(recyclerSweep);
 #endif
 #ifdef RECYCLER_WRITE_BARRIER
     smallFinalizableWithBarrierHeapBucket.Sweep(recyclerSweep);
 #endif
 }

 template <class TBlockAttributes>
 void
 HeapBucketGroup<TBlockAttributes>::DisposeObjects()
 {
     finalizableHeapBucket.DisposeObjects();
 #ifdef RECYCLER_VISITED_HOST
     recyclerVisitedHostHeapBucket.DisposeObjects();
 #endif
 #ifdef RECYCLER_WRITE_BARRIER
     smallFinalizableWithBarrierHeapBucket.DisposeObjects();
 #endif
 }

 template <class TBlockAttributes>
 void
 HeapBucketGroup<TBlockAttributes>::TransferDisposedObjects()
 {
     finalizableHeapBucket.TransferDisposedObjects();
 #ifdef RECYCLER_VISITED_HOST
     recyclerVisitedHostHeapBucket.TransferDisposedObjects();
 #endif
 #ifdef RECYCLER_WRITE_BARRIER
     smallFinalizableWithBarrierHeapBucket.TransferDisposedObjects();
 #endif
 }

 template <class TBlockAttributes>
 void
 HeapBucketGroup<TBlockAttributes>::EnumerateObjects(ObjectInfoBits infoBits, void (*CallBackFunction)(void * address, size_t size))
 {
     heapBucket.EnumerateObjects(infoBits, CallBackFunction);
     leafHeapBucket.EnumerateObjects(infoBits, CallBackFunction);
 #ifdef RECYCLER_WRITE_BARRIER
     smallNormalWithBarrierHeapBucket.EnumerateObjects(infoBits, CallBackFunction);
     smallFinalizableWithBarrierHeapBucket.EnumerateObjects(infoBits, CallBackFunction);
 #endif
     finalizableHeapBucket.EnumerateObjects(infoBits, CallBackFunction);
 #ifdef RECYCLER_VISITED_HOST
     recyclerVisitedHostHeapBucket.EnumerateObjects(infoBits, CallBackFunction);
 #endif
 }

 template <class TBlockAttributes>
 void
 HeapBucketGroup<TBlockAttributes>::FinalizeAllObjects()
 {
     finalizableHeapBucket.FinalizeAllObjects();
 #ifdef RECYCLER_VISITED_HOST
     recyclerVisitedHostHeapBucket.FinalizeAllObjects();
 #endif
 #ifdef RECYCLER_WRITE_BARRIER
     smallFinalizableWithBarrierHeapBucket.FinalizeAllObjects();
 #endif
 }

 template <class TBlockAttributes>
 uint
 HeapBucketGroup<TBlockAttributes>::Rescan(Recycler * recycler, RescanFlags flags)
 {
     return heapBucket.Rescan(recycler, flags) +
         leafHeapBucket.Rescan(recycler, flags) +
 #ifdef RECYCLER_WRITE_BARRIER
         smallNormalWithBarrierHeapBucket.Rescan(recycler, flags) +
         smallFinalizableWithBarrierHeapBucket.Rescan(recycler, flags) +
 #endif
 #ifdef RECYCLER_VISITED_HOST
         recyclerVisitedHostHeapBucket.Rescan(recycler, flags) +
 #endif
         finalizableHeapBucket.Rescan(recycler, flags);
 }

 #if ENABLE_CONCURRENT_GC
 template <class TBlockAttributes>
 void
 HeapBucketGroup<TBlockAttributes>::PrepareSweep()
 {
     heapBucket.PrepareSweep();
     leafHeapBucket.PrepareSweep();
 #ifdef RECYCLER_WRITE_BARRIER
     smallNormalWithBarrierHeapBucket.PrepareSweep();
     smallFinalizableWithBarrierHeapBucket.PrepareSweep();
 #endif
     finalizableHeapBucket.PrepareSweep();
 #ifdef RECYCLER_VISITED_HOST
     recyclerVisitedHostHeapBucket.PrepareSweep();
 #endif
 }

 template <class TBlockAttributes>
 void
 HeapBucketGroup<TBlockAttributes>::SetupBackgroundSweep(RecyclerSweep& recyclerSweep)
 {
     heapBucket.SetupBackgroundSweep(recyclerSweep);
     leafHeapBucket.SetupBackgroundSweep(recyclerSweep);
 #ifdef RECYCLER_WRITE_BARRIER
     smallNormalWithBarrierHeapBucket.SetupBackgroundSweep(recyclerSweep);
 #endif
 }
 #endif
 #if ENABLE_PARTIAL_GC
 template <class TBlockAttributes>
 void
 HeapBucketGroup<TBlockAttributes>::SweepPartialReusePages(RecyclerSweep& recyclerSweep)
 {
     // Leaf heap bucket are always reused for allocation and can be done on the concurrent thread
     // WriteBarrier-TODO: Do the same for write barrier buckets
     heapBucket.SweepPartialReusePages(recyclerSweep);

 #ifdef RECYCLER_WRITE_BARRIER
     smallNormalWithBarrierHeapBucket.SweepPartialReusePages(recyclerSweep);
     smallFinalizableWithBarrierHeapBucket.SweepPartialReusePages(recyclerSweep);
 #endif

     finalizableHeapBucket.SweepPartialReusePages(recyclerSweep);
 #ifdef RECYCLER_VISITED_HOST
     recyclerVisitedHostHeapBucket.SweepPartialReusePages(recyclerSweep);
 #endif
 }

 template <class TBlockAttributes>
 void
 HeapBucketGroup<TBlockAttributes>::FinishPartialCollect(RecyclerSweep * recyclerSweep)
 {
     heapBucket.FinishPartialCollect(recyclerSweep);
 #ifdef RECYCLER_WRITE_BARRIER
     smallNormalWithBarrierHeapBucket.FinishPartialCollect(recyclerSweep);
     smallFinalizableWithBarrierHeapBucket.FinishPartialCollect(recyclerSweep);
 #endif
     finalizableHeapBucket.FinishPartialCollect(recyclerSweep);
 #ifdef RECYCLER_VISITED_HOST
     recyclerVisitedHostHeapBucket.FinishPartialCollect(recyclerSweep);
 #endif

     // Leaf heap block always do a full sweep instead of partial sweep
     // (since touching the page doesn't affect rescan)
     // So just need to verify heap block count (which finishPartialCollect would have done)
     // WriteBarrier-TODO: Do that same for write barrier buckets
     RECYCLER_SLOW_CHECK(leafHeapBucket.VerifyHeapBlockCount(recyclerSweep != nullptr && recyclerSweep->IsBackground()));
 }
 #endif

 #if ENABLE_CONCURRENT_GC
 template <class TBlockAttributes>
 void
 HeapBucketGroup<TBlockAttributes>::SweepPendingObjects(RecyclerSweep& recyclerSweep)
 {
     // For leaf buckets, we can always reuse the page as we don't need to rescan them for partial GC
     // It should have been swept immediately during Sweep
     // WriteBarrier-TODO: Do the same for write barrier buckets
     Assert(recyclerSweep.GetPendingSweepBlockList(&leafHeapBucket) == nullptr);

     heapBucket.SweepPendingObjects(recyclerSweep);
 #ifdef RECYCLER_WRITE_BARRIER
     smallNormalWithBarrierHeapBucket.SweepPendingObjects(recyclerSweep);
     smallFinalizableWithBarrierHeapBucket.SweepPendingObjects(recyclerSweep);
 #endif

     finalizableHeapBucket.SweepPendingObjects(recyclerSweep);
 #ifdef RECYCLER_VISITED_HOST
     recyclerVisitedHostHeapBucket.SweepPendingObjects(recyclerSweep);
 #endif
 }

 template <class TBlockAttributes>
 void
 HeapBucketGroup<TBlockAttributes>::TransferPendingEmptyHeapBlocks(RecyclerSweep& recyclerSweep)
 {
     recyclerSweep.TransferPendingEmptyHeapBlocks(&heapBucket);
     recyclerSweep.TransferPendingEmptyHeapBlocks(&leafHeapBucket);
 #ifdef RECYCLER_WRITE_BARRIER
     recyclerSweep.TransferPendingEmptyHeapBlocks(&smallNormalWithBarrierHeapBucket);
     recyclerSweep.TransferPendingEmptyHeapBlocks(&smallFinalizableWithBarrierHeapBucket);
 #endif
     recyclerSweep.TransferPendingEmptyHeapBlocks(&finalizableHeapBucket);
 #ifdef RECYCLER_VISITED_HOST
     recyclerSweep.TransferPendingEmptyHeapBlocks(&recyclerVisitedHostHeapBucket);
 #endif
 }
 #endif

 #if DBG || defined(RECYCLER_SLOW_CHECK_ENABLED)
 template <class TBlockAttributes>
 size_t
 HeapBucketGroup<TBlockAttributes>::GetNonEmptyHeapBlockCount(bool checkCount) const
 {
     return heapBucket.GetNonEmptyHeapBlockCount(checkCount) +
         finalizableHeapBucket.GetNonEmptyHeapBlockCount(checkCount) +
 #ifdef RECYCLER_VISITED_HOST
         recyclerVisitedHostHeapBucket.GetNonEmptyHeapBlockCount(checkCount) +
 #endif
 #ifdef RECYCLER_WRITE_BARRIER
         smallNormalWithBarrierHeapBucket.GetNonEmptyHeapBlockCount(checkCount) +
         smallFinalizableWithBarrierHeapBucket.GetNonEmptyHeapBlockCount(checkCount) +
 #endif
         leafHeapBucket.GetNonEmptyHeapBlockCount(checkCount);
 }

 template <class TBlockAttributes>
 size_t
 HeapBucketGroup<TBlockAttributes>::GetEmptyHeapBlockCount() const
 {
     return heapBucket.GetEmptyHeapBlockCount() +
         finalizableHeapBucket.GetEmptyHeapBlockCount() +
 #ifdef RECYCLER_VISITED_HOST
         recyclerVisitedHostHeapBucket.GetEmptyHeapBlockCount() +
 #endif
 #ifdef RECYCLER_WRITE_BARRIER
         smallNormalWithBarrierHeapBucket.GetEmptyHeapBlockCount() +
         smallFinalizableWithBarrierHeapBucket.GetEmptyHeapBlockCount() +
 #endif
         leafHeapBucket.GetEmptyHeapBlockCount();
 }
 #endif

 #ifdef RECYCLER_SLOW_CHECK_ENABLED
 template <class TBlockAttributes>
 size_t
 HeapBucketGroup<TBlockAttributes>::Check()
 {
     return heapBucket.Check() + finalizableHeapBucket.Check() + leafHeapBucket.Check()
 #ifdef RECYCLER_VISITED_HOST
         + recyclerVisitedHostHeapBucket.Check()
 #endif
 #ifdef RECYCLER_WRITE_BARRIER
         + smallNormalWithBarrierHeapBucket.Check() + smallFinalizableWithBarrierHeapBucket.Check()
 #endif
         ;
 }
 #endif
 #ifdef RECYCLER_MEMORY_VERIFY
 template <class TBlockAttributes>
 void
 HeapBucketGroup<TBlockAttributes>::Verify()
 {
     heapBucket.Verify();
     finalizableHeapBucket.Verify();
 #ifdef RECYCLER_VISITED_HOST
     recyclerVisitedHostHeapBucket.Verify();
 #endif
     leafHeapBucket.Verify();
 #ifdef RECYCLER_WRITE_BARRIER
     smallNormalWithBarrierHeapBucket.Verify();
     smallFinalizableWithBarrierHeapBucket.Verify();
 #endif
 }
 #endif
 #ifdef RECYCLER_VERIFY_MARK
 template <class TBlockAttributes>
 void
 HeapBucketGroup<TBlockAttributes>::VerifyMark()
 {
     heapBucket.VerifyMark();
     finalizableHeapBucket.VerifyMark();
 #ifdef RECYCLER_VISITED_HOST
     recyclerVisitedHostHeapBucket.VerifyMark();
 #endif
     leafHeapBucket.VerifyMark();
 #ifdef RECYCLER_WRITE_BARRIER
     smallNormalWithBarrierHeapBucket.VerifyMark();
     smallFinalizableWithBarrierHeapBucket.VerifyMark();
 #endif
 }
 #endif

 #if ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP
 template <class TBlockAttributes>
 void
 HeapBucketGroup<TBlockAttributes>::StartAllocationDuringConcurrentSweep()
 {
     // If there were no allocable heap blocks we would not have started alllocations. Stop allocations, only if we started allocations for each of these buckets.
     if (heapBucket.IsAllocationStopped())
     {
         heapBucket.StartAllocationDuringConcurrentSweep();
     }

     if (leafHeapBucket.IsAllocationStopped())
     {
         leafHeapBucket.StartAllocationDuringConcurrentSweep();
     }
 #ifdef RECYCLER_WRITE_BARRIER
     if (smallNormalWithBarrierHeapBucket.IsAllocationStopped())
     {
         smallNormalWithBarrierHeapBucket.StartAllocationDuringConcurrentSweep();
     }
 #endif
 }

 template <class TBlockAttributes>
 bool
 HeapBucketGroup<TBlockAttributes>::DoTwoPassConcurrentSweepPreCheck()
 {
     return heapBucket.DoTwoPassConcurrentSweepPreCheck() ||
     leafHeapBucket.DoTwoPassConcurrentSweepPreCheck()

 #ifdef RECYCLER_WRITE_BARRIER
     || smallNormalWithBarrierHeapBucket.DoTwoPassConcurrentSweepPreCheck();
 #endif
 }

 template <class TBlockAttributes>
 void
 HeapBucketGroup<TBlockAttributes>::FinishConcurrentSweepPass1(RecyclerSweep& recyclerSweep)
 {
     heapBucket.FinishConcurrentSweepPass1(recyclerSweep);
     leafHeapBucket.FinishConcurrentSweepPass1(recyclerSweep);

 #ifdef RECYCLER_WRITE_BARRIER
     smallNormalWithBarrierHeapBucket.FinishConcurrentSweepPass1(recyclerSweep);
 #endif
 }

 template <class TBlockAttributes>
 void
 HeapBucketGroup<TBlockAttributes>::FinishSweepPrep(RecyclerSweep& recyclerSweep)
 {
     heapBucket.FinishSweepPrep(recyclerSweep);
     leafHeapBucket.FinishSweepPrep(recyclerSweep);

 #ifdef RECYCLER_WRITE_BARRIER
     smallNormalWithBarrierHeapBucket.FinishSweepPrep(recyclerSweep);
 #endif
 }

 template <class TBlockAttributes>
 void
 HeapBucketGroup<TBlockAttributes>::FinishConcurrentSweep()
 {
     heapBucket.FinishConcurrentSweep();
     leafHeapBucket.FinishConcurrentSweep();

 #ifdef RECYCLER_WRITE_BARRIER
     smallNormalWithBarrierHeapBucket.FinishConcurrentSweep();
 #endif
 }
 #endif

 #if DBG
 template <class TBlockAttributes>
 bool
 HeapBucketGroup<TBlockAttributes>::AllocatorsAreEmpty()
 {
     return heapBucket.AllocatorsAreEmpty()
         && finalizableHeapBucket.AllocatorsAreEmpty()
 #ifdef RECYCLER_VISITED_HOST
         && recyclerVisitedHostHeapBucket.AllocatorsAreEmpty()
 #endif
         && leafHeapBucket.AllocatorsAreEmpty()
 #ifdef RECYCLER_WRITE_BARRIER
         && smallNormalWithBarrierHeapBucket.AllocatorsAreEmpty()
         && smallFinalizableWithBarrierHeapBucket.AllocatorsAreEmpty()
 #endif
         ;
 }
 #endif

 namespace Memory
 {
     template class HeapBucketGroup<SmallAllocationBlockAttributes>;
     template class HeapBucketGroup<MediumAllocationBlockAttributes>;

     EXPLICIT_INSTANTIATE_WITH_SMALL_HEAP_BLOCK_TYPE(HeapBucketT);
 };