Merge pull request #22116 from LinHu2016/defrag-offheap-1.5

Decouple spine from array reserved regions

Author: amicic

runtime/gc_base/IndexableObjectAllocationModel.cpp

@@ -27,7 +27,7 @@
 #include "MemorySpace.hpp"
 #if defined(J9VM_GC_SPARSE_HEAP_ALLOCATION) || defined(J9VM_GC_ENABLE_DOUBLE_MAP)
 #include "ArrayletLeafIterator.hpp"
-#include "HeapRegionManager.hpp"
+#include "HeapRegionManagerVLHGC.hpp"
 #include "HeapRegionDescriptorVLHGC.hpp"
 #include "Heap.hpp"
 #include "SparseVirtualMemory.hpp"
@@ -334,54 +334,49 @@ MM_IndexableObjectAllocationModel::layoutDiscontiguousArraylet(MM_EnvironmentBas
 
 #if defined(J9VM_GC_SPARSE_HEAP_ALLOCATION)
 MMINLINE J9IndexableObject *
-MM_IndexableObjectAllocationModel::getSparseAddressAndDecommitLeaves(MM_EnvironmentBase *env, J9IndexableObject *spine)
+MM_IndexableObjectAllocationModel::getSparseAddressAndDecommitLeaves(MM_EnvironmentBase *envBase, J9IndexableObject *spine)
 {
 	Assert_MM_true(_numberOfArraylets == _allocateDescription.getNumArraylets());
 
-	MM_GCExtensions *extensions = MM_GCExtensions::getExtensions(env);
+	MM_GCExtensions *extensions = MM_GCExtensions::getExtensions(envBase);
 	GC_ArrayObjectModel *indexableObjectModel = &extensions->indexableObjectModel;
-	const UDATA arrayletLeafSize = env->getOmrVM()->_arrayletLeafSize;
+	const uintptr_t regionSize = extensions->heapRegionManager->getRegionSize();
+
 	uintptr_t byteAmount = 0;
-	UDATA arrayletLeafCount = MM_Math::roundToCeiling(arrayletLeafSize, _dataSize) / arrayletLeafSize;
+	UDATA arrayReservedRegionCount = MM_Math::roundToCeiling(regionSize, _dataSize) / regionSize;
 
 	/* Determine how many bytes to allocate outside of the spine (in arraylet leaves). */
 	Assert_MM_true(_allocateDescription.getBytesRequested() >= _allocateDescription.getContiguousBytes());
 	uintptr_t bytesRemaining = _allocateDescription.getBytesRequested() - _allocateDescription.getContiguousBytes();
 
 	/* Allocate leaf for each arraylet and attach it to its leaf pointer in the spine. */
 	uintptr_t arrayoidIndex = 0;
-	Trc_MM_getSparseAddressAndDecommitLeaves_Entry(env->getLanguageVMThread(), spine, (void *)bytesRemaining, arrayletLeafCount, (void *)arrayletLeafSize);
+	Trc_MM_getSparseAddressAndDecommitLeaves_Entry(envBase->getLanguageVMThread(), spine, (void *)bytesRemaining, arrayReservedRegionCount, (void *)regionSize);
 	while (0 < bytesRemaining) {
 		/* Allocate the next arraylet leaf - leaves are allocated solely for the purpose of
 		   decommitting the memory later on in this function. */
-		void *leaf = env->_objectAllocationInterface->allocateArrayletLeaf(
-				env, &_allocateDescription, _allocateDescription.getMemorySpace(), true);
+		void *leaf = envBase->_objectAllocationInterface->allocateArrayletLeaf(
+				envBase, &_allocateDescription, _allocateDescription.getMemorySpace(), true);
 
 		/* If leaf allocation failed set the result to NULL and return. */
 		if (NULL == leaf) {
-			/* Spine and preceding arraylets are now floating garbage. */
-			Trc_MM_allocateAndConnectNonContiguousArraylet_leafFailure(env->getLanguageVMThread());
+			Trc_MM_allocateAndConnectNonContiguousArraylet_leafFailure(envBase->getLanguageVMThread());
 			_allocateDescription.setSpine(NULL);
 			spine = NULL;
 			break;
 		}
 
-		if (0 == arrayoidIndex) {
-			MM_HeapRegionDescriptorVLHGC *firstLeafRegionDescriptor = (MM_HeapRegionDescriptorVLHGC *)extensions->getHeap()->getHeapRegionManager()->tableDescriptorForAddress(leaf);
-			firstLeafRegionDescriptor->_sparseHeapAllocation = true;
-		}
-
 		/* Disable region for reads and writes, since that'll be done through the contiguous double mapped region */
-		void *highAddress = (void *)((uintptr_t)leaf + arrayletLeafSize);
-		bool ret = extensions->heap->decommitMemory(leaf, arrayletLeafSize, leaf, highAddress);
+		void *highAddress = (void *)((uintptr_t)leaf + regionSize);
+		bool ret = extensions->heap->decommitMemory(leaf, regionSize, leaf, highAddress);
 		if (!ret) {
-			Trc_MM_VirtualMemory_decommitMemory_failure(leaf, arrayletLeafSize);
+			Trc_MM_VirtualMemory_decommitMemory_failure(leaf, regionSize);
 		}
 
 		/* Refresh the spine -- it might move if we GC while allocating the leaf */
 		spine = _allocateDescription.getSpine();
 
-		bytesRemaining -= OMR_MIN(bytesRemaining, arrayletLeafSize);
+		bytesRemaining -= OMR_MIN(bytesRemaining, regionSize);
 		arrayoidIndex += 1;
 	}
 
@@ -390,10 +385,11 @@ MM_IndexableObjectAllocationModel::getSparseAddressAndDecommitLeaves(MM_Environm
 		Assert_MM_true(_layout == GC_ArrayletObjectModel::InlineContiguous);
 		Assert_MM_true(indexableObjectModel->isVirtualLargeObjectHeapEnabled());
 		/* Number of arraylet leaves in the iterator must match the number of leaves calculated */
-		Assert_MM_true(arrayletLeafCount == arrayoidIndex);
+		Assert_MM_true(arrayReservedRegionCount == arrayoidIndex);
 
 		byteAmount = _dataSize;
 		void *virtualLargeObjectHeapAddress = extensions->largeObjectVirtualMemory->allocateSparseFreeEntryAndMapToHeapObject(spine, byteAmount);
+
 		if (NULL != virtualLargeObjectHeapAddress) {
 			indexableObjectModel->setDataAddrForContiguous((J9IndexableObject *)spine, virtualLargeObjectHeapAddress);
 		} else {
@@ -402,7 +398,13 @@ MM_IndexableObjectAllocationModel::getSparseAddressAndDecommitLeaves(MM_Environm
 		}
 	}
 
-	Trc_MM_getSparseAddressAndDecommitLeaves_Exit(env->getLanguageVMThread(), spine, (void *)bytesRemaining);
+	if (NULL == spine) {
+		/* fail to reserve regions or allocateSparseFreeEntry, clean up reserved regions */
+		if (0 < arrayoidIndex) {
+			((MM_HeapRegionManagerVLHGC *)extensions->heapRegionManager)->recycleReservedRegionsForVirtualLargeObjectHeap(envBase, arrayoidIndex);
+		}
+	}
+	Trc_MM_getSparseAddressAndDecommitLeaves_Exit(envBase->getLanguageVMThread(), spine, (void *)bytesRemaining);
 
 	return spine;
 }

runtime/gc_base/RootScanner.cpp

@@ -41,6 +41,7 @@
 #if defined(J9VM_GC_FINALIZATION)
 #include "FinalizeListManager.hpp"
 #endif /* J9VM_GC_FINALIZATION*/
+#include "HashTableIterator.hpp"
 #include "Heap.hpp"
 #include "HeapRegionDescriptor.hpp"
 #include "HeapRegionIterator.hpp"
@@ -60,6 +61,10 @@
 #include "ParallelDispatcher.hpp"
 #include "PointerArrayIterator.hpp"
 #include "SlotObject.hpp"
+#if defined(J9VM_GC_SPARSE_HEAP_ALLOCATION)
+#include "SparseVirtualMemory.hpp"
+#include "SparseAddressOrderedFixedSizeDataPool.hpp"
+#endif /* defined(J9VM_GC_SPARSE_HEAP_ALLOCATION) */
 #include "StringTable.hpp"
 #include "StringTableIncrementalIterator.hpp"
 #include "Task.hpp"
@@ -241,7 +246,7 @@ MM_RootScanner::doStringTableSlot(J9Object **slotPtr, GC_StringTableIterator *st
 
 #if defined(J9VM_GC_SPARSE_HEAP_ALLOCATION)
 void
-MM_RootScanner::doObjectInVirtualLargeObjectHeap(J9Object *objectPtr, bool *sparseHeapAllocation)
+MM_RootScanner::doObjectInVirtualLargeObjectHeap(J9Object *objectPtr, GC_HashTableIterator *sparseDataEntryIterator)
 {
 	/* No need to call doSlot() here since there's nothing to update */
 }
@@ -947,16 +952,16 @@ void
 MM_RootScanner::scanObjectsInVirtualLargeObjectHeap(MM_EnvironmentBase *env)
 {
 	if (_singleThread || J9MODRON_HANDLE_NEXT_WORK_UNIT(env)) {
-		GC_HeapRegionIteratorVLHGC regionIterator(_extensions->heap->getHeapRegionManager());
-		MM_HeapRegionDescriptorVLHGC *region = NULL;
 		reportScanningStarted(RootScannerEntity_virtualLargeObjectHeapObjects);
-		while (NULL != (region = regionIterator.nextRegion())) {
-			if (region->isArrayletLeaf()) {
-				if (region->_sparseHeapAllocation) {
-					J9Object *spineObject = (J9Object *)region->_allocateData.getSpine();
-					Assert_MM_true(NULL != spineObject);
-					doObjectInVirtualLargeObjectHeap(spineObject, &region->_sparseHeapAllocation);
-				}
+		MM_SparseVirtualMemory *largeObjectVirtualMemory = _extensions->largeObjectVirtualMemory;
+		J9HashTable *table = largeObjectVirtualMemory->getSparseDataPool()->getObjectToSparseDataTable();
+		if (NULL != table) {
+			MM_SparseDataTableEntry *sparseDataEntry = NULL;
+			GC_HashTableIterator iterator(table);
+			while (NULL != (sparseDataEntry = (MM_SparseDataTableEntry *)iterator.nextSlot())) {
+				J9Object *spineObject = (J9Object *)sparseDataEntry->_proxyObjPtr;
+				Assert_MM_true(NULL != spineObject);
+				doObjectInVirtualLargeObjectHeap(spineObject, &iterator);
 			}
 		}
 		reportScanningEnded(RootScannerEntity_virtualLargeObjectHeapObjects);
@@ -971,6 +976,7 @@ MM_RootScanner::scanDoubleMappedObjects(MM_EnvironmentBase *env)
 	if (_singleThread || J9MODRON_HANDLE_NEXT_WORK_UNIT(env)) {
 		GC_HeapRegionIteratorVLHGC regionIterator(_extensions->heap->getHeapRegionManager());
 		MM_HeapRegionDescriptorVLHGC *region = NULL;
+
 		reportScanningStarted(RootScannerEntity_DoubleMappedObjects);
 		while (NULL != (region = regionIterator.nextRegion())) {
 			if (region->isArrayletLeaf()) {

runtime/gc_base/RootScanner.hpp

@@ -46,6 +46,7 @@
 #include "Task.hpp"
 #include "VMClassSlotIterator.hpp"
 
+class GC_HashTableIterator;
 class GC_SlotObject;
 class MM_MemoryPool;
 class MM_CollectorLanguageInterfaceImpl;
@@ -570,7 +571,7 @@ class MM_RootScanner : public MM_BaseVirtual
 	 *
 	 * @param objectPtr[in] indexable object's spine
 	 */
-	virtual void doObjectInVirtualLargeObjectHeap(J9Object *objectPtr, bool *sparseHeapAllocation);
+	virtual void doObjectInVirtualLargeObjectHeap(J9Object *objectPtr, GC_HashTableIterator *sparseDataEntryIterator);
 #endif /* defined(J9VM_GC_SPARSE_HEAP_ALLOCATION) */
 
 #if defined(J9VM_GC_ENABLE_DOUBLE_MAP)

runtime/gc_vlhgc/AllocationContextBalanced.cpp

@@ -183,6 +183,7 @@ MM_AllocationContextBalanced::allocateTLH(MM_EnvironmentBase *env, MM_AllocateDe
 		result = lockedReplenishAndAllocate(env, objectAllocationInterface, allocateDescription, MM_MemorySubSpace::ALLOCATION_TYPE_TLH);
 	}
 	unlockCommon();
+
 	/* if that still fails, try to invoke the collector */
 	if (shouldCollectOnFailure && (NULL == result)) {
 		result = _subspace->replenishAllocationContextFailed(env, _subspace, this, objectAllocationInterface, allocateDescription, MM_MemorySubSpace::ALLOCATION_TYPE_TLH);
@@ -256,6 +257,7 @@ MM_AllocationContextBalanced::allocateObject(MM_EnvironmentBase *env, MM_Allocat
 		result = lockedReplenishAndAllocate(env, NULL, allocateDescription, MM_MemorySubSpace::ALLOCATION_TYPE_OBJECT);
 	}
 	unlockCommon();
+
 	/* if that still fails, try to invoke the collector */
 	if (shouldCollectOnFailure && (NULL == result)) {
 		result = _subspace->replenishAllocationContextFailed(env, _subspace, this, NULL, allocateDescription, MM_MemorySubSpace::ALLOCATION_TYPE_OBJECT);
@@ -321,6 +323,7 @@ MM_AllocationContextBalanced::allocateArrayletLeaf(MM_EnvironmentBase *env, MM_A
 	lockCommon();
 	result = lockedReplenishAndAllocate(env, NULL, allocateDescription, MM_MemorySubSpace::ALLOCATION_TYPE_LEAF);
 	unlockCommon();
+
 	/* if that fails, try to invoke the collector */
 	if (shouldCollectOnFailure && (NULL == result)) {
 		result = _subspace->replenishAllocationContextFailed(env, _subspace, this, NULL, allocateDescription, MM_MemorySubSpace::ALLOCATION_TYPE_LEAF);
@@ -355,30 +358,56 @@ MM_AllocationContextBalanced::lockedAllocateArrayletLeaf(MM_EnvironmentBase *env
 	/* look up the spine region since we need to add this region to its leaf list */
 	MM_HeapRegionDescriptorVLHGC *spineRegion = (MM_HeapRegionDescriptorVLHGC *)_heapRegionManager->tableDescriptorForAddress(spine);
 	/* the leaf requires a pointer back to the spine object so that it can verify its liveness elsewhere in the collector */
-	leafAllocateData->setSpine(spine);
-	freeRegionForArrayletLeaf->resetAge(env, (U_64)_subspace->getBytesRemainingBeforeTaxation());
-	/* add the leaf to the spine region's leaf list */
-	/* We own the lock on the spine region's context when this call is made so we can safely manipulate this list.
-	 * An exceptional scenario: A thread allocates a spine (and possibly a few arraylets), but does not complete the allocation. A global GC (or a series of regular PGCs) occurs
-	 * that age out regions to max age. The spine moves into a common context. Now, we successfully resume the leaf allocation, but the common lock that
-	 * we already hold is not sufficient any more. We need to additionally acquire common context' common lock, since multiple spines from different ACs could have come into this state,
-	 * and worse multiple spines originally allocated from different ACs may end up in a single common context region.
-	 */
+	if (!MM_GCExtensions::getExtensions(env)->isVirtualLargeObjectHeapEnabled) {
+		leafAllocateData->setSpine(spine);
+		/* add the leaf to the spine region's leaf list */
+		/* We own the lock on the spine region's context when this call is made so we can safely manipulate this list.
+		 * An exceptional scenario: A thread allocates a spine (and possibly a few arraylets), but does not complete the allocation. A global GC (or a series of regular PGCs) occurs
+		 * that age out regions to max age. The spine moves into a common context. Now, we successfully resume the leaf allocation, but the common lock that
+		 * we already hold is not sufficient any more. We need to additionally acquire common context' common lock, since multiple spines from different ACs could have come into this state,
+		 * and worse multiple spines originally allocated from different ACs may end up in a single common context region.
+		 */
+
+		MM_AllocationContextTarok *spineContext = spineRegion->_allocateData._owningContext;
+		if (this != spineContext) {
+			Assert_MM_true(env->getCommonAllocationContext() == spineContext);
+			/* The common allocation context is always an instance of AllocationContextBalanced */
+			((MM_AllocationContextBalanced *)spineContext)->lockCommon();
+		}
 
-	MM_AllocationContextTarok *spineContext = spineRegion->_allocateData._owningContext;
-	if (this != spineContext) {
-		Assert_MM_true(env->getCommonAllocationContext() == spineContext);
-		/* The common allocation context is always an instance of AllocationContextBalanced */
-		((MM_AllocationContextBalanced *)spineContext)->lockCommon();
+		leafAllocateData->addToArrayletLeafList(env, spineRegion);
+
+		if (this != spineContext) {
+			/* The common allocation context is always an instance of AllocationContextBalanced */
+			((MM_AllocationContextBalanced *)spineContext)->unlockCommon();
+		}
 	}
+#if defined(J9VM_GC_SPARSE_HEAP_ALLOCATION)
+	else {
+		/*
+		 * For now, only a single list of reserved regions owned by the common AC is maintained.
+		 * This is a sub-optimal approach, since due to different order of how regions are reserved and released
+		 * (objects don't die in order they are allocated) it may lead to imbalance of how many regions are committed in each AC.
+		 * In future, allocations should remember (somewhere in Off-heap meta structures) how many regions came from each AC
+		 * \and release exact same number back to each AC.
+		 */
+		MM_AllocationContextTarok *commonContext = (MM_AllocationContextTarok *)env->getCommonAllocationContext();
+		if (this != commonContext) {
+			/* The common allocation context is always an instance of AllocationContextBalanced */
+			((MM_AllocationContextBalanced *)commonContext)->lockCommon();
+		}
 
-	leafAllocateData->addToArrayletLeafList(spineRegion);
-	
-	if (this != spineContext) {
-		/* The common allocation context is always an instance of AllocationContextBalanced */
-		((MM_AllocationContextBalanced *)spineContext)->unlockCommon();
+		leafAllocateData->pushRegionToArrayReservedRegionList(env, ((MM_AllocationContextBalanced *)commonContext)->getArrayReservedRegionListAddress());
+		((MM_AllocationContextBalanced *)commonContext)->incrementArrayReservedRegionCount();
+
+		if (this != commonContext) {
+			/* The common allocation context is always an instance of AllocationContextBalanced */
+			((MM_AllocationContextBalanced *)commonContext)->unlockCommon();
+		}
 	}
+#endif /* defined(J9VM_GC_SPARSE_HEAP_ALLOCATION) */
 
+	freeRegionForArrayletLeaf->resetAge(env, (U_64)_subspace->getBytesRemainingBeforeTaxation());
 	/* store the base address of the leaf for the memset and the return */
 	return freeRegionForArrayletLeaf->getLowAddress();
 }
@@ -1073,3 +1102,33 @@ MM_AllocationContextBalanced::setNumaAffinityForThread(MM_EnvironmentBase *env)
 	return success;
 }
 
+#if defined(J9VM_GC_SPARSE_HEAP_ALLOCATION)
+void
+MM_AllocationContextBalanced::recycleReservedRegionsForVirtualLargeObjectHeap(MM_EnvironmentVLHGC *env, uintptr_t reservedRegionCount, bool needLock)
+{
+	MM_HeapRegionDescriptorVLHGC **head = getArrayReservedRegionListAddress();
+	MM_HeapRegionDescriptorVLHGC *region = NULL;
+
+	if (needLock) {
+		lockCommon();
+	}
+
+	while ((reservedRegionCount > 0) && (NULL != (region = *head))) {
+		region->_allocateData.popRegionFromArrayReservedRegionList(env, head);
+		decrementArrayReservedRegionCount();
+
+		/* Restore/Recommit the reserved region that have been previously decommitted. */
+		 MM_GCExtensions::getExtensions(env)->heap->commitMemory(region->getLowAddress(), _heapRegionManager->getRegionSize());
+
+		region->getSubSpace()->recycleRegion(env, region);
+		reservedRegionCount -= 1;
+	}
+
+	if (needLock) {
+		unlockCommon();
+	}
+
+	Assert_MM_true(0 == reservedRegionCount);
+}
+#endif /* defined(J9VM_GC_SPARSE_HEAP_ALLOCATION) */
+

runtime/gc_vlhgc/AllocationContextBalanced.hpp

@@ -71,6 +71,10 @@ class MM_AllocationContextBalanced : public MM_AllocationContextTarok
 	UDATA *_freeProcessorNodes;	/**< The array listing all the NUMA node numbers which account for the nodes with processors but no memory plus an empty slot for each context to use (element 0 is used by this context) - this is used when setting affinity */
 	UDATA _freeProcessorNodeCount;	/**< The length, in elements, of the _freeProcessorNodes array (always at least 1 after startup) */
 
+#if defined(J9VM_GC_SPARSE_HEAP_ALLOCATION)
+	MM_HeapRegionDescriptorVLHGC *_arrayReservedRegionList; /** for Off-heap case only, adding and removing via region->_allocateData.pushRegionToArrayReservedRegionList/popRegionFromArrayReservedRegionList */
+	uintptr_t _arrayReservedRegionCount;
+#endif /* defined(J9VM_GC_SPARSE_HEAP_ALLOCATION) */
 /* Methods */
 public:
 	static MM_AllocationContextBalanced *newInstance(MM_EnvironmentBase *env, MM_MemorySubSpaceTarok *subspace, UDATA numaNode, UDATA allocationContextNumber);
@@ -270,6 +274,34 @@ class MM_AllocationContextBalanced : public MM_AllocationContextTarok
 	 */
 	virtual bool setNumaAffinityForThread(MM_EnvironmentBase *env);
 
+#if defined(J9VM_GC_SPARSE_HEAP_ALLOCATION)
+	MM_HeapRegionDescriptorVLHGC **getArrayReservedRegionListAddress()
+	{
+		return &_arrayReservedRegionList;
+	}
+
+	uintptr_t getArrayReservedRegionCount()
+	{
+		return _arrayReservedRegionCount;
+	}
+
+	void setArrayReservedRegionCount(uintptr_t reservedRegionCount)
+	{
+		_arrayReservedRegionCount = reservedRegionCount;
+	}
+
+	void incrementArrayReservedRegionCount()
+	{
+		_arrayReservedRegionCount += 1;
+	}
+
+	void decrementArrayReservedRegionCount()
+	{
+		_arrayReservedRegionCount -= 1;
+	}
+
+	void recycleReservedRegionsForVirtualLargeObjectHeap(MM_EnvironmentVLHGC *env, uintptr_t reservedRegionCount, bool needLock = false);
+#endif /* defined(J9VM_GC_SPARSE_HEAP_ALLOCATION) */
 
 protected:
 	virtual void tearDown(MM_EnvironmentBase *env);
@@ -293,6 +325,10 @@ class MM_AllocationContextBalanced : public MM_AllocationContextTarok
 		, _heapRegionManager(NULL)
 		, _freeProcessorNodes(NULL)
 		, _freeProcessorNodeCount(0)
+#if defined(J9VM_GC_SPARSE_HEAP_ALLOCATION)
+		, _arrayReservedRegionList(NULL)
+		, _arrayReservedRegionCount(0)
+#endif /* defined(J9VM_GC_SPARSE_HEAP_ALLOCATION) */
 	{
 		_typeId = __FUNCTION__;
 	}