diff --git a/src/coreclr/nativeaot/Runtime/GCHelpers.cpp b/src/coreclr/nativeaot/Runtime/GCHelpers.cpp index 8820e8f90f04d4..939a637e66ee04 100644 --- a/src/coreclr/nativeaot/Runtime/GCHelpers.cpp +++ b/src/coreclr/nativeaot/Runtime/GCHelpers.cpp @@ -774,6 +774,14 @@ EXTERN_C void QCALLTYPE RhUnregisterFrozenSegment(void* pSegmentHandle) GCHeapUtilities::GetGCHeap()->UnregisterFrozenSegment((segment_handle)pSegmentHandle); } +// The MethodTable the GC uses to mark a range of memory as unused. It is needed to keep +// the frozen object heap walkable when it pads objects to align their data. +EXTERN_C void* QCALLTYPE RhGetFreeObjectMethodTable() +{ + ASSERT(g_pFreeObjectEEType != nullptr); + return g_pFreeObjectEEType; +} + FCIMPL1(uint32_t, RhGetGCDescSize, MethodTable* pMT) { if (!pMT->ContainsGCPointersOrCollectible()) diff --git a/src/coreclr/nativeaot/System.Private.CoreLib/src/Internal/Runtime/FrozenObjectHeapManager.cs b/src/coreclr/nativeaot/System.Private.CoreLib/src/Internal/Runtime/FrozenObjectHeapManager.cs index e9b41c447b84e5..747071b368b0cb 100644 --- a/src/coreclr/nativeaot/System.Private.CoreLib/src/Internal/Runtime/FrozenObjectHeapManager.cs +++ b/src/coreclr/nativeaot/System.Private.CoreLib/src/Internal/Runtime/FrozenObjectHeapManager.cs @@ -4,6 +4,7 @@ using System; using System.Runtime; using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; using System.Threading; using Debug = System.Diagnostics.Debug; @@ -19,10 +20,20 @@ internal unsafe partial class FrozenObjectHeapManager private readonly Lock m_Crst = new Lock(useTrivialWaits: true); private FrozenObjectSegment m_CurrentSegment; + // The MethodTable the GC uses to mark a range of memory as unused, it is what the + // gaps we leave to align the data of large arrays are formatted as. + private static readonly MethodTable* s_freeObjectMethodTable = + (MethodTable*)RuntimeImports.RhGetFreeObjectMethodTable(); + // Default size to reserve for a frozen segment private const nuint FOH_SEGMENT_DEFAULT_SIZE = 4 * 1024 * 1024; // Size to commit on demand in that reserved space private const nuint FOH_COMMIT_SIZE = 64 * 1024; + // The data of arrays of at least FOH_MIN_SIZE_TO_ALIGN bytes is put on a cache + // line. The threshold keeps the gap we leave in front of an array small relative + // to the array. + private const nuint FOH_DATA_ALIGNMENT = 64; + private const nuint FOH_MIN_SIZE_TO_ALIGN = 512; public T? TryAllocateObject() where T : class { @@ -162,6 +173,10 @@ public FrozenObjectSegment(nuint sizeHint) Debug.Assert(objectSize <= FOH_COMMIT_SIZE); Debug.Assert(m_pCurrent >= m_pStart + sizeof(ObjHeader)); + // Gap to leave in front of the object so that its data ends up aligned. + nuint pad = GetAlignmentPadding(type, objectSize); + nuint totalSize = pad + objectSize; + nuint spaceUsed = (nuint)(m_pCurrent - m_pStart); nuint spaceLeft = m_Size - spaceUsed; @@ -169,13 +184,13 @@ public FrozenObjectSegment(nuint sizeHint) Debug.Assert(spaceLeft >= (nuint)sizeof(ObjHeader)); // Test if we have a room for the given object (including extra sizeof(ObjHeader) for next object) - if (spaceLeft - (nuint)sizeof(ObjHeader) < objectSize) + if (spaceLeft - (nuint)sizeof(ObjHeader) < totalSize) { return null; } // Check if we need to commit a new chunk - if (spaceUsed + objectSize + (nuint)sizeof(ObjHeader) > m_SizeCommitted) + while (spaceUsed + totalSize + (nuint)sizeof(ObjHeader) > m_SizeCommitted) { // Make sure we don't go out of bounds during this commit Debug.Assert(m_SizeCommitted + FOH_COMMIT_SIZE <= m_Size); @@ -187,6 +202,15 @@ public FrozenObjectSegment(nuint sizeHint) m_SizeCommitted += FOH_COMMIT_SIZE; } + if (pad != 0) + { + // The gap has to stay walkable, so it is formatted as a free object. + HalfBakedObject* filler = (HalfBakedObject*)m_pCurrent; + filler->SetMethodTable(s_freeObjectMethodTable); + filler->SetNumComponents((uint)(pad - s_freeObjectMethodTable->BaseSize)); + m_pCurrent += pad; + } + HalfBakedObject* obj = (HalfBakedObject*)m_pCurrent; obj->SetMethodTable(type); @@ -196,12 +220,35 @@ public FrozenObjectSegment(nuint sizeHint) return obj; } + + // Arrays are accessed through their data, so it pays off to have that start on a + // cache line as long as the array is big enough to make up for the gap we leave + // in front of it. Strings are not worth it: their characters sit at an offset + // that can never be aligned. + private nuint GetAlignmentPadding(MethodTable* type, nuint objectSize) + { + if ((objectSize < FOH_MIN_SIZE_TO_ALIGN) || !type->IsArray) + { + return 0; + } + + // BaseSize includes the header, which is in front of the object. + nuint dataAddr = (nuint)m_pCurrent + type->BaseSize - (nuint)sizeof(ObjHeader); + nuint pad = (nuint)(-(nint)dataAddr) & (FOH_DATA_ALIGNMENT - 1); + + // The gap is formatted as a free object, so it can't be smaller than one. + nuint minFreeObjSize = s_freeObjectMethodTable->BaseSize; + return ((pad == 0) || (pad >= minFreeObjSize)) ? pad : (pad + FOH_DATA_ALIGNMENT); + } } + [StructLayout(LayoutKind.Sequential)] private struct HalfBakedObject { private MethodTable* _methodTable; + private uint _numComponents; public void SetMethodTable(MethodTable* methodTable) => _methodTable = methodTable; + public void SetNumComponents(uint numComponents) => _numComponents = numComponents; } } } diff --git a/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Runtime/RuntimeImports.cs b/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Runtime/RuntimeImports.cs index b174ba8e688077..ac9cead6eb14b1 100644 --- a/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Runtime/RuntimeImports.cs +++ b/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Runtime/RuntimeImports.cs @@ -150,6 +150,9 @@ internal static void RhReRegisterForFinalize(object obj) [LibraryImport(RuntimeLibrary)] internal static partial void RhUnregisterFrozenSegment(IntPtr pSegmentHandle); + [LibraryImport(RuntimeLibrary)] + internal static unsafe partial void* RhGetFreeObjectMethodTable(); + [MethodImpl(MethodImplOptions.InternalCall)] [RuntimeImport(RuntimeLibrary, "RhRegisterForFullGCNotification")] internal static extern bool RhRegisterForFullGCNotification(int maxGenerationThreshold, int largeObjectHeapThreshold); diff --git a/src/coreclr/vm/frozenobjectheap.cpp b/src/coreclr/vm/frozenobjectheap.cpp index a7fa25d29ccf21..8c17b3a913d725 100644 --- a/src/coreclr/vm/frozenobjectheap.cpp +++ b/src/coreclr/vm/frozenobjectheap.cpp @@ -8,6 +8,10 @@ #define FOH_SEGMENT_DEFAULT_SIZE (4 * 1024 * 1024) // Size to commit on demand in that reserved space #define FOH_COMMIT_SIZE (64 * 1024) +// The data of arrays of at least FOH_MIN_SIZE_TO_ALIGN bytes is put on a cache line. The +// threshold keeps the gap we leave in front of an array small relative to the array. +#define FOH_DATA_ALIGNMENT 64 +#define FOH_MIN_SIZE_TO_ALIGN 512 FrozenObjectHeapManager::FrozenObjectHeapManager(): m_Crst(CrstFrozenObjectHeap, CRST_UNSAFE_ANYMODE), @@ -213,6 +217,10 @@ Object* FrozenObjectSegment::TryAllocateObject(PTR_MethodTable type, size_t obje _ASSERT(objectSize <= FOH_COMMIT_SIZE); _ASSERT(m_pCurrent >= m_pStart + sizeof(ObjHeader)); + // Gap to leave in front of the object so that its data ends up aligned. + const size_t pad = GetAlignmentPadding(type, objectSize); + const size_t totalSize = pad + objectSize; + const size_t spaceUsed = (size_t)(m_pCurrent - m_pStart); const size_t spaceLeft = m_Size - spaceUsed; @@ -220,13 +228,13 @@ Object* FrozenObjectSegment::TryAllocateObject(PTR_MethodTable type, size_t obje _ASSERT(spaceLeft >= sizeof(ObjHeader)); // Test if we have a room for the given object (including extra sizeof(ObjHeader) for next object) - if (spaceLeft - sizeof(ObjHeader) < objectSize) + if (spaceLeft - sizeof(ObjHeader) < totalSize) { return nullptr; } // Check if we need to commit a new chunk - if (spaceUsed + objectSize + sizeof(ObjHeader) > m_SizeCommitted) + while (spaceUsed + totalSize + sizeof(ObjHeader) > m_SizeCommitted) { // Make sure we don't go out of bounds during this commit _ASSERT(m_SizeCommitted + FOH_COMMIT_SIZE <= m_Size); @@ -238,6 +246,16 @@ Object* FrozenObjectSegment::TryAllocateObject(PTR_MethodTable type, size_t obje m_SizeCommitted += FOH_COMMIT_SIZE; } + if (pad != 0) + { + // The gap has to stay walkable, so it is formatted as a free object. + ArrayBase* filler = reinterpret_cast(m_pCurrent); + filler->SetMethodTable(g_pFreeObjectMethodTable); + filler->SetNumComponents((INT32)(pad - g_pFreeObjectMethodTable->GetBaseSize())); + _ASSERT(filler->GetSize() == pad); + m_pCurrent += pad; + } + Object* object = reinterpret_cast(m_pCurrent); object->SetMethodTable(type); @@ -246,14 +264,42 @@ Object* FrozenObjectSegment::TryAllocateObject(PTR_MethodTable type, size_t obje return object; } -Object* FrozenObjectSegment::GetFirstObject() const +// Arrays are accessed through their data, so it pays off to have that start on a cache +// line as long as the array is big enough to make up for the gap we leave in front of it. +// Strings are not worth it: their characters sit at an offset that can never be aligned. +size_t FrozenObjectSegment::GetAlignmentPadding(PTR_MethodTable type, size_t objectSize) const { - if (m_pStart + sizeof(ObjHeader) == m_pCurrent) + if ((objectSize < FOH_MIN_SIZE_TO_ALIGN) || !type->IsArray()) { - // Segment is empty - return nullptr; + return 0; } - return reinterpret_cast(m_pStart + sizeof(ObjHeader)); + + const size_t dataAddr = (size_t)m_pCurrent + ArrayBase::GetDataPtrOffset(type); + const size_t pad = ALIGN_UP(dataAddr, (size_t)FOH_DATA_ALIGNMENT) - dataAddr; + + // The gap is formatted as a free object, so it can't be smaller than one. + const size_t minFreeObjSize = g_pFreeObjectMethodTable->GetBaseSize(); + return ((pad == 0) || (pad >= minFreeObjSize)) ? pad : (pad + FOH_DATA_ALIGNMENT); +} + +// Returns obj, or the first real object after it. The gaps we leave to align array data +// are formatted as free objects and are not something to hand out - gc_heap::walk_heap +// skips them the same way. Returns nullptr once the end of the segment is reached. +Object* FrozenObjectSegment::SkipFreeObjects(uint8_t* obj) const +{ + // FOH doesn't support objects with non-DATA_ALIGNMENT alignment yet. + while ((obj < m_pCurrent) && + (reinterpret_cast(obj)->GetGCSafeMethodTable() == g_pFreeObjectMethodTable)) + { + obj += ALIGN_UP(reinterpret_cast(obj)->GetSize(), DATA_ALIGNMENT); + } + + return (obj < m_pCurrent) ? reinterpret_cast(obj) : nullptr; +} + +Object* FrozenObjectSegment::GetFirstObject() const +{ + return SkipFreeObjects(m_pStart + sizeof(ObjHeader)); } Object* FrozenObjectSegment::GetNextObject(Object* obj) const @@ -262,13 +308,5 @@ Object* FrozenObjectSegment::GetNextObject(Object* obj) const _ASSERT(obj != nullptr); _ASSERT((uint8_t*)obj >= m_pStart + sizeof(ObjHeader) && (uint8_t*)obj < m_pCurrent); - // FOH doesn't support objects with non-DATA_ALIGNMENT alignment yet. - uint8_t* nextObj = (reinterpret_cast(obj) + ALIGN_UP(obj->GetSize(), DATA_ALIGNMENT)); - if (nextObj < m_pCurrent) - { - return reinterpret_cast(nextObj); - } - - // Current object is the last one in the segment - return nullptr; + return SkipFreeObjects(reinterpret_cast(obj) + ALIGN_UP(obj->GetSize(), DATA_ALIGNMENT)); } diff --git a/src/coreclr/vm/frozenobjectheap.h b/src/coreclr/vm/frozenobjectheap.h index e191731d64dd5b..cb675a475a0a84 100644 --- a/src/coreclr/vm/frozenobjectheap.h +++ b/src/coreclr/vm/frozenobjectheap.h @@ -50,6 +50,8 @@ class FrozenObjectSegment private: Object* GetFirstObject() const; Object* GetNextObject(Object* obj) const; + Object* SkipFreeObjects(uint8_t* obj) const; + size_t GetAlignmentPadding(PTR_MethodTable type, size_t objectSize) const; // Start of the reserved memory, the first object starts at "m_pStart + sizeof(ObjHeader)" (its pMT) uint8_t* m_pStart;