Align the data of frozen arrays

src/coreclr/nativeaot/Runtime/GCHelpers.cpp

@@ -774,6 +774,14 @@ EXTERN_C void QCALLTYPE RhUnregisterFrozenSegment(void* pSegmentHandle)
     GCHeapUtilities::GetGCHeap()->UnregisterFrozenSegment((segment_handle)pSegmentHandle);
 }
 
+// The MethodTable the GC uses to mark a range of memory as unused. It is needed to keep
+// the frozen object heap walkable when it pads objects to align their data.
+EXTERN_C void* QCALLTYPE RhGetFreeObjectMethodTable()
+{
+    ASSERT(g_pFreeObjectEEType != nullptr);
+    return g_pFreeObjectEEType;
+}
+
 FCIMPL1(uint32_t, RhGetGCDescSize, MethodTable* pMT)
 {
     if (!pMT->ContainsGCPointersOrCollectible())

src/coreclr/nativeaot/System.Private.CoreLib/src/Internal/Runtime/FrozenObjectHeapManager.cs

@@ -4,6 +4,7 @@
 using System;
 using System.Runtime;
 using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices;
 using System.Threading;
 
 using Debug = System.Diagnostics.Debug;
@@ -19,10 +20,23 @@ internal unsafe partial class FrozenObjectHeapManager
         private readonly Lock m_Crst = new Lock(useTrivialWaits: true);
         private FrozenObjectSegment m_CurrentSegment;
 
+        // The MethodTable the GC uses to mark a range of memory as unused, it is what the
+        // gaps we leave to align the data of large arrays are formatted as.
+        private static readonly MethodTable* s_freeObjectMethodTable =
+            (MethodTable*)RuntimeImports.RhGetFreeObjectMethodTable();
+
         // Default size to reserve for a frozen segment
         private const nuint FOH_SEGMENT_DEFAULT_SIZE = 4 * 1024 * 1024;
         // Size to commit on demand in that reserved space
         private const nuint FOH_COMMIT_SIZE = 64 * 1024;
+        // The data of arrays at least this big is aligned on FOH_DATA_ALIGNMENT, which is
+        // the widest vector the platform can load from it.
+        private const nuint FOH_MIN_SIZE_TO_ALIGN = 128;
+#if TARGET_ARM64
+        private const nuint FOH_DATA_ALIGNMENT = 16;
+#else
+        private const nuint FOH_DATA_ALIGNMENT = 64;
+#endif
 
         public T? TryAllocateObject<T>() where T : class
         {
@@ -162,20 +176,24 @@ public FrozenObjectSegment(nuint sizeHint)
                 Debug.Assert(objectSize <= FOH_COMMIT_SIZE);
                 Debug.Assert(m_pCurrent >= m_pStart + sizeof(ObjHeader));
 
+                // Gap to leave in front of the object so that its data ends up aligned.
+                nuint pad = GetAlignmentPadding(type, objectSize);
+                nuint totalSize = pad + objectSize;
+
                 nuint spaceUsed = (nuint)(m_pCurrent - m_pStart);
                 nuint spaceLeft = m_Size - spaceUsed;
 
                 Debug.Assert(spaceUsed >= (nuint)sizeof(ObjHeader));
                 Debug.Assert(spaceLeft >= (nuint)sizeof(ObjHeader));
 
                 // Test if we have a room for the given object (including extra sizeof(ObjHeader) for next object)
-                if (spaceLeft - (nuint)sizeof(ObjHeader) < objectSize)
+                if (spaceLeft - (nuint)sizeof(ObjHeader) < totalSize)
                 {
                     return null;
                 }
 
                 // Check if we need to commit a new chunk
-                if (spaceUsed + objectSize + (nuint)sizeof(ObjHeader) > m_SizeCommitted)
+                while (spaceUsed + totalSize + (nuint)sizeof(ObjHeader) > m_SizeCommitted)
                 {
                     // Make sure we don't go out of bounds during this commit
                     Debug.Assert(m_SizeCommitted + FOH_COMMIT_SIZE <= m_Size);
@@ -187,6 +205,15 @@ public FrozenObjectSegment(nuint sizeHint)
                     m_SizeCommitted += FOH_COMMIT_SIZE;
                 }
 
+                if (pad != 0)
+                {
+                    // The gap has to stay walkable, so it is formatted as a free object.
+                    HalfBakedObject* filler = (HalfBakedObject*)m_pCurrent;
+                    filler->SetMethodTable(s_freeObjectMethodTable);
+                    filler->SetNumComponents((uint)(pad - s_freeObjectMethodTable->BaseSize));
+                    m_pCurrent += pad;
+                }
+
                 HalfBakedObject* obj = (HalfBakedObject*)m_pCurrent;
                 obj->SetMethodTable(type);
 
@@ -196,12 +223,35 @@ public FrozenObjectSegment(nuint sizeHint)
 
                 return obj;
             }
+
+            // Arrays are accessed through their data, so it pays off to have that start on a
+            // cache line as long as the array is big enough to make up for the gap we leave
+            // in front of it. Strings are not worth it: their characters sit at an offset
+            // that can never be aligned.
+            private nuint GetAlignmentPadding(MethodTable* type, nuint objectSize)
+            {
+                if ((objectSize < FOH_MIN_SIZE_TO_ALIGN) || !type->IsArray)
+                {
+                    return 0;
+                }
+
+                // BaseSize includes the header, which is in front of the object.
+                nuint dataAddr = (nuint)m_pCurrent + type->BaseSize - (nuint)sizeof(ObjHeader);
+                nuint pad = (nuint)(-(nint)dataAddr) & (FOH_DATA_ALIGNMENT - 1);
+
+                // The gap is formatted as a free object, so it can't be smaller than one.
+                nuint minFreeObjSize = s_freeObjectMethodTable->BaseSize;
+                return ((pad == 0) || (pad >= minFreeObjSize)) ? pad : (pad + FOH_DATA_ALIGNMENT);
+            }
         }
 
+        [StructLayout(LayoutKind.Sequential)]
         private struct HalfBakedObject
         {
             private MethodTable* _methodTable;
+            private uint _numComponents;
             public void SetMethodTable(MethodTable* methodTable) => _methodTable = methodTable;
+            public void SetNumComponents(uint numComponents) => _numComponents = numComponents;
         }
     }
 }

src/coreclr/nativeaot/System.Private.CoreLib/src/System/Runtime/RuntimeImports.cs

@@ -150,6 +150,9 @@ internal static void RhReRegisterForFinalize(object obj)
         [LibraryImport(RuntimeLibrary)]
         internal static partial void RhUnregisterFrozenSegment(IntPtr pSegmentHandle);
 
+        [LibraryImport(RuntimeLibrary)]
+        internal static unsafe partial void* RhGetFreeObjectMethodTable();
+
         [MethodImpl(MethodImplOptions.InternalCall)]
         [RuntimeImport(RuntimeLibrary, "RhRegisterForFullGCNotification")]
         internal static extern bool RhRegisterForFullGCNotification(int maxGenerationThreshold, int largeObjectHeapThreshold);

src/coreclr/vm/frozenobjectheap.cpp

@@ -8,6 +8,14 @@
 #define FOH_SEGMENT_DEFAULT_SIZE (4 * 1024 * 1024)
 // Size to commit on demand in that reserved space
 #define FOH_COMMIT_SIZE (64 * 1024)
+// The data of arrays at least this big is aligned on FOH_DATA_ALIGNMENT, which is the
+// widest vector the platform can load from it.
+#define FOH_MIN_SIZE_TO_ALIGN 128
+#ifdef TARGET_ARM64
+#define FOH_DATA_ALIGNMENT 16
+#else
+#define FOH_DATA_ALIGNMENT 64
+#endif
 
 FrozenObjectHeapManager::FrozenObjectHeapManager():
     m_Crst(CrstFrozenObjectHeap, CRST_UNSAFE_ANYMODE),
@@ -213,20 +221,24 @@ Object* FrozenObjectSegment::TryAllocateObject(PTR_MethodTable type, size_t obje
     _ASSERT(objectSize <= FOH_COMMIT_SIZE);
     _ASSERT(m_pCurrent >= m_pStart + sizeof(ObjHeader));
 
+    // Gap to leave in front of the object so that its data ends up aligned.
+    const size_t pad = GetAlignmentPadding(type, objectSize);
+    const size_t totalSize = pad + objectSize;
+
     const size_t spaceUsed = (size_t)(m_pCurrent - m_pStart);
     const size_t spaceLeft = m_Size - spaceUsed;
 
     _ASSERT(spaceUsed >= sizeof(ObjHeader));
     _ASSERT(spaceLeft >= sizeof(ObjHeader));
 
     // Test if we have a room for the given object (including extra sizeof(ObjHeader) for next object)
-    if (spaceLeft - sizeof(ObjHeader) < objectSize)
+    if (spaceLeft - sizeof(ObjHeader) < totalSize)
     {
         return nullptr;
     }
 
     // Check if we need to commit a new chunk
-    if (spaceUsed + objectSize + sizeof(ObjHeader) > m_SizeCommitted)
+    while (spaceUsed + totalSize + sizeof(ObjHeader) > m_SizeCommitted)
     {
         // Make sure we don't go out of bounds during this commit
         _ASSERT(m_SizeCommitted + FOH_COMMIT_SIZE <= m_Size);
@@ -238,6 +250,16 @@ Object* FrozenObjectSegment::TryAllocateObject(PTR_MethodTable type, size_t obje
         m_SizeCommitted += FOH_COMMIT_SIZE;
     }
 
+    if (pad != 0)
+    {
+        // The gap has to stay walkable, so it is formatted as a free object.
+        ArrayBase* filler = reinterpret_cast<ArrayBase*>(m_pCurrent);
+        filler->SetMethodTable(g_pFreeObjectMethodTable);
+        filler->SetNumComponents((INT32)(pad - g_pFreeObjectMethodTable->GetBaseSize()));
+        _ASSERT(filler->GetSize() == pad);
+        m_pCurrent += pad;
+    }
+
     Object* object = reinterpret_cast<Object*>(m_pCurrent);
     object->SetMethodTable(type);
 
@@ -246,14 +268,42 @@ Object* FrozenObjectSegment::TryAllocateObject(PTR_MethodTable type, size_t obje
     return object;
 }
 
-Object* FrozenObjectSegment::GetFirstObject() const
+// Arrays are accessed through their data, so it pays off to have that start on a cache
+// line as long as the array is big enough to make up for the gap we leave in front of it.
+// Strings are not worth it: their characters sit at an offset that can never be aligned.
+size_t FrozenObjectSegment::GetAlignmentPadding(PTR_MethodTable type, size_t objectSize) const
 {
-    if (m_pStart + sizeof(ObjHeader) == m_pCurrent)
+    if ((objectSize < FOH_MIN_SIZE_TO_ALIGN) || !type->IsArray())
     {
-        // Segment is empty
-        return nullptr;
+        return 0;
+    }
+
+    const size_t dataAddr = (size_t)m_pCurrent + ArrayBase::GetDataPtrOffset(type);
+    const size_t pad = ALIGN_UP(dataAddr, (size_t)FOH_DATA_ALIGNMENT) - dataAddr;
+
+    // The gap is formatted as a free object, so it can't be smaller than one.
+    const size_t minFreeObjSize = g_pFreeObjectMethodTable->GetBaseSize();
+    return ((pad == 0) || (pad >= minFreeObjSize)) ? pad : (pad + FOH_DATA_ALIGNMENT);
+}
+
+// Returns obj, or the first real object after it. The gaps we leave to align array data
+// are formatted as free objects and are not something to hand out - gc_heap::walk_heap
+// skips them the same way. Returns nullptr once the end of the segment is reached.
+Object* FrozenObjectSegment::SkipFreeObjects(uint8_t* obj) const
+{
+    // FOH doesn't support objects with non-DATA_ALIGNMENT alignment yet.
+    while ((obj < m_pCurrent) &&
+           (reinterpret_cast<Object*>(obj)->GetGCSafeMethodTable() == g_pFreeObjectMethodTable))
+    {
+        obj += ALIGN_UP(reinterpret_cast<Object*>(obj)->GetSize(), DATA_ALIGNMENT);
     }
-    return reinterpret_cast<Object*>(m_pStart + sizeof(ObjHeader));
+
+    return (obj < m_pCurrent) ? reinterpret_cast<Object*>(obj) : nullptr;
+}
+
+Object* FrozenObjectSegment::GetFirstObject() const
+{
+    return SkipFreeObjects(m_pStart + sizeof(ObjHeader));
 }
 
 Object* FrozenObjectSegment::GetNextObject(Object* obj) const
@@ -262,13 +312,5 @@ Object* FrozenObjectSegment::GetNextObject(Object* obj) const
     _ASSERT(obj != nullptr);
     _ASSERT((uint8_t*)obj >= m_pStart + sizeof(ObjHeader) && (uint8_t*)obj < m_pCurrent);
 
-    // FOH doesn't support objects with non-DATA_ALIGNMENT alignment yet.
-    uint8_t* nextObj = (reinterpret_cast<uint8_t*>(obj) + ALIGN_UP(obj->GetSize(), DATA_ALIGNMENT));
-    if (nextObj < m_pCurrent)
-    {
-        return reinterpret_cast<Object*>(nextObj);
-    }
-
-    // Current object is the last one in the segment
-    return nullptr;
+    return SkipFreeObjects(reinterpret_cast<uint8_t*>(obj) + ALIGN_UP(obj->GetSize(), DATA_ALIGNMENT));
 }

src/coreclr/vm/frozenobjectheap.h

@@ -50,6 +50,8 @@ class FrozenObjectSegment
 private:
     Object* GetFirstObject() const;
     Object* GetNextObject(Object* obj) const;
+    Object* SkipFreeObjects(uint8_t* obj) const;
+    size_t GetAlignmentPadding(PTR_MethodTable type, size_t objectSize) const;
 
     // Start of the reserved memory, the first object starts at "m_pStart + sizeof(ObjHeader)" (its pMT)
     uint8_t* m_pStart;