From 03b3f58f0182de9a4bf0663c38ddac668fb0a1b9 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Fri, 24 Jul 2026 17:56:36 +0200 Subject: [PATCH 01/19] TODO --- src/coreclr/jit/async.cpp | 67 +++++++++++++++++++++++++++++++++++++ src/coreclr/jit/async.h | 18 ++++++++++ src/coreclr/jit/compiler.h | 5 +++ src/coreclr/jit/gentree.cpp | 10 ++++++ src/coreclr/jit/gtlist.h | 6 ++++ 5 files changed, 106 insertions(+) diff --git a/src/coreclr/jit/async.cpp b/src/coreclr/jit/async.cpp index 07be2d280a3aa6..cdf5863b794e77 100644 --- a/src/coreclr/jit/async.cpp +++ b/src/coreclr/jit/async.cpp @@ -45,6 +45,73 @@ #include "jitstd/algorithm.h" #include "async.h" +ContinuationMember ContinuationMember::CustomAwaiterOfLayout(ClassLayout* layout) +{ + ContinuationMember member; + member.Type = ContinuationMemberType::CustomAwaiterOfLayout; + member.m_customAwaiterLayout = layout; + return member; +} + +bool ContinuationMember::AreCompatible(const ContinuationMember& a, const ContinuationMember& b) +{ + if (a.Type != b.Type) + { + return false; + } + + switch (a.Type) + { + case ContinuationMemberType::CustomAwaiterOfLayout: + return ClassLayout::AreCompatible(a.m_customAwaiterLayout, b.m_customAwaiterLayout); + default: + unreached(); + } +} + +#ifdef DEBUG +void ContinuationMember::Print() const +{ + switch (Type) + { + case ContinuationMemberType::CustomAwaiterOfLayout: + printf("CustomAwaiter<%s>", m_customAwaiterLayout->GetClassName()); + break; + default: + unreached(); + } +} +#endif + +size_t Compiler::GetContinuationMemberIndex(const ContinuationMember& member) +{ + if (m_asyncContinuationMembers == nullptr) + { + m_asyncContinuationMembers = new (this, CMK_Async) jitstd::vector(getAllocator(CMK_Async)); + } + else + { + for (size_t i = 0; i < m_asyncContinuationMembers->size(); i++) + { + const ContinuationMember& existingMember = m_asyncContinuationMembers->at(i); + + if (ContinuationMember::AreCompatible(member, existingMember)) + { + return i; + } + } + } + + m_asyncContinuationMembers->push_back(member); + return m_asyncContinuationMembers->size() - 1; +} + +const ContinuationMember& Compiler::GetContinuationMember(size_t index) +{ + assert(index < m_asyncContinuationMembers->size()); + return m_asyncContinuationMembers->at(index); +} + //------------------------------------------------------------------------ // Compiler::SaveAsyncContexts: // Insert code in async methods that saves and restores contexts. diff --git a/src/coreclr/jit/async.h b/src/coreclr/jit/async.h index fb9abf2d8be779..e1ef0bddd3d7f6 100644 --- a/src/coreclr/jit/async.h +++ b/src/coreclr/jit/async.h @@ -1,6 +1,24 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +enum class ContinuationMemberType +{ + CustomAwaiterOfLayout, +}; + +struct ContinuationMember +{ + ContinuationMemberType Type; + + void Print() const; +private: + ClassLayout* m_customAwaiterLayout; + +public: + static ContinuationMember CustomAwaiterOfLayout(ClassLayout* layout); + static bool AreCompatible(const ContinuationMember& a, const ContinuationMember& b); +}; + struct ReturnTypeInfo { var_types ReturnType = TYP_UNDEF; diff --git a/src/coreclr/jit/compiler.h b/src/coreclr/jit/compiler.h index 59de8d694033d9..9009f598d602c6 100644 --- a/src/coreclr/jit/compiler.h +++ b/src/coreclr/jit/compiler.h @@ -95,6 +95,7 @@ enum class WasmValueType : unsigned; #ifdef DEBUG struct IndentStack; #endif +struct ContinuationMember; class Lowering; // defined in lower.h @@ -6237,6 +6238,10 @@ class Compiler PhaseStatus placeLoopAlignInstructions(); #endif + jitstd::vector* m_asyncContinuationMembers = nullptr; + size_t GetContinuationMemberIndex(const ContinuationMember& member); + const ContinuationMember& GetContinuationMember(size_t index); + PhaseStatus SaveAsyncContexts(); void AddContextArgsToAsyncCalls(BasicBlock* block); BasicBlock* CreateReturnBB(unsigned* mergedReturnLcl); diff --git a/src/coreclr/jit/gentree.cpp b/src/coreclr/jit/gentree.cpp index f050c5eeeee885..1effed3647c539 100644 --- a/src/coreclr/jit/gentree.cpp +++ b/src/coreclr/jit/gentree.cpp @@ -13,6 +13,7 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX #include "jitpch.h" #include "hwintrinsic.h" #include "simd.h" +#include "async.h" #ifdef _MSC_VER #pragma hdrstop @@ -14008,6 +14009,15 @@ void Compiler::gtDispLeaf(GenTree* tree, IndentStack* indentStack) case GT_WASM_JEXCEPT: break; + case GT_CONTINUATION_MEMBER_OFFSET: + { + printf(" index=%zu ", tree->AsVal()->gtVal1); + + const ContinuationMember& member = GetContinuationMember(tree->AsVal()->gtVal1); + member.Print(); + printf("\n"); + } + case GT_RET_EXPR: { GenTreeCall* inlineCand = tree->AsRetExpr()->gtInlineCandidate; diff --git a/src/coreclr/jit/gtlist.h b/src/coreclr/jit/gtlist.h index be81ca9f22052d..cfc65897703c5e 100644 --- a/src/coreclr/jit/gtlist.h +++ b/src/coreclr/jit/gtlist.h @@ -41,6 +41,12 @@ GTNODE(GCPOLL , GenTree ,0,0,GTK_LEAF|GTK_NOVALUE|DBK_NOTLI GTNODE(ASYNC_RESUME_INFO, GenTreeVal ,0,0,GTK_LEAF) // Address of async resume info for a state GTNODE(FTN_ENTRY , GenTree ,0,0,GTK_LEAF) // Address of this function's entry point +// Offset into continuation, starting after the MethodTable*, of a special +// member. GenTreeVal::gtVal1 is an index into Compiler::m_continuationMembers. +// The async transformation allocates and lays out the continuation and +// substitutes these by constants at that time. +GTNODE(CONTINUATION_MEMBER_OFFSET, GenTreeVal ,0,0,GTK_LEAF) + //----------------------------------------------------------------------------- // Constant nodes: //----------------------------------------------------------------------------- From fa613ba15b7281872b1a89b3a18591704c3a66aa Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Fri, 24 Jul 2026 19:25:21 +0200 Subject: [PATCH 02/19] WIP --- src/coreclr/inc/corinfo.h | 8 ++ src/coreclr/inc/icorjitinfoimpl_generated.h | 7 + src/coreclr/inc/jiteeversionguid.h | 10 +- src/coreclr/jit/ICorJitInfo_names_generated.h | 1 + .../jit/ICorJitInfo_wrapper_generated.hpp | 13 ++ src/coreclr/jit/async.cpp | 132 ++++++++++++++++-- src/coreclr/jit/async.h | 10 +- src/coreclr/jit/compiler.h | 1 + src/coreclr/jit/gentree.cpp | 2 + src/coreclr/jit/gentree.h | 1 + src/coreclr/jit/gtlist.h | 2 +- src/coreclr/jit/gtstructs.h | 2 +- src/coreclr/jit/importercalls.cpp | 48 +++++++ src/coreclr/jit/wellknownargs.h | 1 + .../tools/Common/JitInterface/CorInfoImpl.cs | 56 ++++++++ .../JitInterface/CorInfoImpl_generated.cs | 17 +++ .../ThunkGenerator/ThunkInput.txt | 1 + .../Compiler/ReadyToRunCodegenCompilation.cs | 3 + .../aot/jitinterface/jitinterface_generated.h | 14 ++ .../tools/superpmi/superpmi-shared/agnostic.h | 7 + .../tools/superpmi/superpmi-shared/lwmlist.h | 1 + .../superpmi-shared/methodcontext.cpp | 67 +++++++++ .../superpmi/superpmi-shared/methodcontext.h | 15 ++ .../superpmi-shim-collector/icorjitinfo.cpp | 14 ++ .../icorjitinfo_generated.cpp | 11 ++ .../icorjitinfo_generated.cpp | 10 ++ .../tools/superpmi/superpmi/icorjitinfo.cpp | 12 ++ src/coreclr/vm/corelib.h | 2 + src/coreclr/vm/jitinterface.cpp | 113 +++++++++++++++ src/coreclr/vm/jitinterface.h | 5 + 30 files changed, 566 insertions(+), 20 deletions(-) diff --git a/src/coreclr/inc/corinfo.h b/src/coreclr/inc/corinfo.h index cd6b359c9f5d76..f35be06c2cf136 100644 --- a/src/coreclr/inc/corinfo.h +++ b/src/coreclr/inc/corinfo.h @@ -3154,6 +3154,14 @@ class ICorStaticInfo // instantiation argument that must be passed to the await call. virtual CORINFO_METHOD_HANDLE getAwaitReturnCall(CORINFO_METHOD_HANDLE callerHandle, CORINFO_CONTEXT_HANDLE* contextHandle, CORINFO_LOOKUP* instArg) = 0; + virtual CORINFO_METHOD_HANDLE getAwaitAwaiterInContinuationCall( + CORINFO_METHOD_HANDLE callerHandle, + CORINFO_SIG_INFO* callSig, + bool isUnsafe, + CORINFO_CONTEXT_HANDLE* contextHandle, + CORINFO_LOOKUP* instArg + ) = 0; + /*********************************************************************************/ // // Diagnostic methods diff --git a/src/coreclr/inc/icorjitinfoimpl_generated.h b/src/coreclr/inc/icorjitinfoimpl_generated.h index 6c0c39f71f0293..829e1073cd8d25 100644 --- a/src/coreclr/inc/icorjitinfoimpl_generated.h +++ b/src/coreclr/inc/icorjitinfoimpl_generated.h @@ -503,6 +503,13 @@ CORINFO_METHOD_HANDLE getAwaitReturnCall( CORINFO_CONTEXT_HANDLE* contextHandle, CORINFO_LOOKUP* instArg) override; +CORINFO_METHOD_HANDLE getAwaitAwaiterInContinuationCall( + CORINFO_METHOD_HANDLE callerHandle, + CORINFO_SIG_INFO* callSig, + bool isUnsafe, + CORINFO_CONTEXT_HANDLE* contextHandle, + CORINFO_LOOKUP* instArg) override; + mdMethodDef getMethodDefFromMethod( CORINFO_METHOD_HANDLE hMethod) override; diff --git a/src/coreclr/inc/jiteeversionguid.h b/src/coreclr/inc/jiteeversionguid.h index 08a3bcde5744d6..cb3210e624b6a2 100644 --- a/src/coreclr/inc/jiteeversionguid.h +++ b/src/coreclr/inc/jiteeversionguid.h @@ -37,11 +37,11 @@ #include -constexpr GUID JITEEVersionIdentifier = { /* 5bb9b1e5-9941-4762-a195-351b6a736588 */ - 0x5bb9b1e5, - 0x9941, - 0x4762, - {0xa1, 0x95, 0x35, 0x1b, 0x6a, 0x73, 0x65, 0x88} +constexpr GUID JITEEVersionIdentifier = { /* 364f7f40-1feb-4b18-bc70-a3cb6481665b */ + 0x364f7f40, + 0x1feb, + 0x4b18, + {0xbc, 0x70, 0xa3, 0xcb, 0x64, 0x81, 0x66, 0x5b} }; #endif // JIT_EE_VERSIONING_GUID_H diff --git a/src/coreclr/jit/ICorJitInfo_names_generated.h b/src/coreclr/jit/ICorJitInfo_names_generated.h index b3ccde2acebc8f..bd0c0dd01da10f 100644 --- a/src/coreclr/jit/ICorJitInfo_names_generated.h +++ b/src/coreclr/jit/ICorJitInfo_names_generated.h @@ -125,6 +125,7 @@ DEF_CLR_API(runWithSPMIErrorTrap) DEF_CLR_API(getEEInfo) DEF_CLR_API(getAsyncInfo) DEF_CLR_API(getAwaitReturnCall) +DEF_CLR_API(getAwaitAwaiterInContinuationCall) DEF_CLR_API(getMethodDefFromMethod) DEF_CLR_API(printMethodName) DEF_CLR_API(getMethodNameFromMetadata) diff --git a/src/coreclr/jit/ICorJitInfo_wrapper_generated.hpp b/src/coreclr/jit/ICorJitInfo_wrapper_generated.hpp index 3f40e71e6bbb26..bec4c91776ee52 100644 --- a/src/coreclr/jit/ICorJitInfo_wrapper_generated.hpp +++ b/src/coreclr/jit/ICorJitInfo_wrapper_generated.hpp @@ -1190,6 +1190,19 @@ CORINFO_METHOD_HANDLE WrapICorJitInfo::getAwaitReturnCall( return temp; } +CORINFO_METHOD_HANDLE WrapICorJitInfo::getAwaitAwaiterInContinuationCall( + CORINFO_METHOD_HANDLE callerHandle, + CORINFO_SIG_INFO* callSig, + bool isUnsafe, + CORINFO_CONTEXT_HANDLE* contextHandle, + CORINFO_LOOKUP* instArg) +{ + API_ENTER(getAwaitAwaiterInContinuationCall); + CORINFO_METHOD_HANDLE temp = wrapHnd->getAwaitAwaiterInContinuationCall(callerHandle, callSig, isUnsafe, contextHandle, instArg); + API_LEAVE(getAwaitAwaiterInContinuationCall); + return temp; +} + mdMethodDef WrapICorJitInfo::getMethodDefFromMethod( CORINFO_METHOD_HANDLE hMethod) { diff --git a/src/coreclr/jit/async.cpp b/src/coreclr/jit/async.cpp index b5aac33248ab4e..0096f2a911cc5b 100644 --- a/src/coreclr/jit/async.cpp +++ b/src/coreclr/jit/async.cpp @@ -48,11 +48,17 @@ ContinuationMember ContinuationMember::CustomAwaiterOfLayout(ClassLayout* layout) { ContinuationMember member; - member.Type = ContinuationMemberType::CustomAwaiterOfLayout; + member.Type = ContinuationMemberType::CustomAwaiterOfLayout; member.m_customAwaiterLayout = layout; return member; } +ClassLayout* ContinuationMember::GetCustomAwaiterLayout() const +{ + assert(Type == ContinuationMemberType::CustomAwaiterOfLayout); + return m_customAwaiterLayout; +} + bool ContinuationMember::AreCompatible(const ContinuationMember& a, const ContinuationMember& b) { if (a.Type != b.Type) @@ -62,10 +68,10 @@ bool ContinuationMember::AreCompatible(const ContinuationMember& a, const Contin switch (a.Type) { - case ContinuationMemberType::CustomAwaiterOfLayout: - return ClassLayout::AreCompatible(a.m_customAwaiterLayout, b.m_customAwaiterLayout); - default: - unreached(); + case ContinuationMemberType::CustomAwaiterOfLayout: + return ClassLayout::AreCompatible(a.m_customAwaiterLayout, b.m_customAwaiterLayout); + default: + unreached(); } } @@ -74,11 +80,11 @@ void ContinuationMember::Print() const { switch (Type) { - case ContinuationMemberType::CustomAwaiterOfLayout: - printf("CustomAwaiter<%s>", m_customAwaiterLayout->GetClassName()); - break; - default: - unreached(); + case ContinuationMemberType::CustomAwaiterOfLayout: + printf("CustomAwaiter<%s>", m_customAwaiterLayout->GetClassName()); + break; + default: + unreached(); } } #endif @@ -106,6 +112,11 @@ size_t Compiler::GetContinuationMemberIndex(const ContinuationMember& member) return m_asyncContinuationMembers->size() - 1; } +size_t Compiler::GetContinuationMemberCount() const +{ + return m_asyncContinuationMembers == nullptr ? 0 : m_asyncContinuationMembers->size(); +} + const ContinuationMember& Compiler::GetContinuationMember(size_t index) { assert(index < m_asyncContinuationMembers->size()); @@ -1430,6 +1441,13 @@ void ContinuationLayout::Dump(int indent) printf("%*s +%03u Keep alive object\n", indent, "", KeepAliveOffset); } + for (size_t i = 0; i < ContinuationMemberOffsets.size(); i++) + { + printf("%*s +%03u ", indent, "", ContinuationMemberOffsets[i]); + JitTls::GetCompiler()->GetContinuationMember(i).Print(); + printf("\n"); + } + for (const LiveLocalInfo& inf : Locals) { printf("%*s +%03u V%02u: %u bytes\n", indent, "", inf.Offset, inf.LclNum, inf.Size); @@ -1484,6 +1502,7 @@ ContinuationLayout* ContinuationLayoutBuilder::Create() { ContinuationLayout* layout = new (m_compiler, CMK_Async) ContinuationLayout(m_compiler); layout->Locals.reserve(m_locals.size()); + layout->ContinuationMemberOffsets.reserve(m_compiler->GetContinuationMemberCount()); for (unsigned lclNum : m_locals) { @@ -1574,6 +1593,17 @@ ContinuationLayout* ContinuationLayoutBuilder::Create() layout->OSRAddressOffset = allocLayout(TARGET_POINTER_SIZE, TARGET_POINTER_SIZE); } + for (size_t i = 0; i < m_compiler->GetContinuationMemberCount(); i++) + { + ClassLayout* memberLayout = m_compiler->GetContinuationMember(i).GetCustomAwaiterLayout(); + unsigned alignment = + memberLayout->IsCustomLayout() + ? (memberLayout->HasGCPtr() ? TARGET_POINTER_SIZE : 1) + : m_compiler->info.compCompHnd->getClassAlignmentRequirement(memberLayout->GetClassHandle()); + layout->ContinuationMemberOffsets.push_back( + allocLayout(std::min(alignment, (unsigned)TARGET_POINTER_SIZE), memberLayout->GetSize())); + } + if (m_needsExecutionContext) { layout->ExecutionContextOffset = allocLayout(TARGET_POINTER_SIZE, TARGET_POINTER_SIZE); @@ -1645,6 +1675,12 @@ ContinuationLayout* ContinuationLayoutBuilder::Create() bitmapBuilder.SetType(ret.Offset, ret.Type.ReturnType, ret.Type.ReturnLayout); } + for (size_t i = 0; i < layout->ContinuationMemberOffsets.size(); i++) + { + bitmapBuilder.SetType(layout->ContinuationMemberOffsets[i], TYP_STRUCT, + m_compiler->GetContinuationMember(i).GetCustomAwaiterLayout()); + } + #ifdef DEBUG if (m_compiler->verbose) { @@ -1987,7 +2023,7 @@ bool AsyncTransformation::IsReusableSuspension(const AsyncState* state, } } - static const WellKnownArg validateArgs[] = {WellKnownArg::AsyncExecutionContext, + static const WellKnownArg validateArgs[] = {WellKnownArg::AsyncAwaiter, WellKnownArg::AsyncExecutionContext, WellKnownArg::AsyncSynchronizationContext}; for (WellKnownArg arg : validateArgs) { @@ -2039,7 +2075,7 @@ bool AsyncTransformation::IsReusableSuspension(const AsyncState* state, // void AsyncTransformation::HandleReusedSuspension(BasicBlock* callBlock, GenTreeCall* call) { - static const WellKnownArg argsToRemove[] = {WellKnownArg::AsyncExecutionContext, + static const WellKnownArg argsToRemove[] = {WellKnownArg::AsyncAwaiter, WellKnownArg::AsyncExecutionContext, WellKnownArg::AsyncSynchronizationContext}; for (WellKnownArg wka : argsToRemove) { @@ -2271,10 +2307,75 @@ void AsyncTransformation::CreateSuspension(BasicBlock* call LIR::AsRange(suspendBB).InsertAtEnd(LIR::SeqTree(m_compiler, storeFlags)); FillInDataOnSuspension(layout, subLayout, suspendBB, mutatedSinceResumption, tailSaveSet); + StoreAsyncAwaiter(callBlock, call, suspendBB, layout); FinishContextHandlingAndSuspension(callBlock, call, suspendBB, layout, subLayout); } +//------------------------------------------------------------------------ +// AsyncTransformation::StoreAsyncAwaiter: +// Move the pseudo awaiter argument into its reserved continuation member. +// +void AsyncTransformation::StoreAsyncAwaiter(BasicBlock* callBlock, + GenTreeCall* call, + BasicBlock* suspendBB, + const ContinuationLayout& layout) +{ + CallArg* awaiterArg = call->gtArgs.FindWellKnownArg(WellKnownArg::AsyncAwaiter); + if (awaiterArg == nullptr) + { + return; + } + + ClassLayout* awaiterLayout = awaiterArg->GetSignatureLayout(); + assert(awaiterLayout != nullptr); + size_t memberIndex = + m_compiler->GetContinuationMemberIndex(ContinuationMember::CustomAwaiterOfLayout(awaiterLayout)); + assert(memberIndex < layout.ContinuationMemberOffsets.size()); + + GenTree* awaiter = awaiterArg->GetNode(); + if (!awaiter->OperIs(GT_LCL_VAR)) + { + LIR::Use use(LIR::AsRange(callBlock), &awaiterArg->NodeRef(), call); + use.ReplaceWithLclVar(m_compiler); + awaiter = use.Def(); + } + + LIR::AsRange(callBlock).Remove(awaiter); + call->gtArgs.RemoveUnsafe(awaiterArg); + + GenTree* continuation = m_compiler->gtNewLclvNode(GetNewContinuationVar(), TYP_REF); + unsigned offset = OFFSETOF__CORINFO_Continuation__data + layout.ContinuationMemberOffsets[memberIndex]; + GenTree* offsetNode = m_compiler->gtNewIconNode((ssize_t)offset, TYP_I_IMPL); + GenTree* address = m_compiler->gtNewOperNode(GT_ADD, TYP_BYREF, continuation, offsetNode); + GenTree* store = m_compiler->gtNewStoreValueNode(awaiterLayout, address, awaiter, GTF_IND_NONFAULTING); + LIR::AsRange(suspendBB).InsertAtEnd(LIR::SeqTree(m_compiler, store)); +} + +//------------------------------------------------------------------------ +// AsyncTransformation::ReplaceContinuationMemberOffsets: +// Replace symbolic continuation member offsets with their final constants. +// +void AsyncTransformation::ReplaceContinuationMemberOffsets(const ContinuationLayout& layout) +{ + for (BasicBlock* block : m_compiler->Blocks()) + { + for (GenTree* node : LIR::AsRange(block)) + { + if (!node->OperIs(GT_CONTINUATION_MEMBER_OFFSET)) + { + continue; + } + + size_t memberIndex = node->AsVal()->gtVal1; + assert(memberIndex < layout.ContinuationMemberOffsets.size()); + ssize_t offset = (OFFSETOF__CORINFO_Continuation__data - SIZEOF__CORINFO_Object) + + layout.ContinuationMemberOffsets[memberIndex]; + node->BashToConst(offset, TYP_INT); + } + } +} + //------------------------------------------------------------------------ // AsyncTransformation::CreateAllocContinuationCall: // Create a call to the JIT helper that allocates a continuation. @@ -3739,6 +3840,7 @@ void AsyncTransformation::CreateResumptionsAndSuspensions() ContinuationLayoutBuilder* sharedLayoutBuilder = ContinuationLayoutBuilder::CreateSharedLayout(m_compiler, m_states); sharedLayout = sharedLayoutBuilder->Create(); + ReplaceContinuationMemberOffsets(*sharedLayout); unsigned numSharedSuspensionsWithContinuationContext = 0; unsigned numSharedSuspensionsWithoutContinuationContext = 0; @@ -3800,12 +3902,18 @@ void AsyncTransformation::CreateResumptionsAndSuspensions() } } + bool replacedContinuationMemberOffsets = sharedLayout != nullptr; JITDUMP("Creating suspensions and resumptions for %zu states\n", m_states.size()); for (const AsyncState& state : m_states) { JITDUMP("State %u suspend @ " FMT_BB ", resume @ " FMT_BB "\n", state.Number, state.SuspensionBB->bbNum, state.ResumptionBB->bbNum); ContinuationLayout* layout = sharedLayout == nullptr ? state.Layout->Create() : sharedLayout; + if (!replacedContinuationMemberOffsets) + { + ReplaceContinuationMemberOffsets(*layout); + replacedContinuationMemberOffsets = true; + } CreateSuspension(state.CallBlock, state.Call, state.SuspensionBB, state.Number, *layout, *state.Layout, state.ResumeReachable, state.MutatedSincePreviousResumption); CreateResumption(state.CallBlock, state.Call, state.ResumptionBB, state.CallDefInfo, *layout, *state.Layout); diff --git a/src/coreclr/jit/async.h b/src/coreclr/jit/async.h index e1ef0bddd3d7f6..4f907d84b9bcf0 100644 --- a/src/coreclr/jit/async.h +++ b/src/coreclr/jit/async.h @@ -15,8 +15,9 @@ struct ContinuationMember ClassLayout* m_customAwaiterLayout; public: + ClassLayout* GetCustomAwaiterLayout() const; static ContinuationMember CustomAwaiterOfLayout(ClassLayout* layout); - static bool AreCompatible(const ContinuationMember& a, const ContinuationMember& b); + static bool AreCompatible(const ContinuationMember& a, const ContinuationMember& b); }; struct ReturnTypeInfo @@ -155,11 +156,13 @@ struct ContinuationLayout unsigned ExecutionContextOffset = UINT_MAX; jitstd::vector Locals; jitstd::vector Returns; + jitstd::vector ContinuationMemberOffsets; CORINFO_CLASS_HANDLE ClassHnd = NO_CLASS_HANDLE; ContinuationLayout(Compiler* comp) : Locals(comp->getAllocator(CMK_Async)) , Returns(comp->getAllocator(CMK_Async)) + , ContinuationMemberOffsets(comp->getAllocator(CMK_Async)) { } @@ -472,6 +475,11 @@ class AsyncTransformation const ContinuationLayoutBuilder& subLayout, SuspensionContextHelper helper); void RestoreContexts(BasicBlock* block, GenTreeCall* call, BasicBlock* insertionBB); + void StoreAsyncAwaiter(BasicBlock* callBlock, + GenTreeCall* call, + BasicBlock* suspendBB, + const ContinuationLayout& layout); + void ReplaceContinuationMemberOffsets(const ContinuationLayout& layout); void CreateCheckAndSuspendAfterCall(BasicBlock* block, GenTreeCall* call, const CallDefinitionInfo& callDefInfo, diff --git a/src/coreclr/jit/compiler.h b/src/coreclr/jit/compiler.h index 11dcf22780eb84..96b4188427c229 100644 --- a/src/coreclr/jit/compiler.h +++ b/src/coreclr/jit/compiler.h @@ -6240,6 +6240,7 @@ class Compiler jitstd::vector* m_asyncContinuationMembers = nullptr; size_t GetContinuationMemberIndex(const ContinuationMember& member); + size_t GetContinuationMemberCount() const; const ContinuationMember& GetContinuationMember(size_t index); PhaseStatus SaveAsyncContexts(); diff --git a/src/coreclr/jit/gentree.cpp b/src/coreclr/jit/gentree.cpp index 1effed3647c539..0a5ac063271b3b 100644 --- a/src/coreclr/jit/gentree.cpp +++ b/src/coreclr/jit/gentree.cpp @@ -1649,6 +1649,7 @@ bool CallArgs::GetCustomRegister(Compiler* comp, CorInfoCallConvExtension cc, We #endif // HAS_FIXED_REGISTER_SET case WellKnownArg::StackArrayLocal: + case WellKnownArg::AsyncAwaiter: case WellKnownArg::AsyncExecutionContext: case WellKnownArg::AsyncSynchronizationContext: // These are pseudo-args; they are not actual arguments, but we @@ -14016,6 +14017,7 @@ void Compiler::gtDispLeaf(GenTree* tree, IndentStack* indentStack) const ContinuationMember& member = GetContinuationMember(tree->AsVal()->gtVal1); member.Print(); printf("\n"); + break; } case GT_RET_EXPR: diff --git a/src/coreclr/jit/gentree.h b/src/coreclr/jit/gentree.h index b451be41097493..27f75d8ee4622e 100644 --- a/src/coreclr/jit/gentree.h +++ b/src/coreclr/jit/gentree.h @@ -4935,6 +4935,7 @@ class CallArg CORINFO_CLASS_HANDLE GetSignatureClassHandle() { return m_signatureLayout == nullptr ? NO_CLASS_HANDLE : m_signatureLayout->GetClassHandle(); } var_types GetSignatureType() { return m_signatureType; } WellKnownArg GetWellKnownArg() { return m_wellKnownArg; } + void SetWellKnownArg(WellKnownArg arg) { m_wellKnownArg = arg; } // clang-format on // Get the real argument node, i.e. not a setup or placeholder node. diff --git a/src/coreclr/jit/gtlist.h b/src/coreclr/jit/gtlist.h index cfc65897703c5e..42ff6a03a84018 100644 --- a/src/coreclr/jit/gtlist.h +++ b/src/coreclr/jit/gtlist.h @@ -42,7 +42,7 @@ GTNODE(ASYNC_RESUME_INFO, GenTreeVal ,0,0,GTK_LEAF) // Addre GTNODE(FTN_ENTRY , GenTree ,0,0,GTK_LEAF) // Address of this function's entry point // Offset into continuation, starting after the MethodTable*, of a special -// member. GenTreeVal::gtVal1 is an index into Compiler::m_continuationMembers. +// member. GenTreeVal::gtVal1 is an index into Compiler::m_asyncContinuationMembers. // The async transformation allocates and lays out the continuation and // substitutes these by constants at that time. GTNODE(CONTINUATION_MEMBER_OFFSET, GenTreeVal ,0,0,GTK_LEAF) diff --git a/src/coreclr/jit/gtstructs.h b/src/coreclr/jit/gtstructs.h index 59cd282c9544e1..bec1093d34d0b9 100644 --- a/src/coreclr/jit/gtstructs.h +++ b/src/coreclr/jit/gtstructs.h @@ -50,7 +50,7 @@ GTSTRUCT_0(UnOp , GT_OP) GTSTRUCT_0(Op , GT_OP) -GTSTRUCT_N(Val , GT_JMP, GT_RECORD_ASYNC_RESUME, GT_ASYNC_RESUME_INFO) +GTSTRUCT_N(Val , GT_JMP, GT_RECORD_ASYNC_RESUME, GT_ASYNC_RESUME_INFO, GT_CONTINUATION_MEMBER_OFFSET) GTSTRUCT_2_SPECIAL(IntConCommon, GT_CNS_INT, GT_CNS_LNG) GTSTRUCT_1(IntCon , GT_CNS_INT) GTSTRUCT_1(LngCon , GT_CNS_LNG) diff --git a/src/coreclr/jit/importercalls.cpp b/src/coreclr/jit/importercalls.cpp index f9a583a850e2c8..2335dc790954b1 100644 --- a/src/coreclr/jit/importercalls.cpp +++ b/src/coreclr/jit/importercalls.cpp @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. #include "jitpch.h" +#include "async.h" //------------------------------------------------------------------------ // impGetInstParamArg: compute the hidden instantiation / generic-context argument @@ -952,6 +953,53 @@ var_types Compiler::impImportCall(OPCODE opcode, impPopCallArgs(sig, call->AsCall()); + if (opts.OptimizationEnabled() && ((ni == NI_System_Runtime_CompilerServices_AsyncHelpers_AwaitAwaiter) || + (ni == NI_System_Runtime_CompilerServices_AsyncHelpers_UnsafeAwaitAwaiter))) + { + CallArg* awaiterArg = call->AsCall()->gtArgs.GetUserArgByIndex(0); + if (varTypeIsStruct(awaiterArg->GetSignatureType())) + { + CORINFO_LOOKUP newInstArgLookup; + CORINFO_SIG_INFO lookupSig = *sig; + if (pResolvedToken->pMethodSpec != nullptr) + { + lookupSig.pSig = pResolvedToken->pMethodSpec; + lookupSig.cbSig = pResolvedToken->cbMethodSpec; + lookupSig.scope = pResolvedToken->tokenScope; + } + CORINFO_METHOD_HANDLE newMethod = info.compCompHnd->getAwaitAwaiterInContinuationCall( + info.compMethodHnd, &lookupSig, + ni == NI_System_Runtime_CompilerServices_AsyncHelpers_UnsafeAwaitAwaiter, &exactContextHnd, + &newInstArgLookup); + + methHnd = newMethod; + call->AsCall()->gtCallMethHnd = newMethod; + callInfo->hMethod = newMethod; + callInfo->methodFlags = info.compCompHnd->getMethodAttribs(newMethod); + info.compCompHnd->getMethodSig(newMethod, &callInfo->sig); + + awaiterArg->SetWellKnownArg(WellKnownArg::AsyncAwaiter); + size_t memberIndex = + GetContinuationMemberIndex(ContinuationMember::CustomAwaiterOfLayout(awaiterArg->GetSignatureLayout())); + GenTree* offset = new (this, GT_CONTINUATION_MEMBER_OFFSET) + GenTreeVal(GT_CONTINUATION_MEMBER_OFFSET, TYP_INT, memberIndex); + call->AsCall()->gtArgs.PushBack(this, NewCallArg::Primitive(offset)); + + if (callInfo->sig.hasTypeArg()) + { + instParam = impLookupToTree(&newInstArgLookup, GTF_ICON_METHOD_HDL, newMethod); + if (instParam == nullptr) + { + return TYP_UNDEF; + } + } + else + { + instParam = nullptr; + } + } + } + // Extra args if ((instParam != nullptr) || (asyncContinuation != nullptr) || (varArgsCookie != nullptr)) { diff --git a/src/coreclr/jit/wellknownargs.h b/src/coreclr/jit/wellknownargs.h index a963226292cbbc..fd7318171bc4b4 100644 --- a/src/coreclr/jit/wellknownargs.h +++ b/src/coreclr/jit/wellknownargs.h @@ -36,6 +36,7 @@ WELL_KNOWN_ARG(SwiftSelf, "swift self", true, false) WELL_KNOWN_ARG(X86TailCallSpecialArg, "tail call", false, false) WELL_KNOWN_ARG(StackArrayLocal, "&lcl arr", false, false) WELL_KNOWN_ARG(RuntimeMethodHandle, "meth hnd", false, false) +WELL_KNOWN_ARG(AsyncAwaiter, "awaiter", false, false) WELL_KNOWN_ARG(AsyncExecutionContext, "exec ctx", false, false) WELL_KNOWN_ARG(AsyncSynchronizationContext, "sync ctx", false, false) WELL_KNOWN_ARG(WasmShadowStackPointer, "wasm sp", false, false) diff --git a/src/coreclr/tools/Common/JitInterface/CorInfoImpl.cs b/src/coreclr/tools/Common/JitInterface/CorInfoImpl.cs index 329abbf03b7f78..c71feb7be6753b 100644 --- a/src/coreclr/tools/Common/JitInterface/CorInfoImpl.cs +++ b/src/coreclr/tools/Common/JitInterface/CorInfoImpl.cs @@ -3674,6 +3674,62 @@ private void getWasmWellKnownGlobals(ref CORINFO_WASM_WELLKNOWN_GLOBALS pWellKno return ObjectToHandle(targetMethod); } + private CORINFO_METHOD_STRUCT_* getAwaitAwaiterInContinuationCall( + CORINFO_METHOD_STRUCT_* callerHandle, + CORINFO_SIG_INFO* callSig, + bool isUnsafe, + CORINFO_CONTEXT_STRUCT** contextHandle, + ref CORINFO_LOOKUP instArg) + { + Debug.Assert(callSig->sigInst.methInstCount == 1); + + instArg.lookupKind.needsRuntimeLookup = false; + instArg.constLookup.accessType = InfoAccessType.IAT_VALUE; + instArg.constLookup.addr = null; + + MethodDesc caller = HandleToObject(callerHandle); + TypeDesc awaiterType = HandleToObject(callSig->sigInst.methInst[0]); + CompilerTypeSystemContext context = _compilation.TypeSystemContext; + DefType asyncHelpers = + context.SystemModule.GetKnownType("System.Runtime.CompilerServices"u8, "AsyncHelpers"u8); + TypeDesc signatureVariable = context.GetSignatureVariable(0, method: true); + MethodSignature signature = new MethodSignature( + MethodSignatureFlags.Static, + 1, + context.GetWellKnownType(WellKnownType.Void), + [signatureVariable, context.GetWellKnownType(WellKnownType.Int32)]); + MethodDesc result = asyncHelpers + .GetKnownMethod( + isUnsafe ? "UnsafeAwaitAwaiterInContinuation"u8 : "AwaitAwaiterInContinuation"u8, + signature) + .MakeInstantiatedMethod(awaiterType); + MethodDesc targetMethod = result.GetCanonMethodTarget(CanonicalFormKind.Specific); + *contextHandle = contextFromMethod(result); + + if (targetMethod.RequiresInstArg()) + { +#if READYTORUN + instArg.constLookup = CreateConstLookupToSymbol( + _compilation.SymbolNodeFactory.CreateReadyToRunHelper( + ReadyToRunHelperId.MethodDictionary, + new MethodWithToken( + result, + _compilation.NodeFactory.Resolver.GetModuleTokenForMethod( + result, + allowDynamicallyCreatedReference: true, + throwIfNotFound: true), + constrainedType: null, + unboxing: false, + genericContextObject: caller))); +#else + ComputeLookup(caller != MethodBeingCompiled, result, ReadyToRunHelperId.MethodDictionary, caller, + ref instArg); +#endif + } + + return ObjectToHandle(targetMethod); + } + private CORINFO_CLASS_STRUCT_* getContinuationType(nuint dataSize, ref bool objRefs, nuint objRefsSize) { Debug.Assert(objRefsSize == (dataSize + (nuint)(PointerSize - 1)) / (nuint)PointerSize); diff --git a/src/coreclr/tools/Common/JitInterface/CorInfoImpl_generated.cs b/src/coreclr/tools/Common/JitInterface/CorInfoImpl_generated.cs index ef00dcf0188b1b..f3389802e9ad5b 100644 --- a/src/coreclr/tools/Common/JitInterface/CorInfoImpl_generated.cs +++ b/src/coreclr/tools/Common/JitInterface/CorInfoImpl_generated.cs @@ -141,6 +141,7 @@ static ICorJitInfoCallbacks() s_callbacks.getEEInfo = &_getEEInfo; s_callbacks.getAsyncInfo = &_getAsyncInfo; s_callbacks.getAwaitReturnCall = &_getAwaitReturnCall; + s_callbacks.getAwaitAwaiterInContinuationCall = &_getAwaitAwaiterInContinuationCall; s_callbacks.getMethodDefFromMethod = &_getMethodDefFromMethod; s_callbacks.printMethodName = &_printMethodName; s_callbacks.getMethodNameFromMetadata = &_getMethodNameFromMetadata; @@ -325,6 +326,7 @@ static ICorJitInfoCallbacks() public delegate* unmanaged getEEInfo; public delegate* unmanaged getAsyncInfo; public delegate* unmanaged getAwaitReturnCall; + public delegate* unmanaged getAwaitAwaiterInContinuationCall; public delegate* unmanaged getMethodDefFromMethod; public delegate* unmanaged printMethodName; public delegate* unmanaged getMethodNameFromMetadata; @@ -2178,6 +2180,21 @@ private static void _getAsyncInfo(IntPtr thisHandle, IntPtr* ppException, CORINF } } + [UnmanagedCallersOnly] + private static CORINFO_METHOD_STRUCT_* _getAwaitAwaiterInContinuationCall(IntPtr thisHandle, IntPtr* ppException, CORINFO_METHOD_STRUCT_* callerHandle, CORINFO_SIG_INFO* callSig, byte isUnsafe, CORINFO_CONTEXT_STRUCT** contextHandle, CORINFO_LOOKUP* instArg) + { + var _this = GetThis(thisHandle); + try + { + return _this.getAwaitAwaiterInContinuationCall(callerHandle, callSig, isUnsafe != 0, contextHandle, ref *instArg); + } + catch (Exception ex) + { + *ppException = _this.AllocException(ex); + return default; + } + } + [UnmanagedCallersOnly] private static mdToken _getMethodDefFromMethod(IntPtr thisHandle, IntPtr* ppException, CORINFO_METHOD_STRUCT_* hMethod) { diff --git a/src/coreclr/tools/Common/JitInterface/ThunkGenerator/ThunkInput.txt b/src/coreclr/tools/Common/JitInterface/ThunkGenerator/ThunkInput.txt index 8a6ab34f947c4b..8368a69891a2d9 100644 --- a/src/coreclr/tools/Common/JitInterface/ThunkGenerator/ThunkInput.txt +++ b/src/coreclr/tools/Common/JitInterface/ThunkGenerator/ThunkInput.txt @@ -296,6 +296,7 @@ FUNCTIONS void getEEInfo(CORINFO_EE_INFO* pEEInfoOut); void getAsyncInfo(CORINFO_ASYNC_INFO* pAsyncInfoOut); CORINFO_METHOD_HANDLE getAwaitReturnCall(CORINFO_METHOD_HANDLE callerHandle, CORINFO_CONTEXT_HANDLE* contextHandle, CORINFO_LOOKUP* instArg); + CORINFO_METHOD_HANDLE getAwaitAwaiterInContinuationCall(CORINFO_METHOD_HANDLE callerHandle, CORINFO_SIG_INFO* callSig, bool isUnsafe, CORINFO_CONTEXT_HANDLE* contextHandle, CORINFO_LOOKUP* instArg); mdMethodDef getMethodDefFromMethod(CORINFO_METHOD_HANDLE hMethod); size_t printMethodName(CORINFO_METHOD_HANDLE ftn, char* buffer, size_t bufferSize, size_t* pRequiredBufferSize) const char* getMethodNameFromMetadata(CORINFO_METHOD_HANDLE ftn, const char **className, const char **namespaceName, const char **enclosingClassNames, size_t maxEnclosingClassNames); diff --git a/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/ReadyToRunCodegenCompilation.cs b/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/ReadyToRunCodegenCompilation.cs index 5834498f43ea3e..da3570706401ef 100644 --- a/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/ReadyToRunCodegenCompilation.cs +++ b/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/ReadyToRunCodegenCompilation.cs @@ -1042,6 +1042,7 @@ private void AddNecessaryAsyncReferences(MethodDesc method) MetadataType taskOfTType = TypeSystemContext.SystemModule.GetKnownType("System.Threading.Tasks"u8, "Task`1"u8); MetadataType valueTaskOfTType = TypeSystemContext.SystemModule.GetKnownType("System.Threading.Tasks"u8, "ValueTask`1"u8); TypeDesc methodVar = TypeSystemContext.GetSignatureVariable(0, method: true); + TypeDesc intType = TypeSystemContext.GetWellKnownType(WellKnownType.Int32); MethodDesc[] requiredMethods = [ // For CorInfoImpl.getAsyncInfo @@ -1064,6 +1065,8 @@ private void AddNecessaryAsyncReferences(MethodDesc method) asyncHelpers.GetKnownMethod("TransparentAwait"u8, new MethodSignature(MethodSignatureFlags.Static, 0, voidType, [valueTaskType])), asyncHelpers.GetKnownMethod("TransparentAwait"u8, new MethodSignature(MethodSignatureFlags.Static, 1, methodVar, [taskOfTType.MakeInstantiatedType(methodVar)])), asyncHelpers.GetKnownMethod("TransparentAwait"u8, new MethodSignature(MethodSignatureFlags.Static, 1, methodVar, [valueTaskOfTType.MakeInstantiatedType(methodVar)])), + asyncHelpers.GetKnownMethod("AwaitAwaiterInContinuation"u8, new MethodSignature(MethodSignatureFlags.Static, 1, voidType, [methodVar, intType])), + asyncHelpers.GetKnownMethod("UnsafeAwaitAwaiterInContinuation"u8, new MethodSignature(MethodSignatureFlags.Static, 1, voidType, [methodVar, intType])), ]; var moduleForNewReferences = ((EcmaMethod)method.GetPrimaryMethodDesc().GetTypicalMethodDefinition()).Module; _tokenManager.EnsureDefTokensAreAvailable([..requiredMethods, ..requiredTypes, ..requiredFields], moduleForNewReferences, true); diff --git a/src/coreclr/tools/aot/jitinterface/jitinterface_generated.h b/src/coreclr/tools/aot/jitinterface/jitinterface_generated.h index 3a0514e49ad2a7..9f036e818e547d 100644 --- a/src/coreclr/tools/aot/jitinterface/jitinterface_generated.h +++ b/src/coreclr/tools/aot/jitinterface/jitinterface_generated.h @@ -132,6 +132,7 @@ struct JitInterfaceCallbacks void (* getEEInfo)(void * thisHandle, CorInfoExceptionClass** ppException, CORINFO_EE_INFO* pEEInfoOut); void (* getAsyncInfo)(void * thisHandle, CorInfoExceptionClass** ppException, CORINFO_ASYNC_INFO* pAsyncInfoOut); CORINFO_METHOD_HANDLE (* getAwaitReturnCall)(void * thisHandle, CorInfoExceptionClass** ppException, CORINFO_METHOD_HANDLE callerHandle, CORINFO_CONTEXT_HANDLE* contextHandle, CORINFO_LOOKUP* instArg); + CORINFO_METHOD_HANDLE (* getAwaitAwaiterInContinuationCall)(void * thisHandle, CorInfoExceptionClass** ppException, CORINFO_METHOD_HANDLE callerHandle, CORINFO_SIG_INFO* callSig, bool isUnsafe, CORINFO_CONTEXT_HANDLE* contextHandle, CORINFO_LOOKUP* instArg); mdMethodDef (* getMethodDefFromMethod)(void * thisHandle, CorInfoExceptionClass** ppException, CORINFO_METHOD_HANDLE hMethod); size_t (* printMethodName)(void * thisHandle, CorInfoExceptionClass** ppException, CORINFO_METHOD_HANDLE ftn, char* buffer, size_t bufferSize, size_t* pRequiredBufferSize); const char* (* getMethodNameFromMetadata)(void * thisHandle, CorInfoExceptionClass** ppException, CORINFO_METHOD_HANDLE ftn, const char** className, const char** namespaceName, const char** enclosingClassNames, size_t maxEnclosingClassNames); @@ -1374,6 +1375,19 @@ class JitInterfaceWrapper : public ICorJitInfo return temp; } + virtual CORINFO_METHOD_HANDLE getAwaitAwaiterInContinuationCall( + CORINFO_METHOD_HANDLE callerHandle, + CORINFO_SIG_INFO* callSig, + bool isUnsafe, + CORINFO_CONTEXT_HANDLE* contextHandle, + CORINFO_LOOKUP* instArg) +{ + CorInfoExceptionClass* pException = nullptr; + CORINFO_METHOD_HANDLE temp = _callbacks->getAwaitAwaiterInContinuationCall(_thisHandle, &pException, callerHandle, callSig, isUnsafe, contextHandle, instArg); + if (pException != nullptr) throw pException; + return temp; +} + virtual mdMethodDef getMethodDefFromMethod( CORINFO_METHOD_HANDLE hMethod) { diff --git a/src/coreclr/tools/superpmi/superpmi-shared/agnostic.h b/src/coreclr/tools/superpmi/superpmi-shared/agnostic.h index 2c36e6500c75ac..0f64d7436d779f 100644 --- a/src/coreclr/tools/superpmi/superpmi-shared/agnostic.h +++ b/src/coreclr/tools/superpmi/superpmi-shared/agnostic.h @@ -255,6 +255,13 @@ struct Agnostic_GetAwaitReturnCallResult Agnostic_CORINFO_LOOKUP instArg; }; +struct Agnostic_GetAwaitAwaiterInContinuationCall +{ + DWORDLONG callerHnd; + Agnostic_CORINFO_SIG_INFO callSig; + DWORD isUnsafe; +}; + struct Agnostic_GetOSRInfo { DWORD index; diff --git a/src/coreclr/tools/superpmi/superpmi-shared/lwmlist.h b/src/coreclr/tools/superpmi/superpmi-shared/lwmlist.h index e44cf227fbd412..c3f540189da97c 100644 --- a/src/coreclr/tools/superpmi/superpmi-shared/lwmlist.h +++ b/src/coreclr/tools/superpmi/superpmi-shared/lwmlist.h @@ -80,6 +80,7 @@ LWM(GetDelegateCtor, Agnostic_GetDelegateCtorIn, Agnostic_GetDelegateCtorOut) LWM(GetEEInfo, DWORD, Agnostic_CORINFO_EE_INFO) LWM(GetAsyncInfo, DWORD, Agnostic_CORINFO_ASYNC_INFO) LWM(GetAwaitReturnCall, DWORDLONG, Agnostic_GetAwaitReturnCallResult) +LWM(GetAwaitAwaiterInContinuationCall, Agnostic_GetAwaitAwaiterInContinuationCall, Agnostic_GetAwaitReturnCallResult) LWM(GetEHinfo, DLD, Agnostic_CORINFO_EH_CLAUSE) LWM(GetStaticFieldContent, DLDDD, DD) LWM(GetObjectContent, DLDD, DD) diff --git a/src/coreclr/tools/superpmi/superpmi-shared/methodcontext.cpp b/src/coreclr/tools/superpmi/superpmi-shared/methodcontext.cpp index db89aaed33f8a7..2bf7973f50d754 100644 --- a/src/coreclr/tools/superpmi/superpmi-shared/methodcontext.cpp +++ b/src/coreclr/tools/superpmi/superpmi-shared/methodcontext.cpp @@ -4483,6 +4483,73 @@ CORINFO_METHOD_HANDLE MethodContext::repGetAwaitReturnCall(CORINFO_METHOD_HANDLE return (CORINFO_METHOD_HANDLE)result.methodHnd; } +void MethodContext::recGetAwaitAwaiterInContinuationCall(CORINFO_METHOD_HANDLE callerHnd, + CORINFO_SIG_INFO* callSig, + bool isUnsafe, + CORINFO_CONTEXT_HANDLE* contextHandle, + CORINFO_LOOKUP* instArg, + CORINFO_METHOD_HANDLE methHnd) +{ + if (GetAwaitAwaiterInContinuationCall == nullptr) + { + GetAwaitAwaiterInContinuationCall = + new LightWeightMap(); + } + + Agnostic_GetAwaitAwaiterInContinuationCall key; + ZeroMemory(&key, sizeof(key)); + key.callerHnd = CastHandle(callerHnd); + key.callSig = + SpmiRecordsHelper::StoreAgnostic_CORINFO_SIG_INFO(*callSig, GetAwaitAwaiterInContinuationCall, SigInstHandleMap); + key.isUnsafe = isUnsafe; + + Agnostic_GetAwaitReturnCallResult value; + ZeroMemory(&value, sizeof(value)); + value.methodHnd = CastHandle(methHnd); + value.contextHandle = CastHandle(*contextHandle); + value.instArg = SpmiRecordsHelper::StoreAgnostic_CORINFO_LOOKUP(instArg); + + GetAwaitAwaiterInContinuationCall->Add(key, value); + DEBUG_REC(dmpGetAwaitAwaiterInContinuationCall(key, value)); +} + +void MethodContext::dmpGetAwaitAwaiterInContinuationCall( + const Agnostic_GetAwaitAwaiterInContinuationCall& key, + Agnostic_GetAwaitReturnCallResult& value) +{ + printf("GetAwaitAwaiterInContinuationCall caller-%016" PRIX64 " sig-%s unsafe-%u " + "methodHnd-%016" PRIX64 " contextHandle-%016" PRIX64 " instArg %s", + key.callerHnd, + SpmiDumpHelper::DumpAgnostic_CORINFO_SIG_INFO( + key.callSig, GetAwaitAwaiterInContinuationCall, SigInstHandleMap).c_str(), + key.isUnsafe, + value.methodHnd, + value.contextHandle, + SpmiDumpHelper::DumpAgnostic_CORINFO_LOOKUP(value.instArg).c_str()); +} + +CORINFO_METHOD_HANDLE MethodContext::repGetAwaitAwaiterInContinuationCall( + CORINFO_METHOD_HANDLE callerHnd, + CORINFO_SIG_INFO* callSig, + bool isUnsafe, + CORINFO_CONTEXT_HANDLE* contextHandle, + CORINFO_LOOKUP* instArg) +{ + Agnostic_GetAwaitAwaiterInContinuationCall key; + ZeroMemory(&key, sizeof(key)); + key.callerHnd = CastHandle(callerHnd); + key.callSig = SpmiRecordsHelper::RestoreAgnostic_CORINFO_SIG_INFO( + *callSig, GetAwaitAwaiterInContinuationCall, SigInstHandleMap); + key.isUnsafe = isUnsafe; + + Agnostic_GetAwaitReturnCallResult value = + LookupByKeyOrMissNoMessage(GetAwaitAwaiterInContinuationCall, key); + DEBUG_REP(dmpGetAwaitAwaiterInContinuationCall(key, value)); + *contextHandle = (CORINFO_CONTEXT_HANDLE)value.contextHandle; + *instArg = SpmiRecordsHelper::RestoreCORINFO_LOOKUP(value.instArg); + return (CORINFO_METHOD_HANDLE)value.methodHnd; +} + void MethodContext::recGetGSCookie(GSCookie* pCookieVal, GSCookie** ppCookieVal) { if (GetGSCookie == nullptr) diff --git a/src/coreclr/tools/superpmi/superpmi-shared/methodcontext.h b/src/coreclr/tools/superpmi/superpmi-shared/methodcontext.h index 5ca689f36ce384..0fa494f306e8b8 100644 --- a/src/coreclr/tools/superpmi/superpmi-shared/methodcontext.h +++ b/src/coreclr/tools/superpmi/superpmi-shared/methodcontext.h @@ -563,6 +563,20 @@ class MethodContext void dmpGetAwaitReturnCall(DWORDLONG key, Agnostic_GetAwaitReturnCallResult& value); CORINFO_METHOD_HANDLE repGetAwaitReturnCall(CORINFO_METHOD_HANDLE callerHnd, CORINFO_CONTEXT_HANDLE* contextHandle, CORINFO_LOOKUP* instArg); + void recGetAwaitAwaiterInContinuationCall(CORINFO_METHOD_HANDLE callerHnd, + CORINFO_SIG_INFO* callSig, + bool isUnsafe, + CORINFO_CONTEXT_HANDLE* contextHandle, + CORINFO_LOOKUP* instArg, + CORINFO_METHOD_HANDLE methHnd); + void dmpGetAwaitAwaiterInContinuationCall(const Agnostic_GetAwaitAwaiterInContinuationCall& key, + Agnostic_GetAwaitReturnCallResult& value); + CORINFO_METHOD_HANDLE repGetAwaitAwaiterInContinuationCall(CORINFO_METHOD_HANDLE callerHnd, + CORINFO_SIG_INFO* callSig, + bool isUnsafe, + CORINFO_CONTEXT_HANDLE* contextHandle, + CORINFO_LOOKUP* instArg); + void recGetWasmWellKnownGlobals(const CORINFO_WASM_WELLKNOWN_GLOBALS* pBaseGlobals); void dmpGetWasmWellKnownGlobals(DWORD key, const Agnostic_CORINFO_WASM_WELLKNOWN_GLOBALS& value); void repGetWasmWellKnownGlobals(CORINFO_WASM_WELLKNOWN_GLOBALS* pWellKnownGlobalsOut); @@ -1224,6 +1238,7 @@ enum mcPackets Packet_GetAwaitReturnCall = 238, Packet_GetAddressAlignment = 239, Packet_GetWasmWellKnownGlobals = 240, + Packet_GetAwaitAwaiterInContinuationCall = 241, }; void SetDebugDumpVariables(); diff --git a/src/coreclr/tools/superpmi/superpmi-shim-collector/icorjitinfo.cpp b/src/coreclr/tools/superpmi/superpmi-shim-collector/icorjitinfo.cpp index 9744fc70aaa470..e9ade6d19548a2 100644 --- a/src/coreclr/tools/superpmi/superpmi-shim-collector/icorjitinfo.cpp +++ b/src/coreclr/tools/superpmi/superpmi-shim-collector/icorjitinfo.cpp @@ -1396,6 +1396,20 @@ CORINFO_METHOD_HANDLE interceptor_ICJI::getAwaitReturnCall(CORINFO_METHOD_HANDLE return result; } +CORINFO_METHOD_HANDLE interceptor_ICJI::getAwaitAwaiterInContinuationCall( + CORINFO_METHOD_HANDLE callerHandle, + CORINFO_SIG_INFO* callSig, + bool isUnsafe, + CORINFO_CONTEXT_HANDLE* contextHandle, + CORINFO_LOOKUP* instArg) +{ + mc->cr->AddCall("getAwaitAwaiterInContinuationCall"); + CORINFO_METHOD_HANDLE result = original_ICorJitInfo->getAwaitAwaiterInContinuationCall( + callerHandle, callSig, isUnsafe, contextHandle, instArg); + mc->recGetAwaitAwaiterInContinuationCall(callerHandle, callSig, isUnsafe, contextHandle, instArg, result); + return result; +} + /*********************************************************************************/ // // Diagnostic methods diff --git a/src/coreclr/tools/superpmi/superpmi-shim-counter/icorjitinfo_generated.cpp b/src/coreclr/tools/superpmi/superpmi-shim-counter/icorjitinfo_generated.cpp index 53166c4d1538d6..1dc724f10fa38a 100644 --- a/src/coreclr/tools/superpmi/superpmi-shim-counter/icorjitinfo_generated.cpp +++ b/src/coreclr/tools/superpmi/superpmi-shim-counter/icorjitinfo_generated.cpp @@ -978,6 +978,17 @@ CORINFO_METHOD_HANDLE interceptor_ICJI::getAwaitReturnCall( return original_ICorJitInfo->getAwaitReturnCall(callerHandle, contextHandle, instArg); } +CORINFO_METHOD_HANDLE interceptor_ICJI::getAwaitAwaiterInContinuationCall( + CORINFO_METHOD_HANDLE callerHandle, + CORINFO_SIG_INFO* callSig, + bool isUnsafe, + CORINFO_CONTEXT_HANDLE* contextHandle, + CORINFO_LOOKUP* instArg) +{ + mcs->AddCall("getAwaitAwaiterInContinuationCall"); + return original_ICorJitInfo->getAwaitAwaiterInContinuationCall(callerHandle, callSig, isUnsafe, contextHandle, instArg); +} + mdMethodDef interceptor_ICJI::getMethodDefFromMethod( CORINFO_METHOD_HANDLE hMethod) { diff --git a/src/coreclr/tools/superpmi/superpmi-shim-simple/icorjitinfo_generated.cpp b/src/coreclr/tools/superpmi/superpmi-shim-simple/icorjitinfo_generated.cpp index 458287f1d32ac2..199f0b6fce2631 100644 --- a/src/coreclr/tools/superpmi/superpmi-shim-simple/icorjitinfo_generated.cpp +++ b/src/coreclr/tools/superpmi/superpmi-shim-simple/icorjitinfo_generated.cpp @@ -857,6 +857,16 @@ CORINFO_METHOD_HANDLE interceptor_ICJI::getAwaitReturnCall( return original_ICorJitInfo->getAwaitReturnCall(callerHandle, contextHandle, instArg); } +CORINFO_METHOD_HANDLE interceptor_ICJI::getAwaitAwaiterInContinuationCall( + CORINFO_METHOD_HANDLE callerHandle, + CORINFO_SIG_INFO* callSig, + bool isUnsafe, + CORINFO_CONTEXT_HANDLE* contextHandle, + CORINFO_LOOKUP* instArg) +{ + return original_ICorJitInfo->getAwaitAwaiterInContinuationCall(callerHandle, callSig, isUnsafe, contextHandle, instArg); +} + mdMethodDef interceptor_ICJI::getMethodDefFromMethod( CORINFO_METHOD_HANDLE hMethod) { diff --git a/src/coreclr/tools/superpmi/superpmi/icorjitinfo.cpp b/src/coreclr/tools/superpmi/superpmi/icorjitinfo.cpp index e71c1dd6071c4a..e5cff9a0c71581 100644 --- a/src/coreclr/tools/superpmi/superpmi/icorjitinfo.cpp +++ b/src/coreclr/tools/superpmi/superpmi/icorjitinfo.cpp @@ -1208,6 +1208,18 @@ CORINFO_METHOD_HANDLE MyICJI::getAwaitReturnCall(CORINFO_METHOD_HANDLE callerHan return jitInstance->mc->repGetAwaitReturnCall(callerHandle, contextHandle, instArg); } +CORINFO_METHOD_HANDLE MyICJI::getAwaitAwaiterInContinuationCall( + CORINFO_METHOD_HANDLE callerHandle, + CORINFO_SIG_INFO* callSig, + bool isUnsafe, + CORINFO_CONTEXT_HANDLE* contextHandle, + CORINFO_LOOKUP* instArg) +{ + jitInstance->mc->cr->AddCall("getAwaitAwaiterInContinuationCall"); + return jitInstance->mc->repGetAwaitAwaiterInContinuationCall( + callerHandle, callSig, isUnsafe, contextHandle, instArg); +} + /*********************************************************************************/ // // Diagnostic methods diff --git a/src/coreclr/vm/corelib.h b/src/coreclr/vm/corelib.h index 1584e25df9f8f1..60747f18d01d72 100644 --- a/src/coreclr/vm/corelib.h +++ b/src/coreclr/vm/corelib.h @@ -723,6 +723,8 @@ DEFINE_METHOD(ASYNC_HELPERS, TRANSPARENT_AWAIT_TASK, Transpa DEFINE_METHOD(ASYNC_HELPERS, TRANSPARENT_AWAIT_VALUETASK, TransparentAwait, SM_ValueTask_RetVoid) DEFINE_METHOD(ASYNC_HELPERS, TRANSPARENT_AWAIT_TASK_OF_T, TransparentAwait, GM_TaskOfT_RetT) DEFINE_METHOD(ASYNC_HELPERS, TRANSPARENT_AWAIT_VALUETASK_OF_T, TransparentAwait, GM_ValueTaskOfT_RetT) +DEFINE_METHOD(ASYNC_HELPERS, AWAIT_AWAITER_IN_CONTINUATION, AwaitAwaiterInContinuation, NoSig) +DEFINE_METHOD(ASYNC_HELPERS, UNSAFE_AWAIT_AWAITER_IN_CONTINUATION, UnsafeAwaitAwaiterInContinuation, NoSig) DEFINE_METHOD(ASYNC_HELPERS, CAPTURE_EXECUTION_CONTEXT, CaptureExecutionContext, NoSig) DEFINE_METHOD(ASYNC_HELPERS, CAPTURE_CONTINUATION_CONTEXT, CaptureContinuationContext, NoSig) DEFINE_METHOD(ASYNC_HELPERS, CAPTURE_CONTEXTS, CaptureContexts, NoSig) diff --git a/src/coreclr/vm/jitinterface.cpp b/src/coreclr/vm/jitinterface.cpp index 5bb833b15d8527..6d132f96b4688a 100644 --- a/src/coreclr/vm/jitinterface.cpp +++ b/src/coreclr/vm/jitinterface.cpp @@ -10445,6 +10445,60 @@ CORINFO_METHOD_HANDLE CEEInfo::getAwaitReturnCall(CORINFO_METHOD_HANDLE callerHa return CORINFO_METHOD_HANDLE(pMD); } +CORINFO_METHOD_HANDLE CEEInfo::getAwaitAwaiterInContinuationCall( + CORINFO_METHOD_HANDLE callerHandle, + CORINFO_SIG_INFO* callSig, + bool isUnsafe, + CORINFO_CONTEXT_HANDLE* contextHandle, + CORINFO_LOOKUP* instArg) +{ + CONTRACTL { + THROWS; + GC_TRIGGERS; + MODE_PREEMPTIVE; + } CONTRACTL_END; + + MethodDesc* pMD = NULL; + + JIT_TO_EE_TRANSITION(); + + _ASSERTE(callSig->sigInst.methInstCount == 1); + TypeHandle awaiterType(callSig->sigInst.methInst[0]); + + instArg->lookupKind.needsRuntimeLookup = false; + instArg->constLookup.accessType = IAT_VALUE; + instArg->constLookup.addr = NULL; + + MethodDesc* pCallerMD = GetMethod(callerHandle); + MethodDesc* pTypicalAwaitMD = CoreLibBinder::GetMethod( + isUnsafe ? METHOD__ASYNC_HELPERS__UNSAFE_AWAIT_AWAITER_IN_CONTINUATION + : METHOD__ASYNC_HELPERS__AWAIT_AWAITER_IN_CONTINUATION); + pMD = MethodDesc::FindOrCreateAssociatedMethodDesc(pTypicalAwaitMD, pTypicalAwaitMD->GetMethodTable(), FALSE, + Instantiation(&awaiterType, 1), TRUE); + + MethodDesc* pInliningContext = pMD; + if (pMD->RequiresInstArg()) + { + if (awaiterType.IsCanonicalSubtype()) + { + ComputeRuntimeLookupForAwaitAwaiterInContinuationCall(pCallerMD, pTypicalAwaitMD, callSig, instArg); + } + else + { + MethodDesc* pContext = MethodDesc::FindOrCreateAssociatedMethodDesc( + pTypicalAwaitMD, pTypicalAwaitMD->GetMethodTable(), FALSE, Instantiation(&awaiterType, 1), FALSE); + instArg->constLookup.addr = pContext; + pInliningContext = pContext; + } + } + + *contextHandle = MAKE_METHODCONTEXT(pInliningContext); + + EE_TO_JIT_TRANSITION(); + + return CORINFO_METHOD_HANDLE(pMD); +} + // Compute the runtime lookup for the instantiation argument for an // AsyncHelpers.TransparentAwait call, to be used for wrapping a return value // from pCallerMD for its runtime async version. @@ -10509,6 +10563,65 @@ void CEEInfo::ComputeRuntimeLookupForAwaitCall(MethodDesc* pCallerMD, MethodDesc FinishComputeRuntimeLookup(sigBuilder, pCallerMD, lookup); } +void CEEInfo::ComputeRuntimeLookupForAwaitAwaiterInContinuationCall( + MethodDesc* pCallerMD, + MethodDesc* pTypicalAwaitMD, + CORINFO_SIG_INFO* callSig, + CORINFO_LOOKUP* lookup) +{ + lookup->lookupKind.needsRuntimeLookup = true; + + CORINFO_RUNTIME_LOOKUP* rlookup = &lookup->runtimeLookup; + rlookup->signature = NULL; + rlookup->indirectFirstOffset = 0; + rlookup->indirectSecondOffset = 0; + rlookup->sizeOffset = CORINFO_NO_SIZE_CHECK; + rlookup->indirections = CORINFO_USEHELPER; + + if (pCallerMD->RequiresInstMethodDescArg()) + { + lookup->lookupKind.runtimeLookupKind = CORINFO_LOOKUP_METHODPARAM; + rlookup->helper = CORINFO_HELP_RUNTIMEHANDLE_METHOD; + } + else if (pCallerMD->RequiresInstMethodTableArg()) + { + lookup->lookupKind.runtimeLookupKind = CORINFO_LOOKUP_CLASSPARAM; + rlookup->helper = CORINFO_HELP_RUNTIMEHANDLE_CLASS; + } + else + { + lookup->lookupKind.runtimeLookupKind = CORINFO_LOOKUP_THISOBJ; + rlookup->helper = CORINFO_HELP_RUNTIMEHANDLE_CLASS; + } + + SigBuilder sigBuilder; + sigBuilder.AppendData(MethodDescSlot); + if (lookup->lookupKind.runtimeLookupKind != CORINFO_LOOKUP_METHODPARAM) + { + sigBuilder.AppendData(pCallerMD->GetMethodTable()->GetNumDicts() - 1); + } + + sigBuilder.AppendElementType(ELEMENT_TYPE_INTERNAL); + sigBuilder.AppendPointer(pTypicalAwaitMD->GetMethodTable()); + sigBuilder.AppendData(ENCODE_METHOD_SIG_MethodInstantiation | ENCODE_METHOD_SIG_InstantiatingStub); + sigBuilder.AppendElementType(ELEMENT_TYPE_INTERNAL); + sigBuilder.AppendPointer(pTypicalAwaitMD->GetMethodTable()); + sigBuilder.AppendData(RidFromToken(pTypicalAwaitMD->GetMemberDef())); + sigBuilder.AppendData(1); + + _ASSERTE(callSig->pSig != NULL); + SigPointer instantiation(callSig->pSig, callSig->cbSig); + BYTE callingConvention; + IfFailThrow(instantiation.GetByte(&callingConvention)); + _ASSERTE(callingConvention == IMAGE_CEE_CS_CALLCONV_GENERICINST); + uint32_t numArgs; + IfFailThrow(instantiation.GetData(&numArgs)); + _ASSERTE(numArgs == 1); + instantiation.ConvertToInternalExactlyOne(GetModule(callSig->scope), NULL, &sigBuilder); + + FinishComputeRuntimeLookup(sigBuilder, pCallerMD, lookup); +} + static MethodTable* getContinuationType( size_t dataSize, bool* objRefs, diff --git a/src/coreclr/vm/jitinterface.h b/src/coreclr/vm/jitinterface.h index 62280728dc1cba..5072ab4c854889 100644 --- a/src/coreclr/vm/jitinterface.h +++ b/src/coreclr/vm/jitinterface.h @@ -413,6 +413,11 @@ class CEEInfo : public ICorJitInfo CORINFO_LOOKUP* pResultLookup); void ComputeRuntimeLookupForAwaitCall(MethodDesc* pCallerMD, MethodDesc* pTypicalAwaitMD, CORINFO_LOOKUP* lookup); + void ComputeRuntimeLookupForAwaitAwaiterInContinuationCall( + MethodDesc* pCallerMD, + MethodDesc* pTypicalAwaitMD, + CORINFO_SIG_INFO* callSig, + CORINFO_LOOKUP* lookup); #if defined(FEATURE_GDBJIT) CalledMethod * GetCalledMethods() { return m_pCalledMethods; } From 97716bea36c8965a0fe01ccc24693fcbfbbf4486 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Fri, 24 Jul 2026 19:35:30 +0200 Subject: [PATCH 03/19] Save nodes to change in previous IR walk --- src/coreclr/jit/async.cpp | 76 +++++++++++++++++++++------------------ src/coreclr/jit/async.h | 42 +++++++++++----------- 2 files changed, 63 insertions(+), 55 deletions(-) diff --git a/src/coreclr/jit/async.cpp b/src/coreclr/jit/async.cpp index 0096f2a911cc5b..6b88111633bee9 100644 --- a/src/coreclr/jit/async.cpp +++ b/src/coreclr/jit/async.cpp @@ -678,7 +678,8 @@ PhaseStatus AsyncTransformation::Run() PhaseStatus result = PhaseStatus::MODIFIED_NOTHING; ArrayStack blocksWithNormalAwaits(m_compiler->getAllocator(CMK_Async)); ArrayStack blocksWithTailAwaits(m_compiler->getAllocator(CMK_Async)); - AggregatedAwaitInfo awaits = FindAwaits(blocksWithNormalAwaits, blocksWithTailAwaits); + ArrayStack continuationMemberOffsets(m_compiler->getAllocator(CMK_Async)); + AggregatedAwaitInfo awaits = FindAwaits(blocksWithNormalAwaits, blocksWithTailAwaits, continuationMemberOffsets); if (awaits.NumNormalAwaits + awaits.NumTailAwaits > 1) { @@ -698,7 +699,8 @@ PhaseStatus AsyncTransformation::Run() // This may have changed blocks, so refind the normal awaits. blocksWithNormalAwaits.Reset(); blocksWithTailAwaits.Reset(); - awaits = FindAwaits(blocksWithNormalAwaits, blocksWithTailAwaits); + continuationMemberOffsets.Reset(); + awaits = FindAwaits(blocksWithNormalAwaits, blocksWithTailAwaits, continuationMemberOffsets); } result = PhaseStatus::MODIFIED_EVERYTHING; @@ -708,6 +710,7 @@ PhaseStatus AsyncTransformation::Run() if (awaits.NumNormalAwaits <= 0) { + assert(continuationMemberOffsets.Empty()); return result; } @@ -822,12 +825,22 @@ PhaseStatus AsyncTransformation::Run() } } - CreateResumptionsAndSuspensions(); + const ContinuationLayout* continuationLayout = CreateResumptionsAndSuspensions(); // After transforming all async calls we have created resumption blocks; // create the resumption switch. CreateResumptionSwitch(); + for (int i = 0; i < continuationMemberOffsets.Height(); i++) + { + GenTree* node = continuationMemberOffsets.Bottom(i); + size_t memberIndex = node->AsVal()->gtVal1; + assert(memberIndex < continuationLayout->ContinuationMemberOffsets.size()); + ssize_t offset = (OFFSETOF__CORINFO_Continuation__data - SIZEOF__CORINFO_Object) + + continuationLayout->ContinuationMemberOffsets[memberIndex]; + node->BashToConst(offset, TYP_INT); + } + m_compiler->fgInvalidateDfsTree(); if (m_compiler->opts.OptimizationDisabled()) @@ -863,12 +876,14 @@ PhaseStatus AsyncTransformation::Run() // Parameters: // blocksWithNormalAwaits - [out] Blocks with normal awaits are pushed onto this stack // blocksWithTailAwaits - [out] Blocks with tail awaits are pushed onto this stack +// continuationMemberOffsets - [out] Symbolic continuation member offset nodes // // Returns: // Information about awaits in the function. // AggregatedAwaitInfo AsyncTransformation::FindAwaits(ArrayStack& blocksWithNormalAwaits, - ArrayStack& blocksWithTailAwaits) + ArrayStack& blocksWithTailAwaits, + ArrayStack& continuationMemberOffsets) { AggregatedAwaitInfo awaits; for (BasicBlock* block : m_compiler->Blocks()) @@ -877,6 +892,12 @@ AggregatedAwaitInfo AsyncTransformation::FindAwaits(ArrayStack& blo bool hasTailAwait = false; for (GenTree* tree : LIR::AsRange(block)) { + if (tree->OperIs(GT_CONTINUATION_MEMBER_OFFSET)) + { + continuationMemberOffsets.Push(tree); + continue; + } + if (!tree->IsCall()) { continue; @@ -2352,30 +2373,6 @@ void AsyncTransformation::StoreAsyncAwaiter(BasicBlock* callBlock, LIR::AsRange(suspendBB).InsertAtEnd(LIR::SeqTree(m_compiler, store)); } -//------------------------------------------------------------------------ -// AsyncTransformation::ReplaceContinuationMemberOffsets: -// Replace symbolic continuation member offsets with their final constants. -// -void AsyncTransformation::ReplaceContinuationMemberOffsets(const ContinuationLayout& layout) -{ - for (BasicBlock* block : m_compiler->Blocks()) - { - for (GenTree* node : LIR::AsRange(block)) - { - if (!node->OperIs(GT_CONTINUATION_MEMBER_OFFSET)) - { - continue; - } - - size_t memberIndex = node->AsVal()->gtVal1; - assert(memberIndex < layout.ContinuationMemberOffsets.size()); - ssize_t offset = (OFFSETOF__CORINFO_Continuation__data - SIZEOF__CORINFO_Object) + - layout.ContinuationMemberOffsets[memberIndex]; - node->BashToConst(offset, TYP_INT); - } - } -} - //------------------------------------------------------------------------ // AsyncTransformation::CreateAllocContinuationCall: // Create a call to the JIT helper that allocates a continuation. @@ -3829,7 +3826,7 @@ void AsyncTransformation::InsertFinishContextHandlingCall(BasicBlock* // Walk all recorded async states and create the suspension and resumption // IR, continuation layouts, and debug info for each one. // -void AsyncTransformation::CreateResumptionsAndSuspensions() +const ContinuationLayout* AsyncTransformation::CreateResumptionsAndSuspensions() { bool useSharedLayout = (m_states.size() > 1) && ReuseContinuations(); @@ -3840,7 +3837,6 @@ void AsyncTransformation::CreateResumptionsAndSuspensions() ContinuationLayoutBuilder* sharedLayoutBuilder = ContinuationLayoutBuilder::CreateSharedLayout(m_compiler, m_states); sharedLayout = sharedLayoutBuilder->Create(); - ReplaceContinuationMemberOffsets(*sharedLayout); unsigned numSharedSuspensionsWithContinuationContext = 0; unsigned numSharedSuspensionsWithoutContinuationContext = 0; @@ -3902,18 +3898,27 @@ void AsyncTransformation::CreateResumptionsAndSuspensions() } } - bool replacedContinuationMemberOffsets = sharedLayout != nullptr; + const ContinuationLayout* continuationLayout = sharedLayout; JITDUMP("Creating suspensions and resumptions for %zu states\n", m_states.size()); for (const AsyncState& state : m_states) { JITDUMP("State %u suspend @ " FMT_BB ", resume @ " FMT_BB "\n", state.Number, state.SuspensionBB->bbNum, state.ResumptionBB->bbNum); ContinuationLayout* layout = sharedLayout == nullptr ? state.Layout->Create() : sharedLayout; - if (!replacedContinuationMemberOffsets) + if (continuationLayout == nullptr) + { + continuationLayout = layout; + } +#ifdef DEBUG + else { - ReplaceContinuationMemberOffsets(*layout); - replacedContinuationMemberOffsets = true; + assert(continuationLayout->ContinuationMemberOffsets.size() == layout->ContinuationMemberOffsets.size()); + for (size_t i = 0; i < continuationLayout->ContinuationMemberOffsets.size(); i++) + { + assert(continuationLayout->ContinuationMemberOffsets[i] == layout->ContinuationMemberOffsets[i]); + } } +#endif CreateSuspension(state.CallBlock, state.Call, state.SuspensionBB, state.Number, *layout, *state.Layout, state.ResumeReachable, state.MutatedSincePreviousResumption); CreateResumption(state.CallBlock, state.Call, state.ResumptionBB, state.CallDefInfo, *layout, *state.Layout); @@ -3921,6 +3926,9 @@ void AsyncTransformation::CreateResumptionsAndSuspensions() JITDUMP("\n"); } + + assert(continuationLayout != nullptr); + return continuationLayout; } //------------------------------------------------------------------------ diff --git a/src/coreclr/jit/async.h b/src/coreclr/jit/async.h index 4f907d84b9bcf0..9a3a88503fbc17 100644 --- a/src/coreclr/jit/async.h +++ b/src/coreclr/jit/async.h @@ -394,7 +394,8 @@ class AsyncTransformation unsigned m_sharedFinishContextHandlingSyncContextVar = BAD_VAR_NUM; AggregatedAwaitInfo FindAwaits(ArrayStack& blocksWithNormalAwaits, - ArrayStack& blocksWithTailAwaits); + ArrayStack& blocksWithTailAwaits, + ArrayStack& continuationMemberOffsets); void TransformTailAwaits(ArrayStack& blocksWithTailAwaits); void TransformTailAwait(BasicBlock* block, GenTreeCall* call, BasicBlock** remainder); @@ -479,7 +480,6 @@ class AsyncTransformation GenTreeCall* call, BasicBlock* suspendBB, const ContinuationLayout& layout); - void ReplaceContinuationMemberOffsets(const ContinuationLayout& layout); void CreateCheckAndSuspendAfterCall(BasicBlock* block, GenTreeCall* call, const CallDefinitionInfo& callDefInfo, @@ -514,25 +514,25 @@ class AsyncTransformation var_types storeType, GenTreeFlags indirFlags = GTF_IND_NONFAULTING); - void CreateDebugInfoForSuspensionPoint(const ContinuationLayout& layout, - const ContinuationLayoutBuilder& subLayout); - unsigned GetReturnedContinuationVar(); - unsigned GetNewContinuationVar(); - unsigned GetResultBaseVar(); - unsigned GetExceptionVar(); - void CreateSharedReturnBB(); - BasicBlock* CreateSharedFinishContextHandlingBB(SuspensionContextHelper helper, - const ContinuationLayout& layout, - bool execContextMayVary, - bool syncContextMayVary); - void InsertFinishContextHandlingCall(BasicBlock* block, - const ContinuationLayout& layout, - SuspensionContextHelper helper, - GenTree* execContext, - GenTree* syncContext); - bool ReuseContinuations(); - void CreateResumptionsAndSuspensions(); - void CreateResumptionSwitch(); + void CreateDebugInfoForSuspensionPoint(const ContinuationLayout& layout, + const ContinuationLayoutBuilder& subLayout); + unsigned GetReturnedContinuationVar(); + unsigned GetNewContinuationVar(); + unsigned GetResultBaseVar(); + unsigned GetExceptionVar(); + void CreateSharedReturnBB(); + BasicBlock* CreateSharedFinishContextHandlingBB(SuspensionContextHelper helper, + const ContinuationLayout& layout, + bool execContextMayVary, + bool syncContextMayVary); + void InsertFinishContextHandlingCall(BasicBlock* block, + const ContinuationLayout& layout, + SuspensionContextHelper helper, + GenTree* execContext, + GenTree* syncContext); + bool ReuseContinuations(); + const ContinuationLayout* CreateResumptionsAndSuspensions(); + void CreateResumptionSwitch(); public: AsyncTransformation(Compiler* comp) From 3db54457011973b74038a14baa53859cc4debcd5 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Fri, 24 Jul 2026 20:11:24 +0200 Subject: [PATCH 04/19] Managed implementation --- .../CompilerServices/AsyncHelpers.CoreCLR.cs | 14 +++- .../Compiler/ReadyToRunCodegenCompilation.cs | 5 +- .../Runtime/CompilerServices/AsyncHelpers.cs | 54 +++++++++++++++ src/tests/async/struct/struct.cs | 69 +++++++++++++++++++ 4 files changed, 138 insertions(+), 4 deletions(-) diff --git a/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncHelpers.CoreCLR.cs b/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncHelpers.CoreCLR.cs index fd3e75881619ff..2fd4bb51f3b564 100644 --- a/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncHelpers.CoreCLR.cs +++ b/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncHelpers.CoreCLR.cs @@ -223,6 +223,8 @@ private ref struct RuntimeAsyncStackState public INotifyCompletion? Notifier; public ValueTaskSourceContinuation? ValueTaskSourceContinuation; public RuntimeAsyncTaskContinuation? TaskContinuation; + public delegate* AwaiterContinuation; + public int AwaiterOffset; // When we suspend in the leaf, the contexts are captured into these fields. public ExecutionContext? LeafExecutionContext; @@ -265,6 +267,8 @@ public void CaptureContexts() [NonVersionable] public void Push(RuntimeAsyncStackState* stackState) { + stackState->AwaiterContinuation = null; + stackState->AwaiterOffset = 0; stackState->Next = StackState; StackState = stackState; CurrentThread ??= Thread.CurrentThread; @@ -805,7 +809,15 @@ internal unsafe bool HandleSuspended(ref RuntimeAsyncAwaitState state) try { - if (stackState->CriticalNotifier is { } critNotifier) + if (stackState->AwaiterContinuation != null) + { + // The awaiter is stored in the continuation for the caller of + // AwaitAwaiterInContinuation or UnsafeAwaitAwaiterInContinuation. + Debug.Assert((headContinuation.Flags & ContinuationFlags.AllContinuationFlags) == 0); + stackState->AwaiterContinuation( + headContinuation, stackState->AwaiterOffset, GetContinuationAction()); + } + else if (stackState->CriticalNotifier is { } critNotifier) { // Result of async call to AwaitAwaiter or UnsafeAwaitAwaiter. // These never have special continuation context handling. diff --git a/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/ReadyToRunCodegenCompilation.cs b/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/ReadyToRunCodegenCompilation.cs index da3570706401ef..1a7a9012ecd2c8 100644 --- a/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/ReadyToRunCodegenCompilation.cs +++ b/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/ReadyToRunCodegenCompilation.cs @@ -1042,7 +1042,6 @@ private void AddNecessaryAsyncReferences(MethodDesc method) MetadataType taskOfTType = TypeSystemContext.SystemModule.GetKnownType("System.Threading.Tasks"u8, "Task`1"u8); MetadataType valueTaskOfTType = TypeSystemContext.SystemModule.GetKnownType("System.Threading.Tasks"u8, "ValueTask`1"u8); TypeDesc methodVar = TypeSystemContext.GetSignatureVariable(0, method: true); - TypeDesc intType = TypeSystemContext.GetWellKnownType(WellKnownType.Int32); MethodDesc[] requiredMethods = [ // For CorInfoImpl.getAsyncInfo @@ -1065,8 +1064,8 @@ private void AddNecessaryAsyncReferences(MethodDesc method) asyncHelpers.GetKnownMethod("TransparentAwait"u8, new MethodSignature(MethodSignatureFlags.Static, 0, voidType, [valueTaskType])), asyncHelpers.GetKnownMethod("TransparentAwait"u8, new MethodSignature(MethodSignatureFlags.Static, 1, methodVar, [taskOfTType.MakeInstantiatedType(methodVar)])), asyncHelpers.GetKnownMethod("TransparentAwait"u8, new MethodSignature(MethodSignatureFlags.Static, 1, methodVar, [valueTaskOfTType.MakeInstantiatedType(methodVar)])), - asyncHelpers.GetKnownMethod("AwaitAwaiterInContinuation"u8, new MethodSignature(MethodSignatureFlags.Static, 1, voidType, [methodVar, intType])), - asyncHelpers.GetKnownMethod("UnsafeAwaitAwaiterInContinuation"u8, new MethodSignature(MethodSignatureFlags.Static, 1, voidType, [methodVar, intType])), + asyncHelpers.GetKnownMethod("AwaitAwaiterInContinuation"u8, null), + asyncHelpers.GetKnownMethod("UnsafeAwaitAwaiterInContinuation"u8, null), ]; var moduleForNewReferences = ((EcmaMethod)method.GetPrimaryMethodDesc().GetTypicalMethodDefinition()).Module; _tokenManager.EnsureDefTokensAreAvailable([..requiredMethods, ..requiredTypes, ..requiredFields], moduleForNewReferences, true); diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncHelpers.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncHelpers.cs index 7e11855f9b82d0..d822d241cce2b4 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncHelpers.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncHelpers.cs @@ -36,6 +36,30 @@ public static unsafe void AwaitAwaiter(TAwaiter awaiter) where TAwaite AsyncSuspend(sentinelContinuation); } + [BypassReadyToRun] + [MethodImpl(MethodImplOptions.NoInlining | MethodImplOptions.Async)] + [StackTraceHidden] + private static unsafe void AwaitAwaiterInContinuation(TAwaiter awaiter, int offset) + where TAwaiter : INotifyCompletion + { + ref RuntimeAsyncAwaitState state = ref t_runtimeAsyncAwaitState; + Continuation? sentinelContinuation = state.SentinelContinuation ??= new Continuation(); + state.StackState->AwaiterContinuation = &AwaiterOnCompletedFromContinuation; + state.StackState->AwaiterOffset = offset; + state.CaptureContexts(); + AsyncSuspend(sentinelContinuation); + } + + private static void AwaiterOnCompletedFromContinuation( + Continuation headContinuation, int offset, Action continuation) + where TAwaiter : INotifyCompletion + { + Continuation awaiterContinuation = headContinuation.Next!; + ref byte data = ref RuntimeHelpers.GetRawData(awaiterContinuation); + TAwaiter awaiter = Unsafe.As(ref Unsafe.Add(ref data, offset)); + awaiter.OnCompleted(continuation); + } + // Must be NoInlining because we use AsyncSuspend to manufacture an explicit suspension point. // It will not capture/restore any local state that is live across it. @@ -70,6 +94,36 @@ public static unsafe void UnsafeAwaitAwaiter(TAwaiter awaiter) where T AsyncSuspend(sentinelContinuation); } + [BypassReadyToRun] + [MethodImpl(MethodImplOptions.NoInlining | MethodImplOptions.Async)] + [StackTraceHidden] + private static unsafe void UnsafeAwaitAwaiterInContinuation(TAwaiter awaiter, int offset) + where TAwaiter : ICriticalNotifyCompletion + { + ref RuntimeAsyncAwaitState state = ref t_runtimeAsyncAwaitState; + Continuation? sentinelContinuation = state.SentinelContinuation ??= new Continuation(); + state.StackState->AwaiterContinuation = &UnsafeAwaiterOnCompletedFromContinuation; + state.StackState->AwaiterOffset = offset; + state.CaptureContexts(); + AsyncSuspend(sentinelContinuation); + } + + private static void UnsafeAwaiterOnCompletedFromContinuation( + Continuation headContinuation, int offset, Action continuation) + where TAwaiter : ICriticalNotifyCompletion + { + if (typeof(TAwaiter) == typeof(YieldAwaitable.YieldAwaiter)) + { + RuntimeAsyncYielder.Instance.UnsafeOnCompleted(continuation); + return; + } + + Continuation awaiterContinuation = headContinuation.Next!; + ref byte data = ref RuntimeHelpers.GetRawData(awaiterContinuation); + TAwaiter awaiter = Unsafe.As(ref Unsafe.Add(ref data, offset)); + awaiter.UnsafeOnCompleted(continuation); + } + /// /// Awaits the specified and returns its result, throwing any exception produced by the task. /// diff --git a/src/tests/async/struct/struct.cs b/src/tests/async/struct/struct.cs index b8d8ab53b14077..a58297e1c753a4 100644 --- a/src/tests/async/struct/struct.cs +++ b/src/tests/async/struct/struct.cs @@ -5,16 +5,21 @@ using System; using System.Runtime.CompilerServices; +using System.Threading; using System.Threading.Tasks; using Xunit; public class Async2Struct { + private static int s_safeAwaiterValue; + private static int s_unsafeAwaiterValue; + [Fact] public static void TestEntryPoint() { Async().Wait(); Async2().Wait(); + CustomAwaiters().Wait(); } [System.Runtime.CompilerServices.RuntimeAsyncMethodGeneration(false)] @@ -32,6 +37,70 @@ private static async Task Async2() AssertEqual(100, s.Value); } + private static async Task CustomAwaiters() + { + Volatile.Write(ref s_safeAwaiterValue, 0); + await new SafeAwaitable(42); + AssertEqual(42, Volatile.Read(ref s_safeAwaiterValue)); + + Volatile.Write(ref s_unsafeAwaiterValue, 0); + await new UnsafeAwaitable(43); + AssertEqual(43, Volatile.Read(ref s_unsafeAwaiterValue)); + } + + private readonly struct SafeAwaitable + { + private readonly int _value; + + public SafeAwaitable(int value) => _value = value; + + public SafeAwaiter GetAwaiter() => new SafeAwaiter(_value); + + public readonly struct SafeAwaiter : INotifyCompletion + { + private readonly int _value; + + public SafeAwaiter(int value) => _value = value; + + public bool IsCompleted => false; + + public void OnCompleted(Action continuation) + { + Volatile.Write(ref s_safeAwaiterValue, _value); + ThreadPool.QueueUserWorkItem(static state => ((Action)state!).Invoke(), continuation); + } + + public void GetResult() => AssertEqual(42, _value); + } + } + + private readonly struct UnsafeAwaitable + { + private readonly int _value; + + public UnsafeAwaitable(int value) => _value = value; + + public UnsafeAwaiter GetAwaiter() => new UnsafeAwaiter(_value); + + public readonly struct UnsafeAwaiter : ICriticalNotifyCompletion + { + private readonly int _value; + + public UnsafeAwaiter(int value) => _value = value; + + public bool IsCompleted => false; + + public void OnCompleted(Action continuation) => throw new InvalidOperationException(); + + public void UnsafeOnCompleted(Action continuation) + { + Volatile.Write(ref s_unsafeAwaiterValue, _value); + ThreadPool.UnsafeQueueUserWorkItem(static state => ((Action)state!).Invoke(), continuation); + } + + public void GetResult() => AssertEqual(43, _value); + } + } [MethodImpl(MethodImplOptions.NoInlining)] private static void AssertEqual(int expected, int val) From bd94f92ba3e59bdc37ae57d87b6c5ffd5c5fd0de Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Fri, 24 Jul 2026 20:43:28 +0200 Subject: [PATCH 05/19] Refactor --- src/coreclr/jit/compiler.h | 9 +++ src/coreclr/jit/gentree.h | 1 - src/coreclr/jit/importercalls.cpp | 117 +++++++++++++++++++----------- 3 files changed, 84 insertions(+), 43 deletions(-) diff --git a/src/coreclr/jit/compiler.h b/src/coreclr/jit/compiler.h index 96b4188427c229..b54967c66114c3 100644 --- a/src/coreclr/jit/compiler.h +++ b/src/coreclr/jit/compiler.h @@ -5247,6 +5247,15 @@ class Compiler unsigned clsFlags, bool isReadonlyCall); + void impTryOptimizeAwaitAwaiter(GenTreeCall* call, + CORINFO_RESOLVED_TOKEN* pResolvedToken, + CORINFO_SIG_INFO* sig, + CORINFO_CALL_INFO* callInfo, + CORINFO_METHOD_HANDLE* methHnd, + CORINFO_CONTEXT_HANDLE* exactContextHnd, + GenTree** instParam, + NamedIntrinsic ni); + void impSetupAsyncCall(GenTreeCall* call, CORINFO_METHOD_HANDLE methHnd, OPCODE opcode, unsigned prefixFlags, NamedIntrinsic ni, const DebugInfo& callDI); void impInsertAsyncArgsForLdvirtftnCall(GenTreeCall* call); diff --git a/src/coreclr/jit/gentree.h b/src/coreclr/jit/gentree.h index 27f75d8ee4622e..b451be41097493 100644 --- a/src/coreclr/jit/gentree.h +++ b/src/coreclr/jit/gentree.h @@ -4935,7 +4935,6 @@ class CallArg CORINFO_CLASS_HANDLE GetSignatureClassHandle() { return m_signatureLayout == nullptr ? NO_CLASS_HANDLE : m_signatureLayout->GetClassHandle(); } var_types GetSignatureType() { return m_signatureType; } WellKnownArg GetWellKnownArg() { return m_wellKnownArg; } - void SetWellKnownArg(WellKnownArg arg) { m_wellKnownArg = arg; } // clang-format on // Get the real argument node, i.e. not a setup or placeholder node. diff --git a/src/coreclr/jit/importercalls.cpp b/src/coreclr/jit/importercalls.cpp index 2335dc790954b1..1c49f4540a1fd0 100644 --- a/src/coreclr/jit/importercalls.cpp +++ b/src/coreclr/jit/importercalls.cpp @@ -128,6 +128,79 @@ GenTree* Compiler::impGetInstParamArg(CORINFO_RESOLVED_TOKEN* pResolvedToken, return instParam; } +//------------------------------------------------------------------------ +// impTryOptimizeAwaitAwaiter: +// Rewrite a struct AwaitAwaiter or UnsafeAwaitAwaiter call to use the +// corresponding helper that reads the awaiter from the continuation. +// +void Compiler::impTryOptimizeAwaitAwaiter(GenTreeCall* call, + CORINFO_RESOLVED_TOKEN* pResolvedToken, + CORINFO_SIG_INFO* sig, + CORINFO_CALL_INFO* callInfo, + CORINFO_METHOD_HANDLE* methHnd, + CORINFO_CONTEXT_HANDLE* exactContextHnd, + GenTree** instParam, + NamedIntrinsic ni) +{ + CallArg* awaiterArg = call->gtArgs.GetUserArgByIndex(0); + if (!varTypeIsStruct(awaiterArg->GetSignatureType())) + { + return; + } + + CORINFO_LOOKUP newInstArgLookup; + CORINFO_SIG_INFO lookupSig = *sig; + if (pResolvedToken->pMethodSpec != nullptr) + { + lookupSig.pSig = pResolvedToken->pMethodSpec; + lookupSig.cbSig = pResolvedToken->cbMethodSpec; + lookupSig.scope = pResolvedToken->tokenScope; + } + + bool isUnsafe = ni == NI_System_Runtime_CompilerServices_AsyncHelpers_UnsafeAwaitAwaiter; + CORINFO_CONTEXT_HANDLE newExactContextHnd; + CORINFO_METHOD_HANDLE newMethod = + info.compCompHnd->getAwaitAwaiterInContinuationCall(info.compMethodHnd, &lookupSig, isUnsafe, + &newExactContextHnd, &newInstArgLookup); + + CORINFO_SIG_INFO newSig; + info.compCompHnd->getMethodSig(newMethod, &newSig); + + GenTree* newInstParam = nullptr; + if (newSig.hasTypeArg()) + { + newInstParam = impLookupToTree(&newInstArgLookup, GTF_ICON_METHOD_HDL, newMethod); + if (newInstParam == nullptr) + { + JITDUMP("Failed to optimize awaiter call [%06u] because its replacement lookup could not be created\n", + dspTreeID(call)); + return; + } + } + + JITDUMP("Optimizing awaiter call [%06u] to read its struct awaiter from the continuation\n", dspTreeID(call)); + + *methHnd = newMethod; + *exactContextHnd = newExactContextHnd; + call->gtCallMethHnd = newMethod; + callInfo->hMethod = newMethod; + callInfo->methodFlags = info.compCompHnd->getMethodAttribs(newMethod); + callInfo->sig = newSig; + *instParam = newInstParam; + + GenTree* awaiter = awaiterArg->GetNode(); + var_types awaiterType = awaiterArg->GetSignatureType(); + ClassLayout* awaiterLayout = awaiterArg->GetSignatureLayout(); + call->gtArgs.Remove(awaiterArg); + call->gtArgs + .PushFront(this, NewCallArg::Struct(awaiter, awaiterType, awaiterLayout).WellKnown(WellKnownArg::AsyncAwaiter)); + + size_t memberIndex = GetContinuationMemberIndex(ContinuationMember::CustomAwaiterOfLayout(awaiterLayout)); + GenTree* offset = + new (this, GT_CONTINUATION_MEMBER_OFFSET) GenTreeVal(GT_CONTINUATION_MEMBER_OFFSET, TYP_INT, memberIndex); + call->gtArgs.PushBack(this, NewCallArg::Primitive(offset)); +} + //------------------------------------------------------------------------ // impImportCall: import a call-inspiring opcode // @@ -956,48 +1029,8 @@ var_types Compiler::impImportCall(OPCODE opcode, if (opts.OptimizationEnabled() && ((ni == NI_System_Runtime_CompilerServices_AsyncHelpers_AwaitAwaiter) || (ni == NI_System_Runtime_CompilerServices_AsyncHelpers_UnsafeAwaitAwaiter))) { - CallArg* awaiterArg = call->AsCall()->gtArgs.GetUserArgByIndex(0); - if (varTypeIsStruct(awaiterArg->GetSignatureType())) - { - CORINFO_LOOKUP newInstArgLookup; - CORINFO_SIG_INFO lookupSig = *sig; - if (pResolvedToken->pMethodSpec != nullptr) - { - lookupSig.pSig = pResolvedToken->pMethodSpec; - lookupSig.cbSig = pResolvedToken->cbMethodSpec; - lookupSig.scope = pResolvedToken->tokenScope; - } - CORINFO_METHOD_HANDLE newMethod = info.compCompHnd->getAwaitAwaiterInContinuationCall( - info.compMethodHnd, &lookupSig, - ni == NI_System_Runtime_CompilerServices_AsyncHelpers_UnsafeAwaitAwaiter, &exactContextHnd, - &newInstArgLookup); - - methHnd = newMethod; - call->AsCall()->gtCallMethHnd = newMethod; - callInfo->hMethod = newMethod; - callInfo->methodFlags = info.compCompHnd->getMethodAttribs(newMethod); - info.compCompHnd->getMethodSig(newMethod, &callInfo->sig); - - awaiterArg->SetWellKnownArg(WellKnownArg::AsyncAwaiter); - size_t memberIndex = - GetContinuationMemberIndex(ContinuationMember::CustomAwaiterOfLayout(awaiterArg->GetSignatureLayout())); - GenTree* offset = new (this, GT_CONTINUATION_MEMBER_OFFSET) - GenTreeVal(GT_CONTINUATION_MEMBER_OFFSET, TYP_INT, memberIndex); - call->AsCall()->gtArgs.PushBack(this, NewCallArg::Primitive(offset)); - - if (callInfo->sig.hasTypeArg()) - { - instParam = impLookupToTree(&newInstArgLookup, GTF_ICON_METHOD_HDL, newMethod); - if (instParam == nullptr) - { - return TYP_UNDEF; - } - } - else - { - instParam = nullptr; - } - } + impTryOptimizeAwaitAwaiter(call->AsCall(), pResolvedToken, sig, callInfo, &methHnd, &exactContextHnd, + &instParam, ni); } // Extra args From cc498a28b78181b94dc62838a092293094ba9edd Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Fri, 24 Jul 2026 20:55:54 +0200 Subject: [PATCH 06/19] Avoid allocating unused ones --- src/coreclr/jit/async.cpp | 87 +++++++++++++++++++++++++++------------ src/coreclr/jit/async.h | 4 +- 2 files changed, 63 insertions(+), 28 deletions(-) diff --git a/src/coreclr/jit/async.cpp b/src/coreclr/jit/async.cpp index 6b88111633bee9..b944f2882c20c2 100644 --- a/src/coreclr/jit/async.cpp +++ b/src/coreclr/jit/async.cpp @@ -825,7 +825,7 @@ PhaseStatus AsyncTransformation::Run() } } - const ContinuationLayout* continuationLayout = CreateResumptionsAndSuspensions(); + const ContinuationLayout* continuationLayout = CreateResumptionsAndSuspensions(continuationMemberOffsets); // After transforming all async calls we have created resumption blocks; // create the resumption switch. @@ -836,6 +836,7 @@ PhaseStatus AsyncTransformation::Run() GenTree* node = continuationMemberOffsets.Bottom(i); size_t memberIndex = node->AsVal()->gtVal1; assert(memberIndex < continuationLayout->ContinuationMemberOffsets.size()); + assert(continuationLayout->ContinuationMemberOffsets[memberIndex] != UINT_MAX); ssize_t offset = (OFFSETOF__CORINFO_Continuation__data - SIZEOF__CORINFO_Object) + continuationLayout->ContinuationMemberOffsets[memberIndex]; node->BashToConst(offset, TYP_INT); @@ -1464,6 +1465,11 @@ void ContinuationLayout::Dump(int indent) for (size_t i = 0; i < ContinuationMemberOffsets.size(); i++) { + if (ContinuationMemberOffsets[i] == UINT_MAX) + { + continue; + } + printf("%*s +%03u ", indent, "", ContinuationMemberOffsets[i]); JitTls::GetCompiler()->GetContinuationMember(i).Print(); printf("\n"); @@ -1519,11 +1525,16 @@ const ReturnInfo* ContinuationLayout::FindReturn(Compiler* comp, GenTreeCall* ca // The finalized ContinuationLayout with computed offsets and a class // handle for the continuation type. // -ContinuationLayout* ContinuationLayoutBuilder::Create() +ContinuationLayout* ContinuationLayoutBuilder::Create(ArrayStack& continuationMemberOffsets) { ContinuationLayout* layout = new (m_compiler, CMK_Async) ContinuationLayout(m_compiler); layout->Locals.reserve(m_locals.size()); - layout->ContinuationMemberOffsets.reserve(m_compiler->GetContinuationMemberCount()); + size_t continuationMemberCount = m_compiler->GetContinuationMemberCount(); + layout->ContinuationMemberOffsets.reserve(continuationMemberCount); + for (size_t i = 0; i < continuationMemberCount; i++) + { + layout->ContinuationMemberOffsets.push_back(UINT_MAX); + } for (unsigned lclNum : m_locals) { @@ -1614,17 +1625,6 @@ ContinuationLayout* ContinuationLayoutBuilder::Create() layout->OSRAddressOffset = allocLayout(TARGET_POINTER_SIZE, TARGET_POINTER_SIZE); } - for (size_t i = 0; i < m_compiler->GetContinuationMemberCount(); i++) - { - ClassLayout* memberLayout = m_compiler->GetContinuationMember(i).GetCustomAwaiterLayout(); - unsigned alignment = - memberLayout->IsCustomLayout() - ? (memberLayout->HasGCPtr() ? TARGET_POINTER_SIZE : 1) - : m_compiler->info.compCompHnd->getClassAlignmentRequirement(memberLayout->GetClassHandle()); - layout->ContinuationMemberOffsets.push_back( - allocLayout(std::min(alignment, (unsigned)TARGET_POINTER_SIZE), memberLayout->GetSize())); - } - if (m_needsExecutionContext) { layout->ExecutionContextOffset = allocLayout(TARGET_POINTER_SIZE, TARGET_POINTER_SIZE); @@ -1648,6 +1648,24 @@ ContinuationLayout* ContinuationLayoutBuilder::Create() ret.Offset = allocLayout(ret.HeapAlignment(), ret.Size); } + for (int i = 0; i < continuationMemberOffsets.Height(); i++) + { + size_t memberIndex = continuationMemberOffsets.Bottom(i)->AsVal()->gtVal1; + assert(memberIndex < continuationMemberCount); + if (layout->ContinuationMemberOffsets[memberIndex] != UINT_MAX) + { + continue; + } + + ClassLayout* memberLayout = m_compiler->GetContinuationMember(memberIndex).GetCustomAwaiterLayout(); + unsigned alignment = + memberLayout->IsCustomLayout() + ? (memberLayout->HasGCPtr() ? TARGET_POINTER_SIZE : 1) + : m_compiler->info.compCompHnd->getClassAlignmentRequirement(memberLayout->GetClassHandle()); + layout->ContinuationMemberOffsets[memberIndex] = + allocLayout(std::min(alignment, (unsigned)TARGET_POINTER_SIZE), memberLayout->GetSize()); + } + if (m_needsKeepAlive) { layout->KeepAliveOffset = allocLayout(TARGET_POINTER_SIZE, TARGET_POINTER_SIZE); @@ -1698,6 +1716,11 @@ ContinuationLayout* ContinuationLayoutBuilder::Create() for (size_t i = 0; i < layout->ContinuationMemberOffsets.size(); i++) { + if (layout->ContinuationMemberOffsets[i] == UINT_MAX) + { + continue; + } + bitmapBuilder.SetType(layout->ContinuationMemberOffsets[i], TYP_STRUCT, m_compiler->GetContinuationMember(i).GetCustomAwaiterLayout()); } @@ -2353,6 +2376,7 @@ void AsyncTransformation::StoreAsyncAwaiter(BasicBlock* callBlock, size_t memberIndex = m_compiler->GetContinuationMemberIndex(ContinuationMember::CustomAwaiterOfLayout(awaiterLayout)); assert(memberIndex < layout.ContinuationMemberOffsets.size()); + assert(layout.ContinuationMemberOffsets[memberIndex] != UINT_MAX); GenTree* awaiter = awaiterArg->GetNode(); if (!awaiter->OperIs(GT_LCL_VAR)) @@ -3826,7 +3850,8 @@ void AsyncTransformation::InsertFinishContextHandlingCall(BasicBlock* // Walk all recorded async states and create the suspension and resumption // IR, continuation layouts, and debug info for each one. // -const ContinuationLayout* AsyncTransformation::CreateResumptionsAndSuspensions() +const ContinuationLayout* AsyncTransformation::CreateResumptionsAndSuspensions( + ArrayStack& continuationMemberOffsets) { bool useSharedLayout = (m_states.size() > 1) && ReuseContinuations(); @@ -3836,7 +3861,7 @@ const ContinuationLayout* AsyncTransformation::CreateResumptionsAndSuspensions() JITDUMP("Creating shared layout:\n"); ContinuationLayoutBuilder* sharedLayoutBuilder = ContinuationLayoutBuilder::CreateSharedLayout(m_compiler, m_states); - sharedLayout = sharedLayoutBuilder->Create(); + sharedLayout = sharedLayoutBuilder->Create(continuationMemberOffsets); unsigned numSharedSuspensionsWithContinuationContext = 0; unsigned numSharedSuspensionsWithoutContinuationContext = 0; @@ -3898,24 +3923,33 @@ const ContinuationLayout* AsyncTransformation::CreateResumptionsAndSuspensions() } } - const ContinuationLayout* continuationLayout = sharedLayout; + const ContinuationLayout* firstLayout = sharedLayout; + const ContinuationLayout* continuationMemberLayout = sharedLayout; JITDUMP("Creating suspensions and resumptions for %zu states\n", m_states.size()); for (const AsyncState& state : m_states) { JITDUMP("State %u suspend @ " FMT_BB ", resume @ " FMT_BB "\n", state.Number, state.SuspensionBB->bbNum, state.ResumptionBB->bbNum); - ContinuationLayout* layout = sharedLayout == nullptr ? state.Layout->Create() : sharedLayout; - if (continuationLayout == nullptr) + ContinuationLayout* layout = + sharedLayout == nullptr ? state.Layout->Create(continuationMemberOffsets) : sharedLayout; + if (firstLayout == nullptr) { - continuationLayout = layout; + firstLayout = layout; + } + + bool hasContinuationMember = state.Call->gtArgs.FindWellKnownArg(WellKnownArg::AsyncAwaiter) != nullptr; + if (hasContinuationMember && (continuationMemberLayout == nullptr)) + { + continuationMemberLayout = layout; } #ifdef DEBUG - else + else if (hasContinuationMember) { - assert(continuationLayout->ContinuationMemberOffsets.size() == layout->ContinuationMemberOffsets.size()); - for (size_t i = 0; i < continuationLayout->ContinuationMemberOffsets.size(); i++) + assert(continuationMemberLayout->ContinuationMemberOffsets.size() == + layout->ContinuationMemberOffsets.size()); + for (size_t i = 0; i < continuationMemberLayout->ContinuationMemberOffsets.size(); i++) { - assert(continuationLayout->ContinuationMemberOffsets[i] == layout->ContinuationMemberOffsets[i]); + assert(continuationMemberLayout->ContinuationMemberOffsets[i] == layout->ContinuationMemberOffsets[i]); } } #endif @@ -3927,8 +3961,9 @@ const ContinuationLayout* AsyncTransformation::CreateResumptionsAndSuspensions() JITDUMP("\n"); } - assert(continuationLayout != nullptr); - return continuationLayout; + assert(firstLayout != nullptr); + assert((continuationMemberOffsets.Height() == 0) || (continuationMemberLayout != nullptr)); + return continuationMemberLayout != nullptr ? continuationMemberLayout : firstLayout; } //------------------------------------------------------------------------ diff --git a/src/coreclr/jit/async.h b/src/coreclr/jit/async.h index 9a3a88503fbc17..aabb7d2bb5f325 100644 --- a/src/coreclr/jit/async.h +++ b/src/coreclr/jit/async.h @@ -140,7 +140,7 @@ struct ContinuationLayoutBuilder static bool Equals(const ContinuationLayoutBuilder& a, const ContinuationLayoutBuilder& b); - struct ContinuationLayout* Create(); + struct ContinuationLayout* Create(ArrayStack& continuationMemberOffsets); static ContinuationLayoutBuilder* CreateSharedLayout(Compiler* comp, const jitstd::vector& states); @@ -531,7 +531,7 @@ class AsyncTransformation GenTree* execContext, GenTree* syncContext); bool ReuseContinuations(); - const ContinuationLayout* CreateResumptionsAndSuspensions(); + const ContinuationLayout* CreateResumptionsAndSuspensions(ArrayStack& continuationMemberOffsets); void CreateResumptionSwitch(); public: From 65826fb0c6462d3ab75330f941099dd8f50686ce Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Fri, 24 Jul 2026 20:59:01 +0200 Subject: [PATCH 07/19] Clean up --- src/coreclr/jit/async.cpp | 33 ++++++++++++--------------------- 1 file changed, 12 insertions(+), 21 deletions(-) diff --git a/src/coreclr/jit/async.cpp b/src/coreclr/jit/async.cpp index b944f2882c20c2..c94fd8a3a57f3b 100644 --- a/src/coreclr/jit/async.cpp +++ b/src/coreclr/jit/async.cpp @@ -831,10 +831,9 @@ PhaseStatus AsyncTransformation::Run() // create the resumption switch. CreateResumptionSwitch(); - for (int i = 0; i < continuationMemberOffsets.Height(); i++) + for (GenTree* node : continuationMemberOffsets.BottomUpOrder()) { - GenTree* node = continuationMemberOffsets.Bottom(i); - size_t memberIndex = node->AsVal()->gtVal1; + size_t memberIndex = node->AsVal()->gtVal1; assert(memberIndex < continuationLayout->ContinuationMemberOffsets.size()); assert(continuationLayout->ContinuationMemberOffsets[memberIndex] != UINT_MAX); ssize_t offset = (OFFSETOF__CORINFO_Continuation__data - SIZEOF__CORINFO_Object) + @@ -1465,14 +1464,12 @@ void ContinuationLayout::Dump(int indent) for (size_t i = 0; i < ContinuationMemberOffsets.size(); i++) { - if (ContinuationMemberOffsets[i] == UINT_MAX) + if (ContinuationMemberOffsets[i] != UINT_MAX) { - continue; + printf("%*s +%03u ", indent, "", ContinuationMemberOffsets[i]); + JitTls::GetCompiler()->GetContinuationMember(i).Print(); + printf("\n"); } - - printf("%*s +%03u ", indent, "", ContinuationMemberOffsets[i]); - JitTls::GetCompiler()->GetContinuationMember(i).Print(); - printf("\n"); } for (const LiveLocalInfo& inf : Locals) @@ -1530,11 +1527,7 @@ ContinuationLayout* ContinuationLayoutBuilder::Create(ArrayStack& cont ContinuationLayout* layout = new (m_compiler, CMK_Async) ContinuationLayout(m_compiler); layout->Locals.reserve(m_locals.size()); size_t continuationMemberCount = m_compiler->GetContinuationMemberCount(); - layout->ContinuationMemberOffsets.reserve(continuationMemberCount); - for (size_t i = 0; i < continuationMemberCount; i++) - { - layout->ContinuationMemberOffsets.push_back(UINT_MAX); - } + layout->ContinuationMemberOffsets.resize(continuationMemberCount, UINT_MAX); for (unsigned lclNum : m_locals) { @@ -1648,9 +1641,9 @@ ContinuationLayout* ContinuationLayoutBuilder::Create(ArrayStack& cont ret.Offset = allocLayout(ret.HeapAlignment(), ret.Size); } - for (int i = 0; i < continuationMemberOffsets.Height(); i++) + for (GenTree* memberOffsetNode : continuationMemberOffsets.BottomUpOrder()) { - size_t memberIndex = continuationMemberOffsets.Bottom(i)->AsVal()->gtVal1; + size_t memberIndex = memberOffsetNode->AsVal()->gtVal1; assert(memberIndex < continuationMemberCount); if (layout->ContinuationMemberOffsets[memberIndex] != UINT_MAX) { @@ -1716,13 +1709,11 @@ ContinuationLayout* ContinuationLayoutBuilder::Create(ArrayStack& cont for (size_t i = 0; i < layout->ContinuationMemberOffsets.size(); i++) { - if (layout->ContinuationMemberOffsets[i] == UINT_MAX) + if (layout->ContinuationMemberOffsets[i] != UINT_MAX) { - continue; + bitmapBuilder.SetType(layout->ContinuationMemberOffsets[i], TYP_STRUCT, + m_compiler->GetContinuationMember(i).GetCustomAwaiterLayout()); } - - bitmapBuilder.SetType(layout->ContinuationMemberOffsets[i], TYP_STRUCT, - m_compiler->GetContinuationMember(i).GetCustomAwaiterLayout()); } #ifdef DEBUG From fefcbfba9f00b7722656538d57c1f363c3f188e7 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Fri, 24 Jul 2026 21:55:35 +0200 Subject: [PATCH 08/19] Fixes --- src/coreclr/jit/async.cpp | 13 ++++++++++++- src/coreclr/jit/gentree.cpp | 1 + src/coreclr/jit/valuenum.cpp | 1 + .../tools/Common/JitInterface/CorInfoImpl.cs | 3 +-- .../System/Runtime/CompilerServices/AsyncHelpers.cs | 10 ++++------ 5 files changed, 19 insertions(+), 9 deletions(-) diff --git a/src/coreclr/jit/async.cpp b/src/coreclr/jit/async.cpp index c94fd8a3a57f3b..dd5ff687a4f0a2 100644 --- a/src/coreclr/jit/async.cpp +++ b/src/coreclr/jit/async.cpp @@ -2384,7 +2384,18 @@ void AsyncTransformation::StoreAsyncAwaiter(BasicBlock* callBlock, unsigned offset = OFFSETOF__CORINFO_Continuation__data + layout.ContinuationMemberOffsets[memberIndex]; GenTree* offsetNode = m_compiler->gtNewIconNode((ssize_t)offset, TYP_I_IMPL); GenTree* address = m_compiler->gtNewOperNode(GT_ADD, TYP_BYREF, continuation, offsetNode); - GenTree* store = m_compiler->gtNewStoreValueNode(awaiterLayout, address, awaiter, GTF_IND_NONFAULTING); + GenTree* store; + if (varTypeIsStruct(awaiter)) + { + store = m_compiler->gtNewStoreValueNode(awaiterLayout, address, awaiter, GTF_IND_NONFAULTING); + } + else + { + var_types storeType = + m_compiler->getPrimitiveTypeForStruct(awaiterLayout->GetSize(), awaiterLayout->GetClassHandle()); + assert(storeType != TYP_UNKNOWN); + store = m_compiler->gtNewStoreValueNode(storeType, address, awaiter, GTF_IND_NONFAULTING); + } LIR::AsRange(suspendBB).InsertAtEnd(LIR::SeqTree(m_compiler, store)); } diff --git a/src/coreclr/jit/gentree.cpp b/src/coreclr/jit/gentree.cpp index 0a5ac063271b3b..543d1d2ade5953 100644 --- a/src/coreclr/jit/gentree.cpp +++ b/src/coreclr/jit/gentree.cpp @@ -11919,6 +11919,7 @@ GenTreeUseEdgeIterator::GenTreeUseEdgeIterator(GenTree* node) case GT_CATCH_ARG: case GT_ASYNC_CONTINUATION: case GT_ASYNC_RESUME_INFO: + case GT_CONTINUATION_MEMBER_OFFSET: case GT_LABEL: case GT_FTN_ADDR: case GT_FTN_ENTRY: diff --git a/src/coreclr/jit/valuenum.cpp b/src/coreclr/jit/valuenum.cpp index 5b08ab7fea7142..6ab8c7ba469efb 100644 --- a/src/coreclr/jit/valuenum.cpp +++ b/src/coreclr/jit/valuenum.cpp @@ -13153,6 +13153,7 @@ void Compiler::fgValueNumberTree(GenTree* tree) case GT_CATCH_ARG: case GT_ASYNC_CONTINUATION: + case GT_CONTINUATION_MEMBER_OFFSET: case GT_SWIFT_ERROR: // We know nothing about the value of these. tree->gtVNPair.SetBoth(vnStore->VNForExpr(compCurBB, tree->TypeGet())); diff --git a/src/coreclr/tools/Common/JitInterface/CorInfoImpl.cs b/src/coreclr/tools/Common/JitInterface/CorInfoImpl.cs index c71feb7be6753b..2cc004d9c64ab7 100644 --- a/src/coreclr/tools/Common/JitInterface/CorInfoImpl.cs +++ b/src/coreclr/tools/Common/JitInterface/CorInfoImpl.cs @@ -3692,12 +3692,11 @@ private void getWasmWellKnownGlobals(ref CORINFO_WASM_WELLKNOWN_GLOBALS pWellKno CompilerTypeSystemContext context = _compilation.TypeSystemContext; DefType asyncHelpers = context.SystemModule.GetKnownType("System.Runtime.CompilerServices"u8, "AsyncHelpers"u8); - TypeDesc signatureVariable = context.GetSignatureVariable(0, method: true); MethodSignature signature = new MethodSignature( MethodSignatureFlags.Static, 1, context.GetWellKnownType(WellKnownType.Void), - [signatureVariable, context.GetWellKnownType(WellKnownType.Int32)]); + [context.GetWellKnownType(WellKnownType.Int32)]); MethodDesc result = asyncHelpers .GetKnownMethod( isUnsafe ? "UnsafeAwaitAwaiterInContinuation"u8 : "AwaitAwaiterInContinuation"u8, diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncHelpers.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncHelpers.cs index d822d241cce2b4..9e638aa9204141 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncHelpers.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncHelpers.cs @@ -39,7 +39,7 @@ public static unsafe void AwaitAwaiter(TAwaiter awaiter) where TAwaite [BypassReadyToRun] [MethodImpl(MethodImplOptions.NoInlining | MethodImplOptions.Async)] [StackTraceHidden] - private static unsafe void AwaitAwaiterInContinuation(TAwaiter awaiter, int offset) + private static unsafe void AwaitAwaiterInContinuation(int offset) where TAwaiter : INotifyCompletion { ref RuntimeAsyncAwaitState state = ref t_runtimeAsyncAwaitState; @@ -54,8 +54,7 @@ private static void AwaiterOnCompletedFromContinuation( Continuation headContinuation, int offset, Action continuation) where TAwaiter : INotifyCompletion { - Continuation awaiterContinuation = headContinuation.Next!; - ref byte data = ref RuntimeHelpers.GetRawData(awaiterContinuation); + ref byte data = ref RuntimeHelpers.GetRawData(headContinuation); TAwaiter awaiter = Unsafe.As(ref Unsafe.Add(ref data, offset)); awaiter.OnCompleted(continuation); } @@ -97,7 +96,7 @@ public static unsafe void UnsafeAwaitAwaiter(TAwaiter awaiter) where T [BypassReadyToRun] [MethodImpl(MethodImplOptions.NoInlining | MethodImplOptions.Async)] [StackTraceHidden] - private static unsafe void UnsafeAwaitAwaiterInContinuation(TAwaiter awaiter, int offset) + private static unsafe void UnsafeAwaitAwaiterInContinuation(int offset) where TAwaiter : ICriticalNotifyCompletion { ref RuntimeAsyncAwaitState state = ref t_runtimeAsyncAwaitState; @@ -118,8 +117,7 @@ private static void UnsafeAwaiterOnCompletedFromContinuation( return; } - Continuation awaiterContinuation = headContinuation.Next!; - ref byte data = ref RuntimeHelpers.GetRawData(awaiterContinuation); + ref byte data = ref RuntimeHelpers.GetRawData(headContinuation); TAwaiter awaiter = Unsafe.As(ref Unsafe.Add(ref data, offset)); awaiter.UnsafeOnCompleted(continuation); } From d5a1d256ab08608d3e3d37d434761231c2fe419f Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Mon, 27 Jul 2026 14:08:23 +0200 Subject: [PATCH 09/19] Add as a new test --- .../custom-struct-awaiters.cs | 85 +++++++++++++++++++ .../custom-struct-awaiters.csproj | 5 ++ src/tests/async/struct/struct.cs | 70 --------------- 3 files changed, 90 insertions(+), 70 deletions(-) create mode 100644 src/tests/async/custom-struct-awaiters/custom-struct-awaiters.cs create mode 100644 src/tests/async/custom-struct-awaiters/custom-struct-awaiters.csproj diff --git a/src/tests/async/custom-struct-awaiters/custom-struct-awaiters.cs b/src/tests/async/custom-struct-awaiters/custom-struct-awaiters.cs new file mode 100644 index 00000000000000..6946cb6975745b --- /dev/null +++ b/src/tests/async/custom-struct-awaiters/custom-struct-awaiters.cs @@ -0,0 +1,85 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Runtime.CompilerServices; +using System.Threading; +using System.Threading.Tasks; +using Xunit; + +public class CustomStructAwaiters +{ + private static int s_safeAwaiterValue; + private static int s_unsafeAwaiterValue; + + [Fact] + public static void TestEntryPoint() + { + Run().Wait(); + } + + private static async Task Run() + { + Volatile.Write(ref s_safeAwaiterValue, 0); + await new SafeAwaitable(42); + Assert.Equal(42, Volatile.Read(ref s_safeAwaiterValue)); + + Volatile.Write(ref s_unsafeAwaiterValue, 0); + await new UnsafeAwaitable(43); + Assert.Equal(43, Volatile.Read(ref s_unsafeAwaiterValue)); + } + + private readonly struct SafeAwaitable + { + private readonly int _value; + + public SafeAwaitable(int value) => _value = value; + + public SafeAwaiter GetAwaiter() => new SafeAwaiter(_value); + + public readonly struct SafeAwaiter : INotifyCompletion + { + private readonly int _value; + + public SafeAwaiter(int value) => _value = value; + + public bool IsCompleted => false; + + public void OnCompleted(Action continuation) + { + Volatile.Write(ref s_safeAwaiterValue, _value); + ThreadPool.QueueUserWorkItem(static state => ((Action)state!).Invoke(), continuation); + } + + public void GetResult() => Assert.Equal(42, _value); + } + } + + private readonly struct UnsafeAwaitable + { + private readonly int _value; + + public UnsafeAwaitable(int value) => _value = value; + + public UnsafeAwaiter GetAwaiter() => new UnsafeAwaiter(_value); + + public readonly struct UnsafeAwaiter : ICriticalNotifyCompletion + { + private readonly int _value; + + public UnsafeAwaiter(int value) => _value = value; + + public bool IsCompleted => false; + + public void OnCompleted(Action continuation) => throw new InvalidOperationException(); + + public void UnsafeOnCompleted(Action continuation) + { + Volatile.Write(ref s_unsafeAwaiterValue, _value); + ThreadPool.UnsafeQueueUserWorkItem(static state => ((Action)state!).Invoke(), continuation); + } + + public void GetResult() => Assert.Equal(43, _value); + } + } +} diff --git a/src/tests/async/custom-struct-awaiters/custom-struct-awaiters.csproj b/src/tests/async/custom-struct-awaiters/custom-struct-awaiters.csproj new file mode 100644 index 00000000000000..3fc50cde4b3443 --- /dev/null +++ b/src/tests/async/custom-struct-awaiters/custom-struct-awaiters.csproj @@ -0,0 +1,5 @@ + + + + + diff --git a/src/tests/async/struct/struct.cs b/src/tests/async/struct/struct.cs index a58297e1c753a4..df393687e5216e 100644 --- a/src/tests/async/struct/struct.cs +++ b/src/tests/async/struct/struct.cs @@ -5,21 +5,16 @@ using System; using System.Runtime.CompilerServices; -using System.Threading; using System.Threading.Tasks; using Xunit; public class Async2Struct { - private static int s_safeAwaiterValue; - private static int s_unsafeAwaiterValue; - [Fact] public static void TestEntryPoint() { Async().Wait(); Async2().Wait(); - CustomAwaiters().Wait(); } [System.Runtime.CompilerServices.RuntimeAsyncMethodGeneration(false)] @@ -37,71 +32,6 @@ private static async Task Async2() AssertEqual(100, s.Value); } - private static async Task CustomAwaiters() - { - Volatile.Write(ref s_safeAwaiterValue, 0); - await new SafeAwaitable(42); - AssertEqual(42, Volatile.Read(ref s_safeAwaiterValue)); - - Volatile.Write(ref s_unsafeAwaiterValue, 0); - await new UnsafeAwaitable(43); - AssertEqual(43, Volatile.Read(ref s_unsafeAwaiterValue)); - } - - private readonly struct SafeAwaitable - { - private readonly int _value; - - public SafeAwaitable(int value) => _value = value; - - public SafeAwaiter GetAwaiter() => new SafeAwaiter(_value); - - public readonly struct SafeAwaiter : INotifyCompletion - { - private readonly int _value; - - public SafeAwaiter(int value) => _value = value; - - public bool IsCompleted => false; - - public void OnCompleted(Action continuation) - { - Volatile.Write(ref s_safeAwaiterValue, _value); - ThreadPool.QueueUserWorkItem(static state => ((Action)state!).Invoke(), continuation); - } - - public void GetResult() => AssertEqual(42, _value); - } - } - - private readonly struct UnsafeAwaitable - { - private readonly int _value; - - public UnsafeAwaitable(int value) => _value = value; - - public UnsafeAwaiter GetAwaiter() => new UnsafeAwaiter(_value); - - public readonly struct UnsafeAwaiter : ICriticalNotifyCompletion - { - private readonly int _value; - - public UnsafeAwaiter(int value) => _value = value; - - public bool IsCompleted => false; - - public void OnCompleted(Action continuation) => throw new InvalidOperationException(); - - public void UnsafeOnCompleted(Action continuation) - { - Volatile.Write(ref s_unsafeAwaiterValue, _value); - ThreadPool.UnsafeQueueUserWorkItem(static state => ((Action)state!).Invoke(), continuation); - } - - public void GetResult() => AssertEqual(43, _value); - } - } - [MethodImpl(MethodImplOptions.NoInlining)] private static void AssertEqual(int expected, int val) { From 18fd270a5276efa8afb5dc2aca2fceb735a714f9 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Mon, 27 Jul 2026 14:22:30 +0200 Subject: [PATCH 10/19] Update tests to right SDK --- .../JIT/Regression/JitBlue/GitHub_130437/GitHub_130437.csproj | 2 +- .../async/awaitingnoasyncgeneric/awaitingnoasyncgeneric.csproj | 2 +- src/tests/async/awaitingnotasync/awaitingnotasync.csproj | 2 +- src/tests/async/byref-param/byref-param.csproj | 2 +- src/tests/async/cancellation/cancellation.csproj | 2 +- src/tests/async/collectible-alc/collectible-alc.csproj | 2 +- .../async/custom-struct-awaiters/custom-struct-awaiters.csproj | 2 +- src/tests/async/eh-microbench/eh-microbench.csproj | 2 +- src/tests/async/execution-context/execution-context.csproj | 2 +- src/tests/async/gc-roots-scan/gc-roots-scan.csproj | 2 +- src/tests/async/implement/implement.csproj | 2 +- src/tests/async/inst-unbox-thunks/inst-unbox-thunks.csproj | 2 +- .../async/mincallcost-microbench/mincallcost-microbench.csproj | 2 +- src/tests/async/object/object.csproj | 2 +- src/tests/async/objects-captured/objects-captured.csproj | 2 +- src/tests/async/override/override.csproj | 2 +- src/tests/async/pgo/pgo.csproj | 2 +- src/tests/async/pinvoke/pinvoke.csproj | 2 +- src/tests/async/regression/125042.csproj | 2 +- src/tests/async/regression/125615.csproj | 2 +- src/tests/async/regression/125805.csproj | 2 +- src/tests/async/regression/128566.csproj | 2 +- src/tests/async/regression/managed-thread-id.csproj | 2 +- src/tests/async/returns/returns.csproj | 2 +- src/tests/async/shared-generic/shared-generic.csproj | 2 +- src/tests/async/staticvirtual/staticvirtual.csproj | 2 +- src/tests/async/struct/struct.csproj | 2 +- .../synchronization-context/synchronization-context.csproj | 2 +- src/tests/async/valuetask-source/valuetask-source-stub.csproj | 2 +- src/tests/async/valuetask-source/valuetask-source.csproj | 2 +- src/tests/async/valuetask/valuetask.csproj | 2 +- src/tests/async/varying-yields/varying-yields-task.csproj | 2 +- src/tests/async/varying-yields/varying-yields-valuetask.csproj | 2 +- src/tests/async/varying-yields/varying-yields.csproj | 2 +- src/tests/async/void/void.csproj | 2 +- src/tests/async/with-yields/with-yields.csproj | 2 +- src/tests/async/without-yields/without-yields.csproj | 2 +- 37 files changed, 37 insertions(+), 37 deletions(-) diff --git a/src/tests/JIT/Regression/JitBlue/GitHub_130437/GitHub_130437.csproj b/src/tests/JIT/Regression/JitBlue/GitHub_130437/GitHub_130437.csproj index 8f66634434cd1a..f07af3a622402e 100644 --- a/src/tests/JIT/Regression/JitBlue/GitHub_130437/GitHub_130437.csproj +++ b/src/tests/JIT/Regression/JitBlue/GitHub_130437/GitHub_130437.csproj @@ -1,4 +1,4 @@ - + $(RepoRoot)src/coreclr/.nuget/Microsoft.NET.Sdk.IL/Microsoft.NET.Sdk.IL.targets.template True diff --git a/src/tests/async/awaitingnoasyncgeneric/awaitingnoasyncgeneric.csproj b/src/tests/async/awaitingnoasyncgeneric/awaitingnoasyncgeneric.csproj index 3fc50cde4b3443..197767e2c4e249 100644 --- a/src/tests/async/awaitingnoasyncgeneric/awaitingnoasyncgeneric.csproj +++ b/src/tests/async/awaitingnoasyncgeneric/awaitingnoasyncgeneric.csproj @@ -1,4 +1,4 @@ - + diff --git a/src/tests/async/awaitingnotasync/awaitingnotasync.csproj b/src/tests/async/awaitingnotasync/awaitingnotasync.csproj index 3fc50cde4b3443..197767e2c4e249 100644 --- a/src/tests/async/awaitingnotasync/awaitingnotasync.csproj +++ b/src/tests/async/awaitingnotasync/awaitingnotasync.csproj @@ -1,4 +1,4 @@ - + diff --git a/src/tests/async/byref-param/byref-param.csproj b/src/tests/async/byref-param/byref-param.csproj index 3fc50cde4b3443..197767e2c4e249 100644 --- a/src/tests/async/byref-param/byref-param.csproj +++ b/src/tests/async/byref-param/byref-param.csproj @@ -1,4 +1,4 @@ - + diff --git a/src/tests/async/cancellation/cancellation.csproj b/src/tests/async/cancellation/cancellation.csproj index 3fc50cde4b3443..197767e2c4e249 100644 --- a/src/tests/async/cancellation/cancellation.csproj +++ b/src/tests/async/cancellation/cancellation.csproj @@ -1,4 +1,4 @@ - + diff --git a/src/tests/async/collectible-alc/collectible-alc.csproj b/src/tests/async/collectible-alc/collectible-alc.csproj index 5bc9c9b10779eb..e65e440150c631 100644 --- a/src/tests/async/collectible-alc/collectible-alc.csproj +++ b/src/tests/async/collectible-alc/collectible-alc.csproj @@ -1,4 +1,4 @@ - + True diff --git a/src/tests/async/custom-struct-awaiters/custom-struct-awaiters.csproj b/src/tests/async/custom-struct-awaiters/custom-struct-awaiters.csproj index 3fc50cde4b3443..197767e2c4e249 100644 --- a/src/tests/async/custom-struct-awaiters/custom-struct-awaiters.csproj +++ b/src/tests/async/custom-struct-awaiters/custom-struct-awaiters.csproj @@ -1,4 +1,4 @@ - + diff --git a/src/tests/async/eh-microbench/eh-microbench.csproj b/src/tests/async/eh-microbench/eh-microbench.csproj index e0c0b671ef1829..6650af6b67607c 100644 --- a/src/tests/async/eh-microbench/eh-microbench.csproj +++ b/src/tests/async/eh-microbench/eh-microbench.csproj @@ -1,4 +1,4 @@ - + BuildOnly diff --git a/src/tests/async/execution-context/execution-context.csproj b/src/tests/async/execution-context/execution-context.csproj index 644025b102673e..3a8b32c0ae764d 100644 --- a/src/tests/async/execution-context/execution-context.csproj +++ b/src/tests/async/execution-context/execution-context.csproj @@ -1,4 +1,4 @@ - + diff --git a/src/tests/async/gc-roots-scan/gc-roots-scan.csproj b/src/tests/async/gc-roots-scan/gc-roots-scan.csproj index 638c237d91983d..21cc479969f131 100644 --- a/src/tests/async/gc-roots-scan/gc-roots-scan.csproj +++ b/src/tests/async/gc-roots-scan/gc-roots-scan.csproj @@ -1,4 +1,4 @@ - + BuildOnly diff --git a/src/tests/async/implement/implement.csproj b/src/tests/async/implement/implement.csproj index 644025b102673e..3a8b32c0ae764d 100644 --- a/src/tests/async/implement/implement.csproj +++ b/src/tests/async/implement/implement.csproj @@ -1,4 +1,4 @@ - + diff --git a/src/tests/async/inst-unbox-thunks/inst-unbox-thunks.csproj b/src/tests/async/inst-unbox-thunks/inst-unbox-thunks.csproj index 8e9fed2b02ad47..d06e5df35dd4a2 100644 --- a/src/tests/async/inst-unbox-thunks/inst-unbox-thunks.csproj +++ b/src/tests/async/inst-unbox-thunks/inst-unbox-thunks.csproj @@ -1,4 +1,4 @@ - + diff --git a/src/tests/async/mincallcost-microbench/mincallcost-microbench.csproj b/src/tests/async/mincallcost-microbench/mincallcost-microbench.csproj index e0c0b671ef1829..6650af6b67607c 100644 --- a/src/tests/async/mincallcost-microbench/mincallcost-microbench.csproj +++ b/src/tests/async/mincallcost-microbench/mincallcost-microbench.csproj @@ -1,4 +1,4 @@ - + BuildOnly diff --git a/src/tests/async/object/object.csproj b/src/tests/async/object/object.csproj index 3fc50cde4b3443..197767e2c4e249 100644 --- a/src/tests/async/object/object.csproj +++ b/src/tests/async/object/object.csproj @@ -1,4 +1,4 @@ - + diff --git a/src/tests/async/objects-captured/objects-captured.csproj b/src/tests/async/objects-captured/objects-captured.csproj index 3fc50cde4b3443..197767e2c4e249 100644 --- a/src/tests/async/objects-captured/objects-captured.csproj +++ b/src/tests/async/objects-captured/objects-captured.csproj @@ -1,4 +1,4 @@ - + diff --git a/src/tests/async/override/override.csproj b/src/tests/async/override/override.csproj index 3fc50cde4b3443..197767e2c4e249 100644 --- a/src/tests/async/override/override.csproj +++ b/src/tests/async/override/override.csproj @@ -1,4 +1,4 @@ - + diff --git a/src/tests/async/pgo/pgo.csproj b/src/tests/async/pgo/pgo.csproj index 8d460faebada27..1c5f5d7e02eb7d 100644 --- a/src/tests/async/pgo/pgo.csproj +++ b/src/tests/async/pgo/pgo.csproj @@ -1,4 +1,4 @@ - + true diff --git a/src/tests/async/pinvoke/pinvoke.csproj b/src/tests/async/pinvoke/pinvoke.csproj index 3fc50cde4b3443..197767e2c4e249 100644 --- a/src/tests/async/pinvoke/pinvoke.csproj +++ b/src/tests/async/pinvoke/pinvoke.csproj @@ -1,4 +1,4 @@ - + diff --git a/src/tests/async/regression/125042.csproj b/src/tests/async/regression/125042.csproj index 3fc50cde4b3443..197767e2c4e249 100644 --- a/src/tests/async/regression/125042.csproj +++ b/src/tests/async/regression/125042.csproj @@ -1,4 +1,4 @@ - + diff --git a/src/tests/async/regression/125615.csproj b/src/tests/async/regression/125615.csproj index 3fc50cde4b3443..197767e2c4e249 100644 --- a/src/tests/async/regression/125615.csproj +++ b/src/tests/async/regression/125615.csproj @@ -1,4 +1,4 @@ - + diff --git a/src/tests/async/regression/125805.csproj b/src/tests/async/regression/125805.csproj index 3fc50cde4b3443..197767e2c4e249 100644 --- a/src/tests/async/regression/125805.csproj +++ b/src/tests/async/regression/125805.csproj @@ -1,4 +1,4 @@ - + diff --git a/src/tests/async/regression/128566.csproj b/src/tests/async/regression/128566.csproj index 3fc50cde4b3443..197767e2c4e249 100644 --- a/src/tests/async/regression/128566.csproj +++ b/src/tests/async/regression/128566.csproj @@ -1,4 +1,4 @@ - + diff --git a/src/tests/async/regression/managed-thread-id.csproj b/src/tests/async/regression/managed-thread-id.csproj index 3fc50cde4b3443..197767e2c4e249 100644 --- a/src/tests/async/regression/managed-thread-id.csproj +++ b/src/tests/async/regression/managed-thread-id.csproj @@ -1,4 +1,4 @@ - + diff --git a/src/tests/async/returns/returns.csproj b/src/tests/async/returns/returns.csproj index ff94a80928aef7..7548a677889c52 100644 --- a/src/tests/async/returns/returns.csproj +++ b/src/tests/async/returns/returns.csproj @@ -1,4 +1,4 @@ - + diff --git a/src/tests/async/shared-generic/shared-generic.csproj b/src/tests/async/shared-generic/shared-generic.csproj index 3fc50cde4b3443..197767e2c4e249 100644 --- a/src/tests/async/shared-generic/shared-generic.csproj +++ b/src/tests/async/shared-generic/shared-generic.csproj @@ -1,4 +1,4 @@ - + diff --git a/src/tests/async/staticvirtual/staticvirtual.csproj b/src/tests/async/staticvirtual/staticvirtual.csproj index 3fc50cde4b3443..197767e2c4e249 100644 --- a/src/tests/async/staticvirtual/staticvirtual.csproj +++ b/src/tests/async/staticvirtual/staticvirtual.csproj @@ -1,4 +1,4 @@ - + diff --git a/src/tests/async/struct/struct.csproj b/src/tests/async/struct/struct.csproj index 3fc50cde4b3443..197767e2c4e249 100644 --- a/src/tests/async/struct/struct.csproj +++ b/src/tests/async/struct/struct.csproj @@ -1,4 +1,4 @@ - + diff --git a/src/tests/async/synchronization-context/synchronization-context.csproj b/src/tests/async/synchronization-context/synchronization-context.csproj index 644025b102673e..3a8b32c0ae764d 100644 --- a/src/tests/async/synchronization-context/synchronization-context.csproj +++ b/src/tests/async/synchronization-context/synchronization-context.csproj @@ -1,4 +1,4 @@ - + diff --git a/src/tests/async/valuetask-source/valuetask-source-stub.csproj b/src/tests/async/valuetask-source/valuetask-source-stub.csproj index 3fc50cde4b3443..197767e2c4e249 100644 --- a/src/tests/async/valuetask-source/valuetask-source-stub.csproj +++ b/src/tests/async/valuetask-source/valuetask-source-stub.csproj @@ -1,4 +1,4 @@ - + diff --git a/src/tests/async/valuetask-source/valuetask-source.csproj b/src/tests/async/valuetask-source/valuetask-source.csproj index 7e637235662d05..c2c78be9d50708 100644 --- a/src/tests/async/valuetask-source/valuetask-source.csproj +++ b/src/tests/async/valuetask-source/valuetask-source.csproj @@ -1,4 +1,4 @@ - + true diff --git a/src/tests/async/valuetask/valuetask.csproj b/src/tests/async/valuetask/valuetask.csproj index 3fc50cde4b3443..197767e2c4e249 100644 --- a/src/tests/async/valuetask/valuetask.csproj +++ b/src/tests/async/valuetask/valuetask.csproj @@ -1,4 +1,4 @@ - + diff --git a/src/tests/async/varying-yields/varying-yields-task.csproj b/src/tests/async/varying-yields/varying-yields-task.csproj index a063851b97b392..b80a06405ee432 100644 --- a/src/tests/async/varying-yields/varying-yields-task.csproj +++ b/src/tests/async/varying-yields/varying-yields-task.csproj @@ -1,4 +1,4 @@ - + ASYNC1_TASK;$(DefineConstants) diff --git a/src/tests/async/varying-yields/varying-yields-valuetask.csproj b/src/tests/async/varying-yields/varying-yields-valuetask.csproj index 7679b339a8019b..9e82183ae9f70b 100644 --- a/src/tests/async/varying-yields/varying-yields-valuetask.csproj +++ b/src/tests/async/varying-yields/varying-yields-valuetask.csproj @@ -1,4 +1,4 @@ - + ASYNC1_VALUETASK;$(DefineConstants) diff --git a/src/tests/async/varying-yields/varying-yields.csproj b/src/tests/async/varying-yields/varying-yields.csproj index 3fc50cde4b3443..197767e2c4e249 100644 --- a/src/tests/async/varying-yields/varying-yields.csproj +++ b/src/tests/async/varying-yields/varying-yields.csproj @@ -1,4 +1,4 @@ - + diff --git a/src/tests/async/void/void.csproj b/src/tests/async/void/void.csproj index 3fc50cde4b3443..197767e2c4e249 100644 --- a/src/tests/async/void/void.csproj +++ b/src/tests/async/void/void.csproj @@ -1,4 +1,4 @@ - + diff --git a/src/tests/async/with-yields/with-yields.csproj b/src/tests/async/with-yields/with-yields.csproj index 3fc50cde4b3443..197767e2c4e249 100644 --- a/src/tests/async/with-yields/with-yields.csproj +++ b/src/tests/async/with-yields/with-yields.csproj @@ -1,4 +1,4 @@ - + diff --git a/src/tests/async/without-yields/without-yields.csproj b/src/tests/async/without-yields/without-yields.csproj index 3fc50cde4b3443..197767e2c4e249 100644 --- a/src/tests/async/without-yields/without-yields.csproj +++ b/src/tests/async/without-yields/without-yields.csproj @@ -1,4 +1,4 @@ - + From 9c50f995997572d816b8157a268f65904168f279 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Mon, 27 Jul 2026 15:22:08 +0200 Subject: [PATCH 11/19] Avoid ABI handling of struct pseudo args, and handle FIELD_LIST --- src/coreclr/jit/async.cpp | 54 +++++++++++++++++++++++-------------- src/coreclr/jit/gentree.cpp | 1 - src/coreclr/jit/morph.cpp | 13 +++++++++ 3 files changed, 47 insertions(+), 21 deletions(-) diff --git a/src/coreclr/jit/async.cpp b/src/coreclr/jit/async.cpp index dd5ff687a4f0a2..fe8cbddaaeb6b0 100644 --- a/src/coreclr/jit/async.cpp +++ b/src/coreclr/jit/async.cpp @@ -2370,33 +2370,47 @@ void AsyncTransformation::StoreAsyncAwaiter(BasicBlock* callBlock, assert(layout.ContinuationMemberOffsets[memberIndex] != UINT_MAX); GenTree* awaiter = awaiterArg->GetNode(); - if (!awaiter->OperIs(GT_LCL_VAR)) + assert(varTypeIsStruct(awaiter)); + + if (awaiter->OperIs(GT_FIELD_LIST)) { - LIR::Use use(LIR::AsRange(callBlock), &awaiterArg->NodeRef(), call); - use.ReplaceWithLclVar(m_compiler); - awaiter = use.Def(); - } + GenTreeFieldList* fieldList = awaiter->AsFieldList(); + for (GenTreeFieldList::Use& use : fieldList->Uses()) + { + if (!use.GetNode()->IsInvariant() && !use.GetNode()->OperIs(GT_LCL_VAR)) + { + LIR::Use lirUse(LIR::AsRange(callBlock), &use.NodeRef(), fieldList); + lirUse.ReplaceWithLclVar(m_compiler); + } - LIR::AsRange(callBlock).Remove(awaiter); - call->gtArgs.RemoveUnsafe(awaiterArg); + GenTree* field = use.GetNode(); + LIR::AsRange(callBlock).Remove(field); - GenTree* continuation = m_compiler->gtNewLclvNode(GetNewContinuationVar(), TYP_REF); - unsigned offset = OFFSETOF__CORINFO_Continuation__data + layout.ContinuationMemberOffsets[memberIndex]; - GenTree* offsetNode = m_compiler->gtNewIconNode((ssize_t)offset, TYP_I_IMPL); - GenTree* address = m_compiler->gtNewOperNode(GT_ADD, TYP_BYREF, continuation, offsetNode); - GenTree* store; - if (varTypeIsStruct(awaiter)) - { - store = m_compiler->gtNewStoreValueNode(awaiterLayout, address, awaiter, GTF_IND_NONFAULTING); + GenTree* continuation = m_compiler->gtNewLclvNode(GetNewContinuationVar(), TYP_REF); + unsigned offset = OFFSETOF__CORINFO_Continuation__data + layout.ContinuationMemberOffsets[memberIndex] + use.GetOffset(); + GenTree* store = StoreAtOffset(continuation, offset, field, use.GetType()); + LIR::AsRange(suspendBB).InsertAtEnd(LIR::SeqTree(m_compiler, store)); + } } else { - var_types storeType = - m_compiler->getPrimitiveTypeForStruct(awaiterLayout->GetSize(), awaiterLayout->GetClassHandle()); - assert(storeType != TYP_UNKNOWN); - store = m_compiler->gtNewStoreValueNode(storeType, address, awaiter, GTF_IND_NONFAULTING); + if (!awaiter->OperIs(GT_LCL_VAR)) + { + LIR::Use use(LIR::AsRange(callBlock), &awaiterArg->NodeRef(), call); + use.ReplaceWithLclVar(m_compiler); + awaiter = use.Def(); + } + + GenTree* continuation = m_compiler->gtNewLclvNode(GetNewContinuationVar(), TYP_REF); + unsigned offset = OFFSETOF__CORINFO_Continuation__data + layout.ContinuationMemberOffsets[memberIndex]; + GenTree* offsetNode = m_compiler->gtNewIconNode((ssize_t)offset, TYP_I_IMPL); + GenTree* address = m_compiler->gtNewOperNode(GT_ADD, TYP_BYREF, continuation, offsetNode); + GenTree* store = m_compiler->gtNewStoreValueNode(awaiterLayout, address, awaiter, GTF_IND_NONFAULTING); + LIR::AsRange(suspendBB).InsertAtEnd(LIR::SeqTree(m_compiler, store)); } - LIR::AsRange(suspendBB).InsertAtEnd(LIR::SeqTree(m_compiler, store)); + + LIR::AsRange(callBlock).Remove(awaiter); + call->gtArgs.RemoveUnsafe(awaiterArg); } //------------------------------------------------------------------------ diff --git a/src/coreclr/jit/gentree.cpp b/src/coreclr/jit/gentree.cpp index 543d1d2ade5953..f2ef3f32885b03 100644 --- a/src/coreclr/jit/gentree.cpp +++ b/src/coreclr/jit/gentree.cpp @@ -14017,7 +14017,6 @@ void Compiler::gtDispLeaf(GenTree* tree, IndentStack* indentStack) const ContinuationMember& member = GetContinuationMember(tree->AsVal()->gtVal1); member.Print(); - printf("\n"); break; } diff --git a/src/coreclr/jit/morph.cpp b/src/coreclr/jit/morph.cpp index 39e2c9f0eea43e..b7ef9b9185585f 100644 --- a/src/coreclr/jit/morph.cpp +++ b/src/coreclr/jit/morph.cpp @@ -2214,6 +2214,19 @@ bool Compiler::fgTryMorphStructArg(CallArg* arg) GenTree* argNode = *use; assert(varTypeIsStruct(argNode)); + if (arg->AbiInfo.NumSegments == 0) + { + // Pseudo arg. One case is WellKnownArg::AsyncAwaiter. We just handle + // these as arbitrary struct operands that can be expanded into + // FIELD_LIST. The async transformation will later store the value into + // the continuation, so FIELD_LIST allows using decomposed stores. + if (fgTryReplaceStructLocalWithFields(&arg->NodeRef())) + { + arg->GetNode()->SetMorphed(this, true); + } + return true; + } + bool isSplit = arg->AbiInfo.IsSplitAcrossRegistersAndStack(); #ifdef TARGET_ARM if ((isSplit && (arg->AbiInfo.CountRegsAndStackSlots() > 4)) || (!isSplit && arg->AbiInfo.HasAnyStackSegment())) From 3d0dde9f13e694e3090abca929fb123059a9f9c0 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Mon, 27 Jul 2026 15:28:14 +0200 Subject: [PATCH 12/19] Remove ability to reuse constants --- src/coreclr/jit/async.cpp | 113 ++++++++---------------------- src/coreclr/jit/async.h | 1 - src/coreclr/jit/jitconfigvalues.h | 6 -- 3 files changed, 28 insertions(+), 92 deletions(-) diff --git a/src/coreclr/jit/async.cpp b/src/coreclr/jit/async.cpp index fe8cbddaaeb6b0..95d7efe615d5eb 100644 --- a/src/coreclr/jit/async.cpp +++ b/src/coreclr/jit/async.cpp @@ -808,21 +808,18 @@ PhaseStatus AsyncTransformation::Run() } while (any); } - if (ReuseContinuations()) + // Set up the local containing the continuation we can reuse. For OSR + // things are special: we can transition to the OSR method after having + // resumed in the tier0 method. In that case we end up with the tier0 + // continuation in the OSR method, but we cannot reuse it. + if (m_compiler->opts.IsOSR()) { - // Set up the local containing the continuation we can reuse. For OSR - // things are special: we can transition to the OSR method after having - // resumed in the tier0 method. In that case we end up with the tier0 - // continuation in the OSR method, but we cannot reuse it. - if (m_compiler->opts.IsOSR()) - { - m_reuseContinuationVar = m_compiler->lvaGrabTemp(false DEBUGARG("OSR reusable continuation")); - m_compiler->lvaGetDesc(m_reuseContinuationVar)->lvType = TYP_REF; - } - else - { - m_reuseContinuationVar = m_compiler->lvaAsyncContinuationArg; - } + m_reuseContinuationVar = m_compiler->lvaGrabTemp(false DEBUGARG("OSR reusable continuation")); + m_compiler->lvaGetDesc(m_reuseContinuationVar)->lvType = TYP_REF; + } + else + { + m_reuseContinuationVar = m_compiler->lvaAsyncContinuationArg; } const ContinuationLayout* continuationLayout = CreateResumptionsAndSuspensions(continuationMemberOffsets); @@ -831,6 +828,7 @@ PhaseStatus AsyncTransformation::Run() // create the resumption switch. CreateResumptionSwitch(); + // Now bash all GT_CONTINUATION_MEMBER_OFFSET into appropriate constants. for (GenTree* node : continuationMemberOffsets.BottomUpOrder()) { size_t memberIndex = node->AsVal()->gtVal1; @@ -2216,7 +2214,7 @@ void AsyncTransformation::CreateSuspension(BasicBlock* call LIR::AsRange(suspendBB).InsertAtEnd(storeNewContinuation); SaveSet tailSaveSet = SaveSet::All; - if (ReuseContinuations() && resumeReachable) + if (resumeReachable) { // Split suspendBB into suspendBB -> [reuse continuation with Next store] -> [allocNewBlock with allocation // call] -> suspendBBTail [empty] @@ -3230,15 +3228,12 @@ BasicBlock* AsyncTransformation::RethrowExceptionOnResumption(BasicBlock* GenTree* storeException = m_compiler->gtNewStoreLclVarNode(exceptionLclNum, exceptionInd); LIR::AsRange(resumeBB).InsertAtEnd(LIR::SeqTree(m_compiler, storeException)); - if (ReuseContinuations()) - { - // If we may reuse this continuation later then make sure we don't see the same exception again. - GenTree* continuation = m_compiler->gtNewLclvNode(m_compiler->lvaAsyncContinuationArg, TYP_REF); - unsigned exceptionOffset = OFFSETOF__CORINFO_Continuation__data + layout.ExceptionOffset; - GenTree* null = m_compiler->gtNewNull(); - GenTree* nullException = StoreAtOffset(continuation, exceptionOffset, null, TYP_REF); - LIR::AsRange(resumeBB).InsertAtEnd(LIR::SeqTree(m_compiler, nullException)); - } + // Since we may reuse this continuation later we make sure we don't see the same exception again. + GenTree* continuation = m_compiler->gtNewLclvNode(m_compiler->lvaAsyncContinuationArg, TYP_REF); + unsigned exceptionOffset = OFFSETOF__CORINFO_Continuation__data + layout.ExceptionOffset; + GenTree* null = m_compiler->gtNewNull(); + GenTree* nullException = StoreAtOffset(continuation, exceptionOffset, null, TYP_REF); + LIR::AsRange(resumeBB).InsertAtEnd(LIR::SeqTree(m_compiler, nullException)); GenTree* exception = m_compiler->gtNewLclVarNode(exceptionLclNum, TYP_REF); GenTree* null = m_compiler->gtNewNull(); @@ -3365,10 +3360,7 @@ void AsyncTransformation::CopyReturnValueOnResumption(GenTreeCall* LIR::AsRange(storeResultBB).InsertAtEnd(LIR::SeqTree(m_compiler, storeResult)); } - if (ReuseContinuations()) - { - ClearReturnValueOnResumption(retInfo, resultOffset, storeResultBB); - } + ClearReturnValueOnResumption(retInfo, resultOffset, storeResultBB); } //------------------------------------------------------------------------ @@ -3869,7 +3861,7 @@ void AsyncTransformation::InsertFinishContextHandlingCall(BasicBlock* const ContinuationLayout* AsyncTransformation::CreateResumptionsAndSuspensions( ArrayStack& continuationMemberOffsets) { - bool useSharedLayout = (m_states.size() > 1) && ReuseContinuations(); + bool useSharedLayout = m_states.size() > 1; ContinuationLayout* sharedLayout = nullptr; if (useSharedLayout) @@ -3939,36 +3931,14 @@ const ContinuationLayout* AsyncTransformation::CreateResumptionsAndSuspensions( } } - const ContinuationLayout* firstLayout = sharedLayout; - const ContinuationLayout* continuationMemberLayout = sharedLayout; + ContinuationLayout* layout = nullptr; JITDUMP("Creating suspensions and resumptions for %zu states\n", m_states.size()); for (const AsyncState& state : m_states) { JITDUMP("State %u suspend @ " FMT_BB ", resume @ " FMT_BB "\n", state.Number, state.SuspensionBB->bbNum, state.ResumptionBB->bbNum); - ContinuationLayout* layout = - sharedLayout == nullptr ? state.Layout->Create(continuationMemberOffsets) : sharedLayout; - if (firstLayout == nullptr) - { - firstLayout = layout; - } + layout = sharedLayout == nullptr ? state.Layout->Create(continuationMemberOffsets) : sharedLayout; - bool hasContinuationMember = state.Call->gtArgs.FindWellKnownArg(WellKnownArg::AsyncAwaiter) != nullptr; - if (hasContinuationMember && (continuationMemberLayout == nullptr)) - { - continuationMemberLayout = layout; - } -#ifdef DEBUG - else if (hasContinuationMember) - { - assert(continuationMemberLayout->ContinuationMemberOffsets.size() == - layout->ContinuationMemberOffsets.size()); - for (size_t i = 0; i < continuationMemberLayout->ContinuationMemberOffsets.size(); i++) - { - assert(continuationMemberLayout->ContinuationMemberOffsets[i] == layout->ContinuationMemberOffsets[i]); - } - } -#endif CreateSuspension(state.CallBlock, state.Call, state.SuspensionBB, state.Number, *layout, *state.Layout, state.ResumeReachable, state.MutatedSincePreviousResumption); CreateResumption(state.CallBlock, state.Call, state.ResumptionBB, state.CallDefInfo, *layout, *state.Layout); @@ -3977,31 +3947,7 @@ const ContinuationLayout* AsyncTransformation::CreateResumptionsAndSuspensions( JITDUMP("\n"); } - assert(firstLayout != nullptr); - assert((continuationMemberOffsets.Height() == 0) || (continuationMemberLayout != nullptr)); - return continuationMemberLayout != nullptr ? continuationMemberLayout : firstLayout; -} - -//------------------------------------------------------------------------ -// AsyncTransformation::ReuseContinuations: -// Returns true if continuation reuse is enabled. -// -// Returns: -// True if so. -// -bool AsyncTransformation::ReuseContinuations() -{ -#ifdef DEBUG - static ConfigMethodRange s_range; - s_range.EnsureInit(JitConfig.JitAsyncReuseContinuationsRange()); - - if (!s_range.Contains(m_compiler->info.compMethodHash())) - { - return false; - } -#endif - - return JitConfig.JitAsyncReuseContinuations() != 0; + return layout; } //------------------------------------------------------------------------ @@ -4250,12 +4196,9 @@ void AsyncTransformation::CreateResumptionSwitch() GenTree* jtrue = m_compiler->gtNewOperNode(GT_JTRUE, TYP_VOID, eqZero); LIR::AsRange(checkOSRAddressOffsetBB).InsertAtEnd(LIR::SeqTree(m_compiler, jtrue)); - if (ReuseContinuations()) - { - // Also, save the fact that we have a reusable continuation - continuationArg = m_compiler->gtNewLclvNode(m_compiler->lvaAsyncContinuationArg, TYP_REF); - GenTree* storeReusable = m_compiler->gtNewStoreLclVarNode(m_reuseContinuationVar, continuationArg); - LIR::AsRange(onContinuationBB).InsertAtBeginning(continuationArg, storeReusable); - } + // Also, save the fact that we have a reusable continuation + continuationArg = m_compiler->gtNewLclvNode(m_compiler->lvaAsyncContinuationArg, TYP_REF); + GenTree* storeReusable = m_compiler->gtNewStoreLclVarNode(m_reuseContinuationVar, continuationArg); + LIR::AsRange(onContinuationBB).InsertAtBeginning(continuationArg, storeReusable); } } diff --git a/src/coreclr/jit/async.h b/src/coreclr/jit/async.h index aabb7d2bb5f325..97c1a96ed3b2be 100644 --- a/src/coreclr/jit/async.h +++ b/src/coreclr/jit/async.h @@ -530,7 +530,6 @@ class AsyncTransformation SuspensionContextHelper helper, GenTree* execContext, GenTree* syncContext); - bool ReuseContinuations(); const ContinuationLayout* CreateResumptionsAndSuspensions(ArrayStack& continuationMemberOffsets); void CreateResumptionSwitch(); diff --git a/src/coreclr/jit/jitconfigvalues.h b/src/coreclr/jit/jitconfigvalues.h index 5e3095e94dd87c..bfb21eaf40c916 100644 --- a/src/coreclr/jit/jitconfigvalues.h +++ b/src/coreclr/jit/jitconfigvalues.h @@ -620,12 +620,6 @@ OPT_CONFIG_STRING(JitAsyncDefaultValueAnalysisRange, // a continuation is being reused. OPT_CONFIG_STRING(JitAsyncPreservedValueAnalysisRange, "JitAsyncPreservedValueAnalysisRange") -// Enable continuation reuse based on method hash range -OPT_CONFIG_STRING(JitAsyncReuseContinuationsRange, "JitAsyncReuseContinuationsRange") -// Save and reuse continuation instances in runtime async functions. Also -// implies use of shared continuation layouts for all suspension points. -RELEASE_CONFIG_INTEGER(JitAsyncReuseContinuations, "JitAsyncReuseContinuations", 1) - RELEASE_CONFIG_INTEGER(JitEnableOptRepeat, "JitEnableOptRepeat", 1) // If zero, do not allow JitOptRepeat RELEASE_CONFIG_METHODSET(JitOptRepeat, "JitOptRepeat") // Runs optimizer multiple times on specified methods RELEASE_CONFIG_INTEGER(JitOptRepeatCount, "JitOptRepeatCount", 2) // Number of times to repeat opts when repeating From 5eb478198cfbcae0f03504f7c8c09b8f4afba215 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Mon, 27 Jul 2026 15:41:16 +0200 Subject: [PATCH 13/19] Run jit-format --- src/coreclr/jit/async.cpp | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/coreclr/jit/async.cpp b/src/coreclr/jit/async.cpp index 95d7efe615d5eb..8b9de8fffdc229 100644 --- a/src/coreclr/jit/async.cpp +++ b/src/coreclr/jit/async.cpp @@ -2385,8 +2385,9 @@ void AsyncTransformation::StoreAsyncAwaiter(BasicBlock* callBlock, LIR::AsRange(callBlock).Remove(field); GenTree* continuation = m_compiler->gtNewLclvNode(GetNewContinuationVar(), TYP_REF); - unsigned offset = OFFSETOF__CORINFO_Continuation__data + layout.ContinuationMemberOffsets[memberIndex] + use.GetOffset(); - GenTree* store = StoreAtOffset(continuation, offset, field, use.GetType()); + unsigned offset = + OFFSETOF__CORINFO_Continuation__data + layout.ContinuationMemberOffsets[memberIndex] + use.GetOffset(); + GenTree* store = StoreAtOffset(continuation, offset, field, use.GetType()); LIR::AsRange(suspendBB).InsertAtEnd(LIR::SeqTree(m_compiler, store)); } } @@ -3229,14 +3230,14 @@ BasicBlock* AsyncTransformation::RethrowExceptionOnResumption(BasicBlock* LIR::AsRange(resumeBB).InsertAtEnd(LIR::SeqTree(m_compiler, storeException)); // Since we may reuse this continuation later we make sure we don't see the same exception again. - GenTree* continuation = m_compiler->gtNewLclvNode(m_compiler->lvaAsyncContinuationArg, TYP_REF); - unsigned exceptionOffset = OFFSETOF__CORINFO_Continuation__data + layout.ExceptionOffset; - GenTree* null = m_compiler->gtNewNull(); - GenTree* nullException = StoreAtOffset(continuation, exceptionOffset, null, TYP_REF); + continuation = m_compiler->gtNewLclvNode(m_compiler->lvaAsyncContinuationArg, TYP_REF); + exceptionOffset = OFFSETOF__CORINFO_Continuation__data + layout.ExceptionOffset; + GenTree* null = m_compiler->gtNewNull(); + GenTree* nullException = StoreAtOffset(continuation, exceptionOffset, null, TYP_REF); LIR::AsRange(resumeBB).InsertAtEnd(LIR::SeqTree(m_compiler, nullException)); GenTree* exception = m_compiler->gtNewLclVarNode(exceptionLclNum, TYP_REF); - GenTree* null = m_compiler->gtNewNull(); + null = m_compiler->gtNewNull(); GenTree* neNull = m_compiler->gtNewOperNode(GT_NE, TYP_INT, exception, null); GenTree* jtrue = m_compiler->gtNewOperNode(GT_JTRUE, TYP_VOID, neNull); LIR::AsRange(resumeBB).InsertAtEnd(exception, null, neNull, jtrue); From 546c9a2ce71a4cdb4bb46476b178869cc18e3eec Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Mon, 27 Jul 2026 18:32:40 +0200 Subject: [PATCH 14/19] Add comments --- src/coreclr/jit/async.cpp | 80 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 79 insertions(+), 1 deletion(-) diff --git a/src/coreclr/jit/async.cpp b/src/coreclr/jit/async.cpp index 8b9de8fffdc229..7e6095995110eb 100644 --- a/src/coreclr/jit/async.cpp +++ b/src/coreclr/jit/async.cpp @@ -45,6 +45,17 @@ #include "jitstd/algorithm.h" #include "async.h" +//------------------------------------------------------------------------ +// ContinuationMember::CustomAwaiterOfLayout: +// Create a continuation member that stores a custom awaiter with the +// specified layout. +// +// Parameters: +// layout - Layout of the custom awaiter. +// +// Returns: +// A continuation member describing the custom awaiter. +// ContinuationMember ContinuationMember::CustomAwaiterOfLayout(ClassLayout* layout) { ContinuationMember member; @@ -53,12 +64,30 @@ ContinuationMember ContinuationMember::CustomAwaiterOfLayout(ClassLayout* layout return member; } +//------------------------------------------------------------------------ +// ContinuationMember::GetCustomAwaiterLayout: +// Get the layout of a custom awaiter continuation member. +// +// Returns: +// The custom awaiter's layout. +// ClassLayout* ContinuationMember::GetCustomAwaiterLayout() const { assert(Type == ContinuationMemberType::CustomAwaiterOfLayout); return m_customAwaiterLayout; } +//------------------------------------------------------------------------ +// ContinuationMember::AreCompatible: +// Check whether two continuation members can share the same storage. +// +// Parameters: +// a - First continuation member. +// b - Second continuation member. +// +// Returns: +// True if the members are compatible; otherwise false. +// bool ContinuationMember::AreCompatible(const ContinuationMember& a, const ContinuationMember& b) { if (a.Type != b.Type) @@ -76,6 +105,10 @@ bool ContinuationMember::AreCompatible(const ContinuationMember& a, const Contin } #ifdef DEBUG +//------------------------------------------------------------------------ +// ContinuationMember::Print: +// Print a description of this continuation member. +// void ContinuationMember::Print() const { switch (Type) @@ -89,6 +122,16 @@ void ContinuationMember::Print() const } #endif +//------------------------------------------------------------------------ +// Compiler::GetContinuationMemberIndex: +// Find or add a continuation member and return its index. +// +// Parameters: +// member - Continuation member to find or add. +// +// Returns: +// The member's index in m_asyncContinuationMembers. +// size_t Compiler::GetContinuationMemberIndex(const ContinuationMember& member) { if (m_asyncContinuationMembers == nullptr) @@ -112,11 +155,28 @@ size_t Compiler::GetContinuationMemberIndex(const ContinuationMember& member) return m_asyncContinuationMembers->size() - 1; } +//------------------------------------------------------------------------ +// Compiler::GetContinuationMemberCount: +// Get the number of continuation members registered by the compiler. +// +// Returns: +// The number of registered continuation members. +// size_t Compiler::GetContinuationMemberCount() const { return m_asyncContinuationMembers == nullptr ? 0 : m_asyncContinuationMembers->size(); } +//------------------------------------------------------------------------ +// Compiler::GetContinuationMember: +// Get a registered continuation member by index. +// +// Parameters: +// index - Index of the continuation member. +// +// Returns: +// The continuation member at the specified index. +// const ContinuationMember& Compiler::GetContinuationMember(size_t index) { assert(index < m_asyncContinuationMembers->size()); @@ -640,7 +700,8 @@ bool ContinuationLayoutBuilder::Equals(const ContinuationLayoutBuilder& a, const } //------------------------------------------------------------------------ -// TransformAsync: Run async transformation. +// Compiler::TransformAsync: +// Run the async transformation. // // Returns: // Suitable phase status. @@ -1516,6 +1577,10 @@ const ReturnInfo* ContinuationLayout::FindReturn(Compiler* comp, GenTreeCall* ca // Finalize the layout by computing offsets for all fields, locals, and // return values. Allocates the continuation type from the VM. // +// Parameters: +// continuationMemberOffsets - Symbolic continuation member offset nodes +// that remain in the method. +// // Returns: // The finalized ContinuationLayout with computed offsets and a class // handle for the continuation type. @@ -2349,6 +2414,12 @@ void AsyncTransformation::CreateSuspension(BasicBlock* call // AsyncTransformation::StoreAsyncAwaiter: // Move the pseudo awaiter argument into its reserved continuation member. // +// Parameters: +// callBlock - Block containing the async awaiter call. +// call - Async awaiter call. +// suspendBB - Suspension block in which to store the awaiter. +// layout - Layout of the continuation. +// void AsyncTransformation::StoreAsyncAwaiter(BasicBlock* callBlock, GenTreeCall* call, BasicBlock* suspendBB, @@ -3859,6 +3930,13 @@ void AsyncTransformation::InsertFinishContextHandlingCall(BasicBlock* // Walk all recorded async states and create the suspension and resumption // IR, continuation layouts, and debug info for each one. // +// Parameters: +// continuationMemberOffsets - Symbolic continuation member offset nodes +// that remain in the method. +// +// Returns: +// The continuation layout containing the continuation member offsets. +// const ContinuationLayout* AsyncTransformation::CreateResumptionsAndSuspensions( ArrayStack& continuationMemberOffsets) { From aff2ffaa915a48366855ff7964d3dd41bacb11fa Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Mon, 27 Jul 2026 20:25:36 +0200 Subject: [PATCH 15/19] WIP --- src/coreclr/jit/importercalls.cpp | 24 +++++++++++++++++-- .../tools/Common/JitInterface/CorInfoImpl.cs | 11 +++++++-- .../ReadyToRun/TypeValidationChecker.cs | 1 + .../CompilerServices/YieldAwaitable.cs | 1 + 4 files changed, 33 insertions(+), 4 deletions(-) diff --git a/src/coreclr/jit/importercalls.cpp b/src/coreclr/jit/importercalls.cpp index e251dcd06f8891..fb1b7dbe248174 100644 --- a/src/coreclr/jit/importercalls.cpp +++ b/src/coreclr/jit/importercalls.cpp @@ -148,6 +148,22 @@ void Compiler::impTryOptimizeAwaitAwaiter(GenTreeCall* call, return; } + if (info.compCompHnd->isIntrinsicType(awaiterArg->GetSignatureClassHandle())) + { + const char* namespaceName; + const char* className = info.compCompHnd->getClassNameFromMetadata(awaiterArg->GetSignatureClassHandle(), &namespaceName); + if ((strcmp(className, "YieldAwaiter") == 0) && (strcmp(namespaceName, "System.Runtime.CompilerServices") == 0)) + { + // YieldAwaiter is specially recognized by + // AsyncHelpers.UnsafeAwaitAwaiter and accomplishes more than just + // avoiding a box. + JITDUMP("Skipping custom awaiter optimization for YieldAwaiter\n"); + return; + } + } + + JITDUMP("Optimizing awaiter call [%06u] to read its struct awaiter from the continuation\n", dspTreeID(call)); + CORINFO_LOOKUP newInstArgLookup; CORINFO_SIG_INFO lookupSig = *sig; if (pResolvedToken->pMethodSpec != nullptr) @@ -163,6 +179,12 @@ void Compiler::impTryOptimizeAwaitAwaiter(GenTreeCall* call, info.compCompHnd->getAwaitAwaiterInContinuationCall(info.compMethodHnd, &lookupSig, isUnsafe, &newExactContextHnd, &newInstArgLookup); + if (newMethod == NO_METHOD_HANDLE) + { + JITDUMP("EE returned no method to call; bailing on optimization\n"); + return; + } + CORINFO_SIG_INFO newSig; info.compCompHnd->getMethodSig(newMethod, &newSig); @@ -178,8 +200,6 @@ void Compiler::impTryOptimizeAwaitAwaiter(GenTreeCall* call, } } - JITDUMP("Optimizing awaiter call [%06u] to read its struct awaiter from the continuation\n", dspTreeID(call)); - *methHnd = newMethod; *exactContextHnd = newExactContextHnd; call->gtCallMethHnd = newMethod; diff --git a/src/coreclr/tools/Common/JitInterface/CorInfoImpl.cs b/src/coreclr/tools/Common/JitInterface/CorInfoImpl.cs index 41f1e9b61a587b..4e01b43dd5d529 100644 --- a/src/coreclr/tools/Common/JitInterface/CorInfoImpl.cs +++ b/src/coreclr/tools/Common/JitInterface/CorInfoImpl.cs @@ -3683,8 +3683,8 @@ private void getWasmWellKnownGlobals(ref CORINFO_WASM_WELLKNOWN_GLOBALS pWellKno if (runtimeDeterminedResult.IsRuntimeDeterminedExactMethod) { // TODO-Async: the instantiation argument would have to be obtained through a runtime - // generic dictionary lookup, which is not yet emitted here, so defer to the runtime JIT. - throw new RequiresRuntimeJitException($"getAwaitReturnCall: runtime-determined exact instantiation requires runtime JIT ({runtimeDeterminedResult})"); + // generic dictionary lookup, which is not yet emitted here, so skip the optimization. + return null; } instArg.constLookup = CreateConstLookupToSymbol( @@ -3739,6 +3739,13 @@ private void getWasmWellKnownGlobals(ref CORINFO_WASM_WELLKNOWN_GLOBALS pWellKno if (targetMethod.RequiresInstArg()) { #if READYTORUN + if (result.IsRuntimeDeterminedExactMethod) + { + // TODO-Async: the instantiation argument would have to be obtained through a runtime + // generic dictionary lookup, which is not yet emitted here, so defer to the runtime JIT. + throw new RequiresRuntimeJitException($"getAwaitAwaiterInContinuationCall: runtime-determined exact instantiation requires runtime JIT ({result})"); + } + instArg.constLookup = CreateConstLookupToSymbol( _compilation.SymbolNodeFactory.CreateReadyToRunHelper( ReadyToRunHelperId.MethodDictionary, diff --git a/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/DependencyAnalysis/ReadyToRun/TypeValidationChecker.cs b/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/DependencyAnalysis/ReadyToRun/TypeValidationChecker.cs index 0962855a091fee..6671793efd9150 100644 --- a/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/DependencyAnalysis/ReadyToRun/TypeValidationChecker.cs +++ b/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/DependencyAnalysis/ReadyToRun/TypeValidationChecker.cs @@ -73,6 +73,7 @@ private async Task CanSkipValidationInstance(EcmaModule module) // Spot check that the system module ALWAYS succeeds if (failAtEnd) { + LogErrors(s => Console.WriteLine(s)); throw new InternalCompilerErrorException("System module failed to validate all types"); } } diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/YieldAwaitable.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/YieldAwaitable.cs index 0ccdb4412bf48f..05f694569bba3f 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/YieldAwaitable.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/YieldAwaitable.cs @@ -43,6 +43,7 @@ public readonly struct YieldAwaitable /// Provides an awaiter that switches into a target environment. /// This type is intended for compiler use only. + [Intrinsic] // Recognized by runtime async public readonly struct YieldAwaiter : ICriticalNotifyCompletion, IStateMachineBoxAwareAwaiter { /// Gets whether a yield is not required. From 864edb3877e8242c54623ce1129b68db9bb631cc Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Tue, 28 Jul 2026 10:51:58 +0200 Subject: [PATCH 16/19] Fix --- src/coreclr/jit/async.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/coreclr/jit/async.cpp b/src/coreclr/jit/async.cpp index 7e6095995110eb..d7ec6e048c050c 100644 --- a/src/coreclr/jit/async.cpp +++ b/src/coreclr/jit/async.cpp @@ -2461,6 +2461,8 @@ void AsyncTransformation::StoreAsyncAwaiter(BasicBlock* callBlock, GenTree* store = StoreAtOffset(continuation, offset, field, use.GetType()); LIR::AsRange(suspendBB).InsertAtEnd(LIR::SeqTree(m_compiler, store)); } + + LIR::AsRange(callBlock).Remove(fieldList); } else { @@ -2471,6 +2473,8 @@ void AsyncTransformation::StoreAsyncAwaiter(BasicBlock* callBlock, awaiter = use.Def(); } + LIR::AsRange(callBlock).Remove(awaiter); + GenTree* continuation = m_compiler->gtNewLclvNode(GetNewContinuationVar(), TYP_REF); unsigned offset = OFFSETOF__CORINFO_Continuation__data + layout.ContinuationMemberOffsets[memberIndex]; GenTree* offsetNode = m_compiler->gtNewIconNode((ssize_t)offset, TYP_I_IMPL); @@ -2479,7 +2483,6 @@ void AsyncTransformation::StoreAsyncAwaiter(BasicBlock* callBlock, LIR::AsRange(suspendBB).InsertAtEnd(LIR::SeqTree(m_compiler, store)); } - LIR::AsRange(callBlock).Remove(awaiter); call->gtArgs.RemoveUnsafe(awaiterArg); } From 26a253a222c57c095c01d87a77fcc7a9f0cafd1b Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Tue, 28 Jul 2026 12:48:24 +0200 Subject: [PATCH 17/19] Support struct awaiter continuation optimization under NativeAOT Change getAwaitAwaiterInContinuationCall to take the CORINFO_RESOLVED_TOKEN of the call instead of its CORINFO_SIG_INFO. Under NativeAOT the signature instantiation is already canonical, so the runtime determined form needed for the generic dictionary lookup could not be recovered from it. The VM now reads both the awaiter type and the MethodSpec blob off the resolved token, which also removes the need for the JIT to stuff the MethodSpec into a copy of the signature. Also: - Recompute the call entry point after switching the target method. It was computed for the original AwaitAwaiter call, so AOT emitted a call to the wrong symbol and never compiled the InContinuation instantiations. - Report the dependencies for the rewrite from the IL scanner. Scanning does not see these calls otherwise, so the generic dictionary slot was missing from the precomputed dictionary layout. - Do not check the namespace when recognizing YieldAwaiter. It is a nested type and some hosts report an empty namespace for those, which meant the opt-out never fired under NativeAOT. - Add shared generic awaiters to the custom-struct-awaiters test. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 757a9e4c-44f3-4255-86f6-065ebb823ed8 --- src/coreclr/inc/corinfo.h | 15 +++- src/coreclr/inc/icorjitinfoimpl_generated.h | 2 +- src/coreclr/inc/jiteeversionguid.h | 10 +-- .../jit/ICorJitInfo_wrapper_generated.hpp | 4 +- src/coreclr/jit/compiler.h | 1 - src/coreclr/jit/importercalls.cpp | 47 ++++++---- .../tools/Common/JitInterface/CorInfoImpl.cs | 41 +++++---- .../JitInterface/CorInfoImpl_generated.cs | 6 +- .../ThunkGenerator/ThunkInput.txt | 2 +- .../IL/ILImporter.Scanner.cs | 83 ++++++++++++++++++ .../aot/jitinterface/jitinterface_generated.h | 6 +- .../tools/superpmi/superpmi-shared/agnostic.h | 14 +-- .../superpmi-shared/methodcontext.cpp | 17 ++-- .../superpmi/superpmi-shared/methodcontext.h | 4 +- .../superpmi-shim-collector/icorjitinfo.cpp | 6 +- .../icorjitinfo_generated.cpp | 4 +- .../icorjitinfo_generated.cpp | 4 +- .../tools/superpmi/superpmi/icorjitinfo.cpp | 4 +- src/coreclr/vm/jitinterface.cpp | 20 +++-- src/coreclr/vm/jitinterface.h | 2 +- .../custom-struct-awaiters.cs | 87 +++++++++++++++++++ 21 files changed, 293 insertions(+), 86 deletions(-) diff --git a/src/coreclr/inc/corinfo.h b/src/coreclr/inc/corinfo.h index f4f63bc3e41739..f85cf0acb238a0 100644 --- a/src/coreclr/inc/corinfo.h +++ b/src/coreclr/inc/corinfo.h @@ -3164,9 +3164,22 @@ class ICorStaticInfo // instantiation argument that must be passed to the await call. virtual CORINFO_METHOD_HANDLE getAwaitReturnCall(CORINFO_METHOD_HANDLE callerHandle, CORINFO_CONTEXT_HANDLE* contextHandle, CORINFO_LOOKUP* instArg) = 0; + // Get the method to use to await a struct awaiter that is stored inside the + // continuation instead of being boxed. 'pResolvedToken' is the resolved + // token of the AsyncHelpers.AwaitAwaiter/UnsafeAwaitAwaiter call site that + // is being replaced, and 'isUnsafe' indicates whether the unsafe variant is + // being replaced. + // + // Returns the method handle of the call to insert, or NULL if the + // transformation cannot be performed. 'contextHandle' is set to the context + // to use when inlining the call, exactly as getCallInfo would report it for + // a direct call to it (it may be an approximate/shared instantiation when + // 'instArg' requires a runtime lookup). 'instArg' is filled with the + // (potentially runtime-looked-up) instantiation argument that must be + // passed to the call. virtual CORINFO_METHOD_HANDLE getAwaitAwaiterInContinuationCall( CORINFO_METHOD_HANDLE callerHandle, - CORINFO_SIG_INFO* callSig, + CORINFO_RESOLVED_TOKEN* pResolvedToken, bool isUnsafe, CORINFO_CONTEXT_HANDLE* contextHandle, CORINFO_LOOKUP* instArg diff --git a/src/coreclr/inc/icorjitinfoimpl_generated.h b/src/coreclr/inc/icorjitinfoimpl_generated.h index aae0a091f92096..5a7fb30fe7e14d 100644 --- a/src/coreclr/inc/icorjitinfoimpl_generated.h +++ b/src/coreclr/inc/icorjitinfoimpl_generated.h @@ -508,7 +508,7 @@ CORINFO_METHOD_HANDLE getAwaitReturnCall( CORINFO_METHOD_HANDLE getAwaitAwaiterInContinuationCall( CORINFO_METHOD_HANDLE callerHandle, - CORINFO_SIG_INFO* callSig, + CORINFO_RESOLVED_TOKEN* pResolvedToken, bool isUnsafe, CORINFO_CONTEXT_HANDLE* contextHandle, CORINFO_LOOKUP* instArg) override; diff --git a/src/coreclr/inc/jiteeversionguid.h b/src/coreclr/inc/jiteeversionguid.h index 48bf2b0cc65582..578c88b80b5252 100644 --- a/src/coreclr/inc/jiteeversionguid.h +++ b/src/coreclr/inc/jiteeversionguid.h @@ -37,11 +37,11 @@ #include -constexpr GUID JITEEVersionIdentifier = { /* 10b303cf-6607-4e80-8861-c7577259a06a */ - 0x10b303cf, - 0x6607, - 0x4e80, - {0x88, 0x61, 0xc7, 0x57, 0x72, 0x59, 0xa0, 0x6a} +constexpr GUID JITEEVersionIdentifier = { /* 305cdd16-2cee-49af-ae7c-2537eb37cae9 */ + 0x305cdd16, + 0x2cee, + 0x49af, + {0xae, 0x7c, 0x25, 0x37, 0xeb, 0x37, 0xca, 0xe9} }; #endif // JIT_EE_VERSIONING_GUID_H diff --git a/src/coreclr/jit/ICorJitInfo_wrapper_generated.hpp b/src/coreclr/jit/ICorJitInfo_wrapper_generated.hpp index 8c08675dc5e941..89e8e05606cfbb 100644 --- a/src/coreclr/jit/ICorJitInfo_wrapper_generated.hpp +++ b/src/coreclr/jit/ICorJitInfo_wrapper_generated.hpp @@ -1201,13 +1201,13 @@ CORINFO_METHOD_HANDLE WrapICorJitInfo::getAwaitReturnCall( CORINFO_METHOD_HANDLE WrapICorJitInfo::getAwaitAwaiterInContinuationCall( CORINFO_METHOD_HANDLE callerHandle, - CORINFO_SIG_INFO* callSig, + CORINFO_RESOLVED_TOKEN* pResolvedToken, bool isUnsafe, CORINFO_CONTEXT_HANDLE* contextHandle, CORINFO_LOOKUP* instArg) { API_ENTER(getAwaitAwaiterInContinuationCall); - CORINFO_METHOD_HANDLE temp = wrapHnd->getAwaitAwaiterInContinuationCall(callerHandle, callSig, isUnsafe, contextHandle, instArg); + CORINFO_METHOD_HANDLE temp = wrapHnd->getAwaitAwaiterInContinuationCall(callerHandle, pResolvedToken, isUnsafe, contextHandle, instArg); API_LEAVE(getAwaitAwaiterInContinuationCall); return temp; } diff --git a/src/coreclr/jit/compiler.h b/src/coreclr/jit/compiler.h index a3c73ed2b84fcc..49f10506bce79c 100644 --- a/src/coreclr/jit/compiler.h +++ b/src/coreclr/jit/compiler.h @@ -5250,7 +5250,6 @@ class Compiler void impTryOptimizeAwaitAwaiter(GenTreeCall* call, CORINFO_RESOLVED_TOKEN* pResolvedToken, - CORINFO_SIG_INFO* sig, CORINFO_CALL_INFO* callInfo, CORINFO_METHOD_HANDLE* methHnd, CORINFO_CONTEXT_HANDLE* exactContextHnd, diff --git a/src/coreclr/jit/importercalls.cpp b/src/coreclr/jit/importercalls.cpp index fb1b7dbe248174..60860489e90171 100644 --- a/src/coreclr/jit/importercalls.cpp +++ b/src/coreclr/jit/importercalls.cpp @@ -133,9 +133,18 @@ GenTree* Compiler::impGetInstParamArg(CORINFO_RESOLVED_TOKEN* pResolvedToken, // Rewrite a struct AwaitAwaiter or UnsafeAwaitAwaiter call to use the // corresponding helper that reads the awaiter from the continuation. // +// Arguments: +// call - The call to the awaiter helper +// pResolvedToken - Resolved token of the call +// callInfo - EE supplied info for the call; updated on success +// methHnd - [in, out] Method handle of the call; updated on success +// exactContextHnd - [in, out] Exact context of the call; updated on success +// instParam - [in, out] Instantiation argument of the call; updated on success +// ni - Named intrinsic of the call, used to determine whether +// this is the unsafe variant +// void Compiler::impTryOptimizeAwaitAwaiter(GenTreeCall* call, CORINFO_RESOLVED_TOKEN* pResolvedToken, - CORINFO_SIG_INFO* sig, CORINFO_CALL_INFO* callInfo, CORINFO_METHOD_HANDLE* methHnd, CORINFO_CONTEXT_HANDLE* exactContextHnd, @@ -150,9 +159,12 @@ void Compiler::impTryOptimizeAwaitAwaiter(GenTreeCall* call, if (info.compCompHnd->isIntrinsicType(awaiterArg->GetSignatureClassHandle())) { - const char* namespaceName; - const char* className = info.compCompHnd->getClassNameFromMetadata(awaiterArg->GetSignatureClassHandle(), &namespaceName); - if ((strcmp(className, "YieldAwaiter") == 0) && (strcmp(namespaceName, "System.Runtime.CompilerServices") == 0)) + // Note: no namespace check here. YieldAwaiter is a nested type, and + // some hosts report an empty namespace for those. Being an intrinsic + // type is enough to know this is the well known type from CoreLib. + const char* className = + info.compCompHnd->getClassNameFromMetadata(awaiterArg->GetSignatureClassHandle(), nullptr); + if (strcmp(className, "YieldAwaiter") == 0) { // YieldAwaiter is specially recognized by // AsyncHelpers.UnsafeAwaitAwaiter and accomplishes more than just @@ -164,19 +176,11 @@ void Compiler::impTryOptimizeAwaitAwaiter(GenTreeCall* call, JITDUMP("Optimizing awaiter call [%06u] to read its struct awaiter from the continuation\n", dspTreeID(call)); - CORINFO_LOOKUP newInstArgLookup; - CORINFO_SIG_INFO lookupSig = *sig; - if (pResolvedToken->pMethodSpec != nullptr) - { - lookupSig.pSig = pResolvedToken->pMethodSpec; - lookupSig.cbSig = pResolvedToken->cbMethodSpec; - lookupSig.scope = pResolvedToken->tokenScope; - } - + CORINFO_LOOKUP newInstArgLookup; bool isUnsafe = ni == NI_System_Runtime_CompilerServices_AsyncHelpers_UnsafeAwaitAwaiter; CORINFO_CONTEXT_HANDLE newExactContextHnd; CORINFO_METHOD_HANDLE newMethod = - info.compCompHnd->getAwaitAwaiterInContinuationCall(info.compMethodHnd, &lookupSig, isUnsafe, + info.compCompHnd->getAwaitAwaiterInContinuationCall(info.compMethodHnd, pResolvedToken, isUnsafe, &newExactContextHnd, &newInstArgLookup); if (newMethod == NO_METHOD_HANDLE) @@ -200,6 +204,17 @@ void Compiler::impTryOptimizeAwaitAwaiter(GenTreeCall* call, } } +#ifdef FEATURE_READYTORUN + if (IsAot()) + { + // The entry point was computed for the original method, so recompute it + // for the replacement. + CORINFO_CONST_LOOKUP newEntryPoint; + info.compCompHnd->getFunctionEntryPoint(newMethod, &newEntryPoint); + call->setEntryPoint(newEntryPoint); + } +#endif + *methHnd = newMethod; *exactContextHnd = newExactContextHnd; call->gtCallMethHnd = newMethod; @@ -1049,8 +1064,8 @@ var_types Compiler::impImportCall(OPCODE opcode, if (opts.OptimizationEnabled() && ((ni == NI_System_Runtime_CompilerServices_AsyncHelpers_AwaitAwaiter) || (ni == NI_System_Runtime_CompilerServices_AsyncHelpers_UnsafeAwaitAwaiter))) { - impTryOptimizeAwaitAwaiter(call->AsCall(), pResolvedToken, sig, callInfo, &methHnd, &exactContextHnd, - &instParam, ni); + impTryOptimizeAwaitAwaiter(call->AsCall(), pResolvedToken, callInfo, &methHnd, &exactContextHnd, &instParam, + ni); } // Extra args diff --git a/src/coreclr/tools/Common/JitInterface/CorInfoImpl.cs b/src/coreclr/tools/Common/JitInterface/CorInfoImpl.cs index 4e01b43dd5d529..86f75f4472837e 100644 --- a/src/coreclr/tools/Common/JitInterface/CorInfoImpl.cs +++ b/src/coreclr/tools/Common/JitInterface/CorInfoImpl.cs @@ -3707,19 +3707,25 @@ private void getWasmWellKnownGlobals(ref CORINFO_WASM_WELLKNOWN_GLOBALS pWellKno private CORINFO_METHOD_STRUCT_* getAwaitAwaiterInContinuationCall( CORINFO_METHOD_STRUCT_* callerHandle, - CORINFO_SIG_INFO* callSig, + ref CORINFO_RESOLVED_TOKEN pResolvedToken, bool isUnsafe, CORINFO_CONTEXT_STRUCT** contextHandle, ref CORINFO_LOOKUP instArg) { - Debug.Assert(callSig->sigInst.methInstCount == 1); - instArg.lookupKind.needsRuntimeLookup = false; instArg.constLookup.accessType = InfoAccessType.IAT_VALUE; instArg.constLookup.addr = null; MethodDesc caller = HandleToObject(callerHandle); - TypeDesc awaiterType = HandleToObject(callSig->sigInst.methInst[0]); + MethodDesc awaitAwaiterMethod = HandleToObject(pResolvedToken.hMethod); + Debug.Assert(awaitAwaiterMethod.Instantiation.Length == 1); + TypeDesc awaiterType = awaitAwaiterMethod.Instantiation[0]; + + // The resolved token gives us the canonical form of the awaiter + // type; the dependency analysis needs the runtime determined form. + var runtimeDeterminedAwaitAwaiterMethod = (MethodDesc)GetRuntimeDeterminedObjectForToken(ref pResolvedToken); + TypeDesc runtimeDeterminedAwaiterType = runtimeDeterminedAwaitAwaiterMethod.Instantiation[0]; + CompilerTypeSystemContext context = _compilation.TypeSystemContext; DefType asyncHelpers = context.SystemModule.GetKnownType("System.Runtime.CompilerServices"u8, "AsyncHelpers"u8); @@ -3728,39 +3734,40 @@ private void getWasmWellKnownGlobals(ref CORINFO_WASM_WELLKNOWN_GLOBALS pWellKno 1, context.GetWellKnownType(WellKnownType.Void), [context.GetWellKnownType(WellKnownType.Int32)]); - MethodDesc result = asyncHelpers - .GetKnownMethod( - isUnsafe ? "UnsafeAwaitAwaiterInContinuation"u8 : "AwaitAwaiterInContinuation"u8, - signature) - .MakeInstantiatedMethod(awaiterType); - MethodDesc targetMethod = result.GetCanonMethodTarget(CanonicalFormKind.Specific); + MethodDesc typicalMethod = asyncHelpers.GetKnownMethod( + isUnsafe ? "UnsafeAwaitAwaiterInContinuation"u8 : "AwaitAwaiterInContinuation"u8, + signature); + MethodDesc result = typicalMethod.MakeInstantiatedMethod(awaiterType); + MethodDesc runtimeDeterminedResult = typicalMethod.MakeInstantiatedMethod(runtimeDeterminedAwaiterType); + + MethodDesc targetMethod = runtimeDeterminedResult.GetCanonMethodTarget(CanonicalFormKind.Specific); *contextHandle = contextFromMethod(result); if (targetMethod.RequiresInstArg()) { #if READYTORUN - if (result.IsRuntimeDeterminedExactMethod) + if (runtimeDeterminedResult.IsRuntimeDeterminedExactMethod) { // TODO-Async: the instantiation argument would have to be obtained through a runtime - // generic dictionary lookup, which is not yet emitted here, so defer to the runtime JIT. - throw new RequiresRuntimeJitException($"getAwaitAwaiterInContinuationCall: runtime-determined exact instantiation requires runtime JIT ({result})"); + // generic dictionary lookup, which is not yet emitted here, so skip the optimization. + return null; } instArg.constLookup = CreateConstLookupToSymbol( _compilation.SymbolNodeFactory.CreateReadyToRunHelper( ReadyToRunHelperId.MethodDictionary, new MethodWithToken( - result, + runtimeDeterminedResult, _compilation.NodeFactory.Resolver.GetModuleTokenForMethod( - result, + runtimeDeterminedResult, allowDynamicallyCreatedReference: true, throwIfNotFound: true), constrainedType: null, unboxing: false, genericContextObject: caller))); #else - ComputeLookup(caller != MethodBeingCompiled, result, ReadyToRunHelperId.MethodDictionary, caller, - ref instArg); + ComputeLookup(caller != MethodBeingCompiled, runtimeDeterminedResult, ReadyToRunHelperId.MethodDictionary, + caller, ref instArg); #endif } diff --git a/src/coreclr/tools/Common/JitInterface/CorInfoImpl_generated.cs b/src/coreclr/tools/Common/JitInterface/CorInfoImpl_generated.cs index 2c705be59d0e0c..3cad520435442b 100644 --- a/src/coreclr/tools/Common/JitInterface/CorInfoImpl_generated.cs +++ b/src/coreclr/tools/Common/JitInterface/CorInfoImpl_generated.cs @@ -328,7 +328,7 @@ static ICorJitInfoCallbacks() public delegate* unmanaged getEEInfo; public delegate* unmanaged getAsyncInfo; public delegate* unmanaged getAwaitReturnCall; - public delegate* unmanaged getAwaitAwaiterInContinuationCall; + public delegate* unmanaged getAwaitAwaiterInContinuationCall; public delegate* unmanaged getMethodDefFromMethod; public delegate* unmanaged printMethodName; public delegate* unmanaged getMethodNameFromMetadata; @@ -2198,12 +2198,12 @@ private static void _getAsyncInfo(IntPtr thisHandle, IntPtr* ppException, CORINF } [UnmanagedCallersOnly] - private static CORINFO_METHOD_STRUCT_* _getAwaitAwaiterInContinuationCall(IntPtr thisHandle, IntPtr* ppException, CORINFO_METHOD_STRUCT_* callerHandle, CORINFO_SIG_INFO* callSig, byte isUnsafe, CORINFO_CONTEXT_STRUCT** contextHandle, CORINFO_LOOKUP* instArg) + private static CORINFO_METHOD_STRUCT_* _getAwaitAwaiterInContinuationCall(IntPtr thisHandle, IntPtr* ppException, CORINFO_METHOD_STRUCT_* callerHandle, CORINFO_RESOLVED_TOKEN* pResolvedToken, byte isUnsafe, CORINFO_CONTEXT_STRUCT** contextHandle, CORINFO_LOOKUP* instArg) { var _this = GetThis(thisHandle); try { - return _this.getAwaitAwaiterInContinuationCall(callerHandle, callSig, isUnsafe != 0, contextHandle, ref *instArg); + return _this.getAwaitAwaiterInContinuationCall(callerHandle, ref *pResolvedToken, isUnsafe != 0, contextHandle, ref *instArg); } catch (Exception ex) { diff --git a/src/coreclr/tools/Common/JitInterface/ThunkGenerator/ThunkInput.txt b/src/coreclr/tools/Common/JitInterface/ThunkGenerator/ThunkInput.txt index 1036b4fde3dfa9..2408217150bfb6 100644 --- a/src/coreclr/tools/Common/JitInterface/ThunkGenerator/ThunkInput.txt +++ b/src/coreclr/tools/Common/JitInterface/ThunkGenerator/ThunkInput.txt @@ -297,7 +297,7 @@ FUNCTIONS void getEEInfo(CORINFO_EE_INFO* pEEInfoOut); void getAsyncInfo(CORINFO_ASYNC_INFO* pAsyncInfoOut); CORINFO_METHOD_HANDLE getAwaitReturnCall(CORINFO_METHOD_HANDLE callerHandle, CORINFO_CONTEXT_HANDLE* contextHandle, CORINFO_LOOKUP* instArg); - CORINFO_METHOD_HANDLE getAwaitAwaiterInContinuationCall(CORINFO_METHOD_HANDLE callerHandle, CORINFO_SIG_INFO* callSig, bool isUnsafe, CORINFO_CONTEXT_HANDLE* contextHandle, CORINFO_LOOKUP* instArg); + CORINFO_METHOD_HANDLE getAwaitAwaiterInContinuationCall(CORINFO_METHOD_HANDLE callerHandle, CORINFO_RESOLVED_TOKEN* pResolvedToken, bool isUnsafe, CORINFO_CONTEXT_HANDLE* contextHandle, CORINFO_LOOKUP* instArg); mdMethodDef getMethodDefFromMethod(CORINFO_METHOD_HANDLE hMethod); size_t printMethodName(CORINFO_METHOD_HANDLE ftn, char* buffer, size_t bufferSize, size_t* pRequiredBufferSize) const char* getMethodNameFromMetadata(CORINFO_METHOD_HANDLE ftn, const char **className, const char **namespaceName, const char **enclosingClassNames, size_t maxEnclosingClassNames); diff --git a/src/coreclr/tools/aot/ILCompiler.Compiler/IL/ILImporter.Scanner.cs b/src/coreclr/tools/aot/ILCompiler.Compiler/IL/ILImporter.Scanner.cs index 4b7e02f0abd725..3f59e6fe137e2a 100644 --- a/src/coreclr/tools/aot/ILCompiler.Compiler/IL/ILImporter.Scanner.cs +++ b/src/coreclr/tools/aot/ILCompiler.Compiler/IL/ILImporter.Scanner.cs @@ -700,6 +700,14 @@ private void ImportCall(ILOpcode opcode, MethodDesc method, MethodDesc runtimeDe { return; } + + if (IsAsyncHelpersAwaitAwaiter(method, out bool isUnsafeAwaitAwaiter)) + { + // Not an early out: the call itself is still emitted, the JIT just may + // rewrite it to the corresponding "in continuation" helper. + RegisterDependenciesOnAwaitAwaiterInContinuation(runtimeDeterminedMethod, isUnsafeAwaitAwaiter, + reason); + } } } @@ -1073,6 +1081,57 @@ private void RegisterDependenciesOnAsyncCall() _dependencies.Add(_factory.MethodEntrypoint(asyncHelpers.GetKnownMethod("FinishSuspensionWithContinuationContext"u8, null)), asyncReason); } + // The JIT rewrites calls to AsyncHelpers.AwaitAwaiter/UnsafeAwaitAwaiter with a struct + // awaiter into calls to AsyncHelpers.AwaitAwaiterInContinuation/UnsafeAwaitAwaiterInContinuation + // to avoid boxing the awaiter. Report the dependencies for that rewrite here since scanning + // does not otherwise see those calls. + private void RegisterDependenciesOnAwaitAwaiterInContinuation(MethodDesc runtimeDeterminedMethod, bool isUnsafe, string reason) + { + TypeDesc awaiterType = runtimeDeterminedMethod.Instantiation[0]; + if (!awaiterType.IsValueType) + { + return; + } + + // YieldAwaiter is specially recognized by AsyncHelpers.UnsafeAwaitAwaiter, so the JIT + // does not rewrite it. Note: no namespace check here, mirroring the JIT; being an + // intrinsic type is enough to know this is the well known type from CoreLib. + if (awaiterType.IsIntrinsic + && awaiterType is MetadataType awaiterMetadataType + && awaiterMetadataType.Name == "YieldAwaiter"u8) + { + return; + } + + CompilerTypeSystemContext context = _compilation.TypeSystemContext; + DefType asyncHelpers = context.SystemModule.GetKnownType("System.Runtime.CompilerServices"u8, "AsyncHelpers"u8); + MethodSignature signature = new MethodSignature( + MethodSignatureFlags.Static, + 1, + context.GetWellKnownType(WellKnownType.Void), + [context.GetWellKnownType(WellKnownType.Int32)]); + MethodDesc runtimeDeterminedResult = asyncHelpers + .GetKnownMethod(isUnsafe ? "UnsafeAwaitAwaiterInContinuation"u8 : "AwaitAwaiterInContinuation"u8, signature) + .MakeInstantiatedMethod(awaiterType); + + MethodDesc targetMethod = runtimeDeterminedResult.GetCanonMethodTarget(CanonicalFormKind.Specific); + + if (runtimeDeterminedResult.IsRuntimeDeterminedExactMethod) + { + _dependencies.Add(GetGenericLookupHelper(ReadyToRunHelperId.MethodDictionary, runtimeDeterminedResult), reason); + _dependencies.Add(_factory.CanonicalEntrypoint(targetMethod), reason); + } + else + { + if (targetMethod.RequiresInstArg()) + { + _dependencies.Add(_factory.MethodGenericDictionary(runtimeDeterminedResult), reason); + } + + _dependencies.Add(GetMethodEntrypoint(targetMethod), reason); + } + } + private void ImportLdFtn(int token, ILOpcode opCode) { ImportCall(opCode, token); @@ -1848,6 +1907,30 @@ private static bool IsAsyncHelpersAwait(MethodDesc method) return false; } + private static bool IsAsyncHelpersAwaitAwaiter(MethodDesc method, out bool isUnsafe) + { + isUnsafe = false; + + if (method.IsIntrinsic && method.Instantiation.Length == 1) + { + Utf8Span methodName = method.Name; + bool isAwaitAwaiter = methodName == "AwaitAwaiter"u8; + if (isAwaitAwaiter || methodName == "UnsafeAwaitAwaiter"u8) + { + MetadataType owningType = method.OwningType as MetadataType; + if (owningType != null) + { + isUnsafe = !isAwaitAwaiter; + return owningType.Module == method.Context.SystemModule + && owningType.Name == "AsyncHelpers"u8 + && owningType.Namespace == "System.Runtime.CompilerServices"u8; + } + } + } + + return false; + } + private static bool IsTaskConfigureAwait(MethodDesc method) { if (method.IsIntrinsic && method.Name == "ConfigureAwait"u8) diff --git a/src/coreclr/tools/aot/jitinterface/jitinterface_generated.h b/src/coreclr/tools/aot/jitinterface/jitinterface_generated.h index c1387f628af346..63c3f1ea266574 100644 --- a/src/coreclr/tools/aot/jitinterface/jitinterface_generated.h +++ b/src/coreclr/tools/aot/jitinterface/jitinterface_generated.h @@ -133,7 +133,7 @@ struct JitInterfaceCallbacks void (* getEEInfo)(void * thisHandle, CorInfoExceptionClass** ppException, CORINFO_EE_INFO* pEEInfoOut); void (* getAsyncInfo)(void * thisHandle, CorInfoExceptionClass** ppException, CORINFO_ASYNC_INFO* pAsyncInfoOut); CORINFO_METHOD_HANDLE (* getAwaitReturnCall)(void * thisHandle, CorInfoExceptionClass** ppException, CORINFO_METHOD_HANDLE callerHandle, CORINFO_CONTEXT_HANDLE* contextHandle, CORINFO_LOOKUP* instArg); - CORINFO_METHOD_HANDLE (* getAwaitAwaiterInContinuationCall)(void * thisHandle, CorInfoExceptionClass** ppException, CORINFO_METHOD_HANDLE callerHandle, CORINFO_SIG_INFO* callSig, bool isUnsafe, CORINFO_CONTEXT_HANDLE* contextHandle, CORINFO_LOOKUP* instArg); + CORINFO_METHOD_HANDLE (* getAwaitAwaiterInContinuationCall)(void * thisHandle, CorInfoExceptionClass** ppException, CORINFO_METHOD_HANDLE callerHandle, CORINFO_RESOLVED_TOKEN* pResolvedToken, bool isUnsafe, CORINFO_CONTEXT_HANDLE* contextHandle, CORINFO_LOOKUP* instArg); mdMethodDef (* getMethodDefFromMethod)(void * thisHandle, CorInfoExceptionClass** ppException, CORINFO_METHOD_HANDLE hMethod); size_t (* printMethodName)(void * thisHandle, CorInfoExceptionClass** ppException, CORINFO_METHOD_HANDLE ftn, char* buffer, size_t bufferSize, size_t* pRequiredBufferSize); const char* (* getMethodNameFromMetadata)(void * thisHandle, CorInfoExceptionClass** ppException, CORINFO_METHOD_HANDLE ftn, const char** className, const char** namespaceName, const char** enclosingClassNames, size_t maxEnclosingClassNames); @@ -1387,13 +1387,13 @@ class JitInterfaceWrapper : public ICorJitInfo virtual CORINFO_METHOD_HANDLE getAwaitAwaiterInContinuationCall( CORINFO_METHOD_HANDLE callerHandle, - CORINFO_SIG_INFO* callSig, + CORINFO_RESOLVED_TOKEN* pResolvedToken, bool isUnsafe, CORINFO_CONTEXT_HANDLE* contextHandle, CORINFO_LOOKUP* instArg) { CorInfoExceptionClass* pException = nullptr; - CORINFO_METHOD_HANDLE temp = _callbacks->getAwaitAwaiterInContinuationCall(_thisHandle, &pException, callerHandle, callSig, isUnsafe, contextHandle, instArg); + CORINFO_METHOD_HANDLE temp = _callbacks->getAwaitAwaiterInContinuationCall(_thisHandle, &pException, callerHandle, pResolvedToken, isUnsafe, contextHandle, instArg); if (pException != nullptr) throw pException; return temp; } diff --git a/src/coreclr/tools/superpmi/superpmi-shared/agnostic.h b/src/coreclr/tools/superpmi/superpmi-shared/agnostic.h index 0f64d7436d779f..84416ade0044e7 100644 --- a/src/coreclr/tools/superpmi/superpmi-shared/agnostic.h +++ b/src/coreclr/tools/superpmi/superpmi-shared/agnostic.h @@ -255,13 +255,6 @@ struct Agnostic_GetAwaitReturnCallResult Agnostic_CORINFO_LOOKUP instArg; }; -struct Agnostic_GetAwaitAwaiterInContinuationCall -{ - DWORDLONG callerHnd; - Agnostic_CORINFO_SIG_INFO callSig; - DWORD isUnsafe; -}; - struct Agnostic_GetOSRInfo { DWORD index; @@ -299,6 +292,13 @@ struct Agnostic_CORINFO_RESOLVED_TOKEN Agnostic_CORINFO_RESOLVED_TOKENout outValue; }; +struct Agnostic_GetAwaitAwaiterInContinuationCall +{ + DWORDLONG callerHnd; + Agnostic_CORINFO_RESOLVED_TOKEN ResolvedToken; + DWORD isUnsafe; +}; + struct Agnostic_GetFieldInfo { Agnostic_CORINFO_RESOLVED_TOKEN ResolvedToken; diff --git a/src/coreclr/tools/superpmi/superpmi-shared/methodcontext.cpp b/src/coreclr/tools/superpmi/superpmi-shared/methodcontext.cpp index 778254d4c7e867..73e71489981d19 100644 --- a/src/coreclr/tools/superpmi/superpmi-shared/methodcontext.cpp +++ b/src/coreclr/tools/superpmi/superpmi-shared/methodcontext.cpp @@ -4484,7 +4484,7 @@ CORINFO_METHOD_HANDLE MethodContext::repGetAwaitReturnCall(CORINFO_METHOD_HANDLE } void MethodContext::recGetAwaitAwaiterInContinuationCall(CORINFO_METHOD_HANDLE callerHnd, - CORINFO_SIG_INFO* callSig, + CORINFO_RESOLVED_TOKEN* pResolvedToken, bool isUnsafe, CORINFO_CONTEXT_HANDLE* contextHandle, CORINFO_LOOKUP* instArg, @@ -4499,8 +4499,8 @@ void MethodContext::recGetAwaitAwaiterInContinuationCall(CORINFO_METHOD_HANDLE c Agnostic_GetAwaitAwaiterInContinuationCall key; ZeroMemory(&key, sizeof(key)); key.callerHnd = CastHandle(callerHnd); - key.callSig = - SpmiRecordsHelper::StoreAgnostic_CORINFO_SIG_INFO(*callSig, GetAwaitAwaiterInContinuationCall, SigInstHandleMap); + key.ResolvedToken = + SpmiRecordsHelper::StoreAgnostic_CORINFO_RESOLVED_TOKEN(pResolvedToken, GetAwaitAwaiterInContinuationCall); key.isUnsafe = isUnsafe; Agnostic_GetAwaitReturnCallResult value; @@ -4517,11 +4517,10 @@ void MethodContext::dmpGetAwaitAwaiterInContinuationCall( const Agnostic_GetAwaitAwaiterInContinuationCall& key, Agnostic_GetAwaitReturnCallResult& value) { - printf("GetAwaitAwaiterInContinuationCall caller-%016" PRIX64 " sig-%s unsafe-%u " + printf("GetAwaitAwaiterInContinuationCall caller-%016" PRIX64 " rt{%s} unsafe-%u " "methodHnd-%016" PRIX64 " contextHandle-%016" PRIX64 " instArg %s", key.callerHnd, - SpmiDumpHelper::DumpAgnostic_CORINFO_SIG_INFO( - key.callSig, GetAwaitAwaiterInContinuationCall, SigInstHandleMap).c_str(), + SpmiDumpHelper::DumpAgnostic_CORINFO_RESOLVED_TOKEN(key.ResolvedToken).c_str(), key.isUnsafe, value.methodHnd, value.contextHandle, @@ -4530,7 +4529,7 @@ void MethodContext::dmpGetAwaitAwaiterInContinuationCall( CORINFO_METHOD_HANDLE MethodContext::repGetAwaitAwaiterInContinuationCall( CORINFO_METHOD_HANDLE callerHnd, - CORINFO_SIG_INFO* callSig, + CORINFO_RESOLVED_TOKEN* pResolvedToken, bool isUnsafe, CORINFO_CONTEXT_HANDLE* contextHandle, CORINFO_LOOKUP* instArg) @@ -4538,8 +4537,8 @@ CORINFO_METHOD_HANDLE MethodContext::repGetAwaitAwaiterInContinuationCall( Agnostic_GetAwaitAwaiterInContinuationCall key; ZeroMemory(&key, sizeof(key)); key.callerHnd = CastHandle(callerHnd); - key.callSig = SpmiRecordsHelper::RestoreAgnostic_CORINFO_SIG_INFO( - *callSig, GetAwaitAwaiterInContinuationCall, SigInstHandleMap); + key.ResolvedToken = + SpmiRecordsHelper::RestoreAgnostic_CORINFO_RESOLVED_TOKEN(pResolvedToken, GetAwaitAwaiterInContinuationCall); key.isUnsafe = isUnsafe; Agnostic_GetAwaitReturnCallResult value = diff --git a/src/coreclr/tools/superpmi/superpmi-shared/methodcontext.h b/src/coreclr/tools/superpmi/superpmi-shared/methodcontext.h index 939c2bed40ae27..3a4664fab8af6e 100644 --- a/src/coreclr/tools/superpmi/superpmi-shared/methodcontext.h +++ b/src/coreclr/tools/superpmi/superpmi-shared/methodcontext.h @@ -568,7 +568,7 @@ class MethodContext CORINFO_METHOD_HANDLE repGetAwaitReturnCall(CORINFO_METHOD_HANDLE callerHnd, CORINFO_CONTEXT_HANDLE* contextHandle, CORINFO_LOOKUP* instArg); void recGetAwaitAwaiterInContinuationCall(CORINFO_METHOD_HANDLE callerHnd, - CORINFO_SIG_INFO* callSig, + CORINFO_RESOLVED_TOKEN* pResolvedToken, bool isUnsafe, CORINFO_CONTEXT_HANDLE* contextHandle, CORINFO_LOOKUP* instArg, @@ -576,7 +576,7 @@ class MethodContext void dmpGetAwaitAwaiterInContinuationCall(const Agnostic_GetAwaitAwaiterInContinuationCall& key, Agnostic_GetAwaitReturnCallResult& value); CORINFO_METHOD_HANDLE repGetAwaitAwaiterInContinuationCall(CORINFO_METHOD_HANDLE callerHnd, - CORINFO_SIG_INFO* callSig, + CORINFO_RESOLVED_TOKEN* pResolvedToken, bool isUnsafe, CORINFO_CONTEXT_HANDLE* contextHandle, CORINFO_LOOKUP* instArg); diff --git a/src/coreclr/tools/superpmi/superpmi-shim-collector/icorjitinfo.cpp b/src/coreclr/tools/superpmi/superpmi-shim-collector/icorjitinfo.cpp index f1d9cfad2fad54..82babd5a9d9964 100644 --- a/src/coreclr/tools/superpmi/superpmi-shim-collector/icorjitinfo.cpp +++ b/src/coreclr/tools/superpmi/superpmi-shim-collector/icorjitinfo.cpp @@ -1406,15 +1406,15 @@ CORINFO_METHOD_HANDLE interceptor_ICJI::getAwaitReturnCall(CORINFO_METHOD_HANDLE CORINFO_METHOD_HANDLE interceptor_ICJI::getAwaitAwaiterInContinuationCall( CORINFO_METHOD_HANDLE callerHandle, - CORINFO_SIG_INFO* callSig, + CORINFO_RESOLVED_TOKEN* pResolvedToken, bool isUnsafe, CORINFO_CONTEXT_HANDLE* contextHandle, CORINFO_LOOKUP* instArg) { mc->cr->AddCall("getAwaitAwaiterInContinuationCall"); CORINFO_METHOD_HANDLE result = original_ICorJitInfo->getAwaitAwaiterInContinuationCall( - callerHandle, callSig, isUnsafe, contextHandle, instArg); - mc->recGetAwaitAwaiterInContinuationCall(callerHandle, callSig, isUnsafe, contextHandle, instArg, result); + callerHandle, pResolvedToken, isUnsafe, contextHandle, instArg); + mc->recGetAwaitAwaiterInContinuationCall(callerHandle, pResolvedToken, isUnsafe, contextHandle, instArg, result); return result; } diff --git a/src/coreclr/tools/superpmi/superpmi-shim-counter/icorjitinfo_generated.cpp b/src/coreclr/tools/superpmi/superpmi-shim-counter/icorjitinfo_generated.cpp index 14a88356f7a280..de53f39764e9db 100644 --- a/src/coreclr/tools/superpmi/superpmi-shim-counter/icorjitinfo_generated.cpp +++ b/src/coreclr/tools/superpmi/superpmi-shim-counter/icorjitinfo_generated.cpp @@ -987,13 +987,13 @@ CORINFO_METHOD_HANDLE interceptor_ICJI::getAwaitReturnCall( CORINFO_METHOD_HANDLE interceptor_ICJI::getAwaitAwaiterInContinuationCall( CORINFO_METHOD_HANDLE callerHandle, - CORINFO_SIG_INFO* callSig, + CORINFO_RESOLVED_TOKEN* pResolvedToken, bool isUnsafe, CORINFO_CONTEXT_HANDLE* contextHandle, CORINFO_LOOKUP* instArg) { mcs->AddCall("getAwaitAwaiterInContinuationCall"); - return original_ICorJitInfo->getAwaitAwaiterInContinuationCall(callerHandle, callSig, isUnsafe, contextHandle, instArg); + return original_ICorJitInfo->getAwaitAwaiterInContinuationCall(callerHandle, pResolvedToken, isUnsafe, contextHandle, instArg); } mdMethodDef interceptor_ICJI::getMethodDefFromMethod( diff --git a/src/coreclr/tools/superpmi/superpmi-shim-simple/icorjitinfo_generated.cpp b/src/coreclr/tools/superpmi/superpmi-shim-simple/icorjitinfo_generated.cpp index 2d6199fae28c6a..e51bb1ecbc4f1f 100644 --- a/src/coreclr/tools/superpmi/superpmi-shim-simple/icorjitinfo_generated.cpp +++ b/src/coreclr/tools/superpmi/superpmi-shim-simple/icorjitinfo_generated.cpp @@ -865,12 +865,12 @@ CORINFO_METHOD_HANDLE interceptor_ICJI::getAwaitReturnCall( CORINFO_METHOD_HANDLE interceptor_ICJI::getAwaitAwaiterInContinuationCall( CORINFO_METHOD_HANDLE callerHandle, - CORINFO_SIG_INFO* callSig, + CORINFO_RESOLVED_TOKEN* pResolvedToken, bool isUnsafe, CORINFO_CONTEXT_HANDLE* contextHandle, CORINFO_LOOKUP* instArg) { - return original_ICorJitInfo->getAwaitAwaiterInContinuationCall(callerHandle, callSig, isUnsafe, contextHandle, instArg); + return original_ICorJitInfo->getAwaitAwaiterInContinuationCall(callerHandle, pResolvedToken, isUnsafe, contextHandle, instArg); } mdMethodDef interceptor_ICJI::getMethodDefFromMethod( diff --git a/src/coreclr/tools/superpmi/superpmi/icorjitinfo.cpp b/src/coreclr/tools/superpmi/superpmi/icorjitinfo.cpp index 15b4c8d0d0c398..4b8c67aef84bb3 100644 --- a/src/coreclr/tools/superpmi/superpmi/icorjitinfo.cpp +++ b/src/coreclr/tools/superpmi/superpmi/icorjitinfo.cpp @@ -1216,14 +1216,14 @@ CORINFO_METHOD_HANDLE MyICJI::getAwaitReturnCall(CORINFO_METHOD_HANDLE callerHan CORINFO_METHOD_HANDLE MyICJI::getAwaitAwaiterInContinuationCall( CORINFO_METHOD_HANDLE callerHandle, - CORINFO_SIG_INFO* callSig, + CORINFO_RESOLVED_TOKEN* pResolvedToken, bool isUnsafe, CORINFO_CONTEXT_HANDLE* contextHandle, CORINFO_LOOKUP* instArg) { jitInstance->mc->cr->AddCall("getAwaitAwaiterInContinuationCall"); return jitInstance->mc->repGetAwaitAwaiterInContinuationCall( - callerHandle, callSig, isUnsafe, contextHandle, instArg); + callerHandle, pResolvedToken, isUnsafe, contextHandle, instArg); } /*********************************************************************************/ diff --git a/src/coreclr/vm/jitinterface.cpp b/src/coreclr/vm/jitinterface.cpp index 784da9288c1ff1..c783d529d5d6bc 100644 --- a/src/coreclr/vm/jitinterface.cpp +++ b/src/coreclr/vm/jitinterface.cpp @@ -10476,7 +10476,7 @@ CORINFO_METHOD_HANDLE CEEInfo::getAwaitReturnCall(CORINFO_METHOD_HANDLE callerHa CORINFO_METHOD_HANDLE CEEInfo::getAwaitAwaiterInContinuationCall( CORINFO_METHOD_HANDLE callerHandle, - CORINFO_SIG_INFO* callSig, + CORINFO_RESOLVED_TOKEN* pResolvedToken, bool isUnsafe, CORINFO_CONTEXT_HANDLE* contextHandle, CORINFO_LOOKUP* instArg) @@ -10491,8 +10491,9 @@ CORINFO_METHOD_HANDLE CEEInfo::getAwaitAwaiterInContinuationCall( JIT_TO_EE_TRANSITION(); - _ASSERTE(callSig->sigInst.methInstCount == 1); - TypeHandle awaiterType(callSig->sigInst.methInst[0]); + MethodDesc* pAwaitAwaiterMD = GetMethod(pResolvedToken->hMethod); + _ASSERTE(pAwaitAwaiterMD->GetNumGenericMethodArgs() == 1); + TypeHandle awaiterType = pAwaitAwaiterMD->GetMethodInstantiation()[0]; instArg->lookupKind.needsRuntimeLookup = false; instArg->constLookup.accessType = IAT_VALUE; @@ -10510,7 +10511,10 @@ CORINFO_METHOD_HANDLE CEEInfo::getAwaitAwaiterInContinuationCall( { if (awaiterType.IsCanonicalSubtype()) { - ComputeRuntimeLookupForAwaitAwaiterInContinuationCall(pCallerMD, pTypicalAwaitMD, callSig, instArg); + // We need the exact instantiation from the call site to be able to + // create the runtime lookup. + _ASSERTE(pResolvedToken->pMethodSpec != NULL); + ComputeRuntimeLookupForAwaitAwaiterInContinuationCall(pCallerMD, pTypicalAwaitMD, pResolvedToken, instArg); } else { @@ -10595,7 +10599,7 @@ void CEEInfo::ComputeRuntimeLookupForAwaitCall(MethodDesc* pCallerMD, MethodDesc void CEEInfo::ComputeRuntimeLookupForAwaitAwaiterInContinuationCall( MethodDesc* pCallerMD, MethodDesc* pTypicalAwaitMD, - CORINFO_SIG_INFO* callSig, + CORINFO_RESOLVED_TOKEN* pResolvedToken, CORINFO_LOOKUP* lookup) { lookup->lookupKind.needsRuntimeLookup = true; @@ -10638,15 +10642,15 @@ void CEEInfo::ComputeRuntimeLookupForAwaitAwaiterInContinuationCall( sigBuilder.AppendData(RidFromToken(pTypicalAwaitMD->GetMemberDef())); sigBuilder.AppendData(1); - _ASSERTE(callSig->pSig != NULL); - SigPointer instantiation(callSig->pSig, callSig->cbSig); + _ASSERTE(pResolvedToken->pMethodSpec != NULL); + SigPointer instantiation(pResolvedToken->pMethodSpec, pResolvedToken->cbMethodSpec); BYTE callingConvention; IfFailThrow(instantiation.GetByte(&callingConvention)); _ASSERTE(callingConvention == IMAGE_CEE_CS_CALLCONV_GENERICINST); uint32_t numArgs; IfFailThrow(instantiation.GetData(&numArgs)); _ASSERTE(numArgs == 1); - instantiation.ConvertToInternalExactlyOne(GetModule(callSig->scope), NULL, &sigBuilder); + instantiation.ConvertToInternalExactlyOne(GetModule(pResolvedToken->tokenScope), NULL, &sigBuilder); FinishComputeRuntimeLookup(sigBuilder, pCallerMD, lookup); } diff --git a/src/coreclr/vm/jitinterface.h b/src/coreclr/vm/jitinterface.h index 5072ab4c854889..3aea6b17a59f12 100644 --- a/src/coreclr/vm/jitinterface.h +++ b/src/coreclr/vm/jitinterface.h @@ -416,7 +416,7 @@ class CEEInfo : public ICorJitInfo void ComputeRuntimeLookupForAwaitAwaiterInContinuationCall( MethodDesc* pCallerMD, MethodDesc* pTypicalAwaitMD, - CORINFO_SIG_INFO* callSig, + CORINFO_RESOLVED_TOKEN* pResolvedToken, CORINFO_LOOKUP* lookup); #if defined(FEATURE_GDBJIT) diff --git a/src/tests/async/custom-struct-awaiters/custom-struct-awaiters.cs b/src/tests/async/custom-struct-awaiters/custom-struct-awaiters.cs index 6946cb6975745b..9fdc473d074667 100644 --- a/src/tests/async/custom-struct-awaiters/custom-struct-awaiters.cs +++ b/src/tests/async/custom-struct-awaiters/custom-struct-awaiters.cs @@ -27,6 +27,37 @@ private static async Task Run() Volatile.Write(ref s_unsafeAwaiterValue, 0); await new UnsafeAwaitable(43); Assert.Equal(43, Volatile.Read(ref s_unsafeAwaiterValue)); + + // Reference type instantiations share code, so the awaiters here are + // runtime determined and require runtime lookups. + await RunGeneric("shared generic"); + await RunGeneric(new object()); + // Value type instantiations are exact. + await RunGeneric(44); + } + + private static async Task RunGeneric(T value) + { + Tracker.Value = default; + Tracker.Completions = 0; + + T safeResult = await new GenericSafeAwaitable(value); + Assert.Equal(value, safeResult); + Assert.Equal(value, Tracker.Value); + Assert.Equal(1, Volatile.Read(ref Tracker.Completions)); + + Tracker.Value = default; + + T unsafeResult = await new GenericUnsafeAwaitable(value); + Assert.Equal(value, unsafeResult); + Assert.Equal(value, Tracker.Value); + Assert.Equal(2, Volatile.Read(ref Tracker.Completions)); + } + + private static class Tracker + { + public static T? Value; + public static int Completions; } private readonly struct SafeAwaitable @@ -82,4 +113,60 @@ public void UnsafeOnCompleted(Action continuation) public void GetResult() => Assert.Equal(43, _value); } } + + private readonly struct GenericSafeAwaitable + { + private readonly T _value; + + public GenericSafeAwaitable(T value) => _value = value; + + public GenericSafeAwaiter GetAwaiter() => new GenericSafeAwaiter(_value); + } + + private readonly struct GenericSafeAwaiter : INotifyCompletion + { + private readonly T _value; + + public GenericSafeAwaiter(T value) => _value = value; + + public bool IsCompleted => false; + + public void OnCompleted(Action continuation) + { + Tracker.Value = _value; + Interlocked.Increment(ref Tracker.Completions); + ThreadPool.QueueUserWorkItem(static state => ((Action)state!).Invoke(), continuation); + } + + public T GetResult() => _value; + } + + private readonly struct GenericUnsafeAwaitable + { + private readonly T _value; + + public GenericUnsafeAwaitable(T value) => _value = value; + + public GenericUnsafeAwaiter GetAwaiter() => new GenericUnsafeAwaiter(_value); + } + + private readonly struct GenericUnsafeAwaiter : ICriticalNotifyCompletion + { + private readonly T _value; + + public GenericUnsafeAwaiter(T value) => _value = value; + + public bool IsCompleted => false; + + public void OnCompleted(Action continuation) => throw new InvalidOperationException(); + + public void UnsafeOnCompleted(Action continuation) + { + Tracker.Value = _value; + Interlocked.Increment(ref Tracker.Completions); + ThreadPool.UnsafeQueueUserWorkItem(static state => ((Action)state!).Invoke(), continuation); + } + + public T GetResult() => _value; + } } From 516da6db6a4d2eea802b9863f2bbee9354aa738a Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Tue, 28 Jul 2026 13:03:15 +0200 Subject: [PATCH 18/19] Document why the InContinuation helpers need manifest token pre-seeding Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 757a9e4c-44f3-4255-86f6-065ebb823ed8 --- .../Compiler/ReadyToRunCodegenCompilation.cs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/ReadyToRunCodegenCompilation.cs b/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/ReadyToRunCodegenCompilation.cs index 1a7a9012ecd2c8..e1cff1cb709e25 100644 --- a/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/ReadyToRunCodegenCompilation.cs +++ b/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/ReadyToRunCodegenCompilation.cs @@ -1064,6 +1064,16 @@ private void AddNecessaryAsyncReferences(MethodDesc method) asyncHelpers.GetKnownMethod("TransparentAwait"u8, new MethodSignature(MethodSignatureFlags.Static, 0, voidType, [valueTaskType])), asyncHelpers.GetKnownMethod("TransparentAwait"u8, new MethodSignature(MethodSignatureFlags.Static, 1, methodVar, [taskOfTType.MakeInstantiatedType(methodVar)])), asyncHelpers.GetKnownMethod("TransparentAwait"u8, new MethodSignature(MethodSignatureFlags.Static, 1, methodVar, [valueTaskOfTType.MakeInstantiatedType(methodVar)])), + + // For CorInfoImpl.getAwaitAwaiterInContinuationCall. Same as above: the JIT rewrites calls + // to AsyncHelpers.AwaitAwaiter/UnsafeAwaitAwaiter into calls to these, so nothing in the + // caller's IL refers to them and their manifest tokens must be pre-seeded here. + // + // Only the typical definitions need tokens. The method fixup signature emits the method's + // def/ref token and encodes the instantiation separately as type signatures (see + // SignatureBuilder.EmitMethodSpecificationSignature), so no MethodSpec token is required. + // The instantiation argument is the awaiter type, which the caller already refers to in its + // own IL, so that is guaranteed to be encodable as well. asyncHelpers.GetKnownMethod("AwaitAwaiterInContinuation"u8, null), asyncHelpers.GetKnownMethod("UnsafeAwaitAwaiterInContinuation"u8, null), ]; From ed5e8ed5c5356d050388c50b0d5525d677804a3c Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Tue, 28 Jul 2026 13:27:38 +0200 Subject: [PATCH 19/19] Fix --- .../tools/Common/JitInterface/CorInfoImpl.cs | 18 ++++++++++++++---- .../ReadyToRun/TypeValidationChecker.cs | 4 ++-- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/src/coreclr/tools/Common/JitInterface/CorInfoImpl.cs b/src/coreclr/tools/Common/JitInterface/CorInfoImpl.cs index 86f75f4472837e..37b06f69469469 100644 --- a/src/coreclr/tools/Common/JitInterface/CorInfoImpl.cs +++ b/src/coreclr/tools/Common/JitInterface/CorInfoImpl.cs @@ -3683,8 +3683,8 @@ private void getWasmWellKnownGlobals(ref CORINFO_WASM_WELLKNOWN_GLOBALS pWellKno if (runtimeDeterminedResult.IsRuntimeDeterminedExactMethod) { // TODO-Async: the instantiation argument would have to be obtained through a runtime - // generic dictionary lookup, which is not yet emitted here, so skip the optimization. - return null; + // generic dictionary lookup, which is not yet emitted here, so defer to the runtime JIT. + throw new RequiresRuntimeJitException($"getAwaitReturnCall: runtime-determined exact instantiation requires runtime JIT ({runtimeDeterminedResult})"); } instArg.constLookup = CreateConstLookupToSymbol( @@ -3749,8 +3749,18 @@ private void getWasmWellKnownGlobals(ref CORINFO_WASM_WELLKNOWN_GLOBALS pWellKno if (runtimeDeterminedResult.IsRuntimeDeterminedExactMethod) { // TODO-Async: the instantiation argument would have to be obtained through a runtime - // generic dictionary lookup, which is not yet emitted here, so skip the optimization. - return null; + // generic dictionary lookup, which is not yet emitted here. + if (((ReadyToRunCompilerContext)context).TargetAllowsRuntimeCodeGeneration) + { + // Leave this method to runtime JIT that will be able to avoid the box + throw new RequiresRuntimeJitException($"getAwaitReturnCall: runtime-determined exact instantiation requires runtime JIT ({runtimeDeterminedResult})"); + } + else + { + // Skip the optimization, which will result in a box, + // but is still better than interpreter fallback + return null; + } } instArg.constLookup = CreateConstLookupToSymbol( diff --git a/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/DependencyAnalysis/ReadyToRun/TypeValidationChecker.cs b/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/DependencyAnalysis/ReadyToRun/TypeValidationChecker.cs index 6671793efd9150..f0c13d8bf9ac7f 100644 --- a/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/DependencyAnalysis/ReadyToRun/TypeValidationChecker.cs +++ b/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/DependencyAnalysis/ReadyToRun/TypeValidationChecker.cs @@ -73,8 +73,8 @@ private async Task CanSkipValidationInstance(EcmaModule module) // Spot check that the system module ALWAYS succeeds if (failAtEnd) { - LogErrors(s => Console.WriteLine(s)); - throw new InternalCompilerErrorException("System module failed to validate all types"); + throw new InternalCompilerErrorException("System module failed to validate all types:" + Environment.NewLine + + string.Join(Environment.NewLine, _typeLoadValidationErrors.Select(e => e.type.ToString()))); } } #endif