From f9d010dada3676ea588cf3cb17a2a8bae10f04bd Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sun, 28 Jun 2026 09:37:48 +0000 Subject: [PATCH 1/7] Auto-enable grouped MoE on loaded / PEFT'd models via loader hook Wraps the FastLlamaModel and FastBaseModel from_pretrained / get_peft_model leaves with wrap_loader_for_grouped_moe so the grouped-GEMM MoE forward is installed on the live instance after the model and its compiled module are built. Gated by UNSLOTH_MOE_GROUPED and wrapped in try/except, so it is a no-op when the unsloth_zoo module is absent or no eligible MoE block exists. --- unsloth/models/llama.py | 10 ++++++++++ unsloth/models/vision.py | 9 +++++++++ 2 files changed, 19 insertions(+) diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py index 14ee5ee24ea..937aafb4533 100644 --- a/unsloth/models/llama.py +++ b/unsloth/models/llama.py @@ -3736,3 +3736,13 @@ def _for_training(m): from .rl import PatchFastRL PatchFastRL(FastLanguageModel = FastLlamaModel) + + +# Auto-enable grouped-GEMM MoE (transformers<5 ModuleList experts) on built / PEFT'd +# models. Wraps the loader leaves once; guarded so it never breaks model loading. +try: + from unsloth_zoo.temporary_patches.moe_grouped_modulelist import wrap_loader_for_grouped_moe + FastLlamaModel.from_pretrained = staticmethod(wrap_loader_for_grouped_moe(FastLlamaModel.from_pretrained)) + FastLlamaModel.get_peft_model = staticmethod(wrap_loader_for_grouped_moe(FastLlamaModel.get_peft_model)) +except Exception: + pass diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py index bdc2bd9ef62..21332ee8675 100644 --- a/unsloth/models/vision.py +++ b/unsloth/models/vision.py @@ -2119,3 +2119,12 @@ def check_dataset_for_missing_videos( warnings.warn(error_msg, stacklevel = 2) return missing + + +# Auto-enable grouped-GEMM MoE (transformers<5 ModuleList experts); see llama.py. +try: + from unsloth_zoo.temporary_patches.moe_grouped_modulelist import wrap_loader_for_grouped_moe + FastBaseModel.from_pretrained = staticmethod(wrap_loader_for_grouped_moe(FastBaseModel.from_pretrained)) + FastBaseModel.get_peft_model = staticmethod(wrap_loader_for_grouped_moe(FastBaseModel.get_peft_model)) +except Exception: + pass From c0bc6ef0f1da9db729b989de2f843e8527af1f4a Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sun, 28 Jun 2026 09:38:16 +0000 Subject: [PATCH 2/7] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- unsloth/models/llama.py | 8 ++++++-- unsloth/models/vision.py | 8 ++++++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py index 937aafb4533..7819b0b9dac 100644 --- a/unsloth/models/llama.py +++ b/unsloth/models/llama.py @@ -3742,7 +3742,11 @@ def _for_training(m): # models. Wraps the loader leaves once; guarded so it never breaks model loading. try: from unsloth_zoo.temporary_patches.moe_grouped_modulelist import wrap_loader_for_grouped_moe - FastLlamaModel.from_pretrained = staticmethod(wrap_loader_for_grouped_moe(FastLlamaModel.from_pretrained)) - FastLlamaModel.get_peft_model = staticmethod(wrap_loader_for_grouped_moe(FastLlamaModel.get_peft_model)) + FastLlamaModel.from_pretrained = staticmethod( + wrap_loader_for_grouped_moe(FastLlamaModel.from_pretrained) + ) + FastLlamaModel.get_peft_model = staticmethod( + wrap_loader_for_grouped_moe(FastLlamaModel.get_peft_model) + ) except Exception: pass diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py index 21332ee8675..852cfb9a3eb 100644 --- a/unsloth/models/vision.py +++ b/unsloth/models/vision.py @@ -2124,7 +2124,11 @@ def check_dataset_for_missing_videos( # Auto-enable grouped-GEMM MoE (transformers<5 ModuleList experts); see llama.py. try: from unsloth_zoo.temporary_patches.moe_grouped_modulelist import wrap_loader_for_grouped_moe - FastBaseModel.from_pretrained = staticmethod(wrap_loader_for_grouped_moe(FastBaseModel.from_pretrained)) - FastBaseModel.get_peft_model = staticmethod(wrap_loader_for_grouped_moe(FastBaseModel.get_peft_model)) + FastBaseModel.from_pretrained = staticmethod( + wrap_loader_for_grouped_moe(FastBaseModel.from_pretrained) + ) + FastBaseModel.get_peft_model = staticmethod( + wrap_loader_for_grouped_moe(FastBaseModel.get_peft_model) + ) except Exception: pass From 51f07a6656d754c962b78eb85fd58a3796137e28 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sun, 28 Jun 2026 10:45:46 +0000 Subject: [PATCH 3/7] Install grouped-MoE loader wrappers before PatchFastRL --- unsloth/models/llama.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py index 7819b0b9dac..392ab555dc8 100644 --- a/unsloth/models/llama.py +++ b/unsloth/models/llama.py @@ -3735,11 +3735,9 @@ def _for_training(m): from .rl import PatchFastRL -PatchFastRL(FastLanguageModel = FastLlamaModel) - - # Auto-enable grouped-GEMM MoE (transformers<5 ModuleList experts) on built / PEFT'd -# models. Wraps the loader leaves once; guarded so it never breaks model loading. +# models. Wrap the loader leaves before PatchFastRL so any patcher that captures these +# entry points sees the wrapped versions. Guarded so it never breaks model loading. try: from unsloth_zoo.temporary_patches.moe_grouped_modulelist import wrap_loader_for_grouped_moe FastLlamaModel.from_pretrained = staticmethod( @@ -3750,3 +3748,5 @@ def _for_training(m): ) except Exception: pass + +PatchFastRL(FastLanguageModel = FastLlamaModel) From 0085ca43703a036d88eb11967e8f23626cd04865 Mon Sep 17 00:00:00 2001 From: danielhanchen Date: Wed, 1 Jul 2026 12:49:39 +0000 Subject: [PATCH 4/7] Re-evaluate grouped MoE after loading a PEFT adapter When loading an existing adapter through FastLanguageModel.from_pretrained, the base model is evaluated for grouped MoE when the wrapped from_pretrained leaf returns, but the adapter is attached afterwards via PeftModel and patch_peft_model. Re-run auto_enable_grouped_moe on the final model so blocks whose experts gained LoRA are restored to the original loop, attention-only adapters keep the grouped path on their frozen experts, and recompute is re-derived from the final gradient-checkpointing state. Guarded so it never blocks adapter loading. --- unsloth/models/loader.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py index 562afdd645d..0fd913c680f 100644 --- a/unsloth/models/loader.py +++ b/unsloth/models/loader.py @@ -876,6 +876,16 @@ def from_pretrained( ) # Patch it as well! model = dispatch_model.patch_peft_model(model, use_gradient_checkpointing) + # The base model was evaluated for grouped MoE when dispatch_model.from_pretrained + # returned; re-evaluate now that the adapter is attached so blocks whose experts + # gained LoRA are cleanly restored to the original loop, attention-only adapters keep + # the grouped path on their frozen experts, and recompute is re-derived from the final + # gradient-checkpointing state. Guarded so it never blocks adapter loading. + try: + from unsloth_zoo.temporary_patches.moe_grouped_modulelist import auto_enable_grouped_moe + auto_enable_grouped_moe(model) + except Exception: + pass # optional speedup; never block model loading # Patch Tiled MLP # to turn on set UNSLOTH_TILED_MLP to "arctic", "target", or "target:{GB}"" From ef8d27bc690ccba6ebd48a2b971bf5e93c7e74f1 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 1 Jul 2026 12:50:07 +0000 Subject: [PATCH 5/7] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- unsloth/models/loader.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py index 0fd913c680f..4185ae9dc39 100644 --- a/unsloth/models/loader.py +++ b/unsloth/models/loader.py @@ -882,7 +882,9 @@ def from_pretrained( # the grouped path on their frozen experts, and recompute is re-derived from the final # gradient-checkpointing state. Guarded so it never blocks adapter loading. try: - from unsloth_zoo.temporary_patches.moe_grouped_modulelist import auto_enable_grouped_moe + from unsloth_zoo.temporary_patches.moe_grouped_modulelist import ( + auto_enable_grouped_moe, + ) auto_enable_grouped_moe(model) except Exception: pass # optional speedup; never block model loading From ec6fe59d21bbc9311404ca82f8bcca8c354ad587 Mon Sep 17 00:00:00 2001 From: danielhanchen Date: Fri, 3 Jul 2026 09:17:27 +0000 Subject: [PATCH 6/7] Trim comments in the grouped MoE loader hooks Shorten the loader re-eval and llama.py wrapper comments; code is unchanged (verified comment-only). --- unsloth/models/llama.py | 5 ++--- unsloth/models/loader.py | 7 ++----- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py index 392ab555dc8..496d5660755 100644 --- a/unsloth/models/llama.py +++ b/unsloth/models/llama.py @@ -3735,9 +3735,8 @@ def _for_training(m): from .rl import PatchFastRL -# Auto-enable grouped-GEMM MoE (transformers<5 ModuleList experts) on built / PEFT'd -# models. Wrap the loader leaves before PatchFastRL so any patcher that captures these -# entry points sees the wrapped versions. Guarded so it never breaks model loading. +# Auto-enable grouped-GEMM MoE (tf<5 ModuleList experts) on built / PEFT'd models. Wrap the +# loader leaves before PatchFastRL so downstream patchers see the wrapped versions. Guarded. try: from unsloth_zoo.temporary_patches.moe_grouped_modulelist import wrap_loader_for_grouped_moe FastLlamaModel.from_pretrained = staticmethod( diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py index 4185ae9dc39..a56829a516d 100644 --- a/unsloth/models/loader.py +++ b/unsloth/models/loader.py @@ -876,11 +876,8 @@ def from_pretrained( ) # Patch it as well! model = dispatch_model.patch_peft_model(model, use_gradient_checkpointing) - # The base model was evaluated for grouped MoE when dispatch_model.from_pretrained - # returned; re-evaluate now that the adapter is attached so blocks whose experts - # gained LoRA are cleanly restored to the original loop, attention-only adapters keep - # the grouped path on their frozen experts, and recompute is re-derived from the final - # gradient-checkpointing state. Guarded so it never blocks adapter loading. + # Re-evaluate grouped MoE now the adapter is attached: an expert-LoRA block falls back + # to the original loop, an attention-only adapter keeps the grouped path. Guarded. try: from unsloth_zoo.temporary_patches.moe_grouped_modulelist import ( auto_enable_grouped_moe, From 25a03e2bf9f17082d602d2134f57898e4c3efa90 Mon Sep 17 00:00:00 2001 From: danielhanchen Date: Mon, 6 Jul 2026 07:31:48 +0000 Subject: [PATCH 7/7] Re-evaluate grouped MoE after loading a PEFT adapter on the vision path --- unsloth/models/loader.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py index a56829a516d..0897b5ba1a4 100644 --- a/unsloth/models/loader.py +++ b/unsloth/models/loader.py @@ -1818,6 +1818,15 @@ def _patched_car( model = FastBaseModel.post_patch_model( model, use_gradient_checkpointing, trust_remote_code = trust_remote_code ) + # Re-evaluate grouped MoE now the adapter is attached: an expert-LoRA block falls back + # to the original loop, an attention-only adapter keeps the grouped path. Guarded. + try: + from unsloth_zoo.temporary_patches.moe_grouped_modulelist import ( + auto_enable_grouped_moe, + ) + auto_enable_grouped_moe(model) + except Exception: + pass # optional speedup; never block model loading # Apply QAT if specified if qat_scheme is not None: