From c68e60ed91767fd221a5b3adcbe989785fdc4628 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Fri, 3 Jul 2026 14:25:38 +0000 Subject: [PATCH 1/9] Scope MoE expert LoRA detection to actual MLP projection targets _moe_target_set_from_string treated any regex containing the substring mlp or ffn as targeting the expert MLP projections. Unsloth's auto-generated attention-only regex lists mlp, ffn and feed_forward as allowed intermediate path segments while its final group matches only q_proj/k_proj/v_proj/o_proj, so attention-only finetuning on MoE models silently enabled expert LoRA as well: the experts were trained and every MoE layer paid the extra expert LoRA grouped matmuls. Detect expert intent from the projection names themselves (gate_proj/up_proj/down_proj/gate_up_proj) instead of the mlp substring. --- unsloth/models/_utils.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py index 599a5c0262..b961a7ede6 100644 --- a/unsloth/models/_utils.py +++ b/unsloth/models/_utils.py @@ -3507,8 +3507,14 @@ def _moe_target_set_from_string(target_modules: str) -> set[str]: return {target_modules} is_regex = re.search(r"[*+?()[\]{}|\\^$]", target_modules) is not None - targets_mlp = "mlp" in target_modules or "ffn" in target_modules - if is_regex and "proj" in target_modules and targets_mlp: + # Key expert-LoRA detection on the projection names themselves: the auto + # q/k/v/o regex lists mlp|ffn as path segments, so a bare "mlp" substring + # check silently enabled expert LoRA for attention-only finetuning. + targets_mlp_proj = any( + name in target_modules + for name in ("gate_proj", "up_proj", "down_proj", "gate_up_proj") + ) + if is_regex and targets_mlp_proj: return set(_MOE_BROAD_MLP_TARGETS) return set() From c36e674aa9a904781ec32fa007815ad03eeb4ad4 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 3 Jul 2026 18:42:09 +0000 Subject: [PATCH 2/9] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- unsloth/models/_utils.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py index b961a7ede6..5d8d54bba5 100644 --- a/unsloth/models/_utils.py +++ b/unsloth/models/_utils.py @@ -3511,8 +3511,7 @@ def _moe_target_set_from_string(target_modules: str) -> set[str]: # q/k/v/o regex lists mlp|ffn as path segments, so a bare "mlp" substring # check silently enabled expert LoRA for attention-only finetuning. targets_mlp_proj = any( - name in target_modules - for name in ("gate_proj", "up_proj", "down_proj", "gate_up_proj") + name in target_modules for name in ("gate_proj", "up_proj", "down_proj", "gate_up_proj") ) if is_regex and targets_mlp_proj: return set(_MOE_BROAD_MLP_TARGETS) From 0f60ede9ed0c4011afcc2f4739c523c29e86f4bb Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sat, 4 Jul 2026 12:57:31 +0000 Subject: [PATCH 3/9] Tighten comments --- unsloth/models/_utils.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py index 5d8d54bba5..de03a66043 100644 --- a/unsloth/models/_utils.py +++ b/unsloth/models/_utils.py @@ -3507,9 +3507,8 @@ def _moe_target_set_from_string(target_modules: str) -> set[str]: return {target_modules} is_regex = re.search(r"[*+?()[\]{}|\\^$]", target_modules) is not None - # Key expert-LoRA detection on the projection names themselves: the auto - # q/k/v/o regex lists mlp|ffn as path segments, so a bare "mlp" substring - # check silently enabled expert LoRA for attention-only finetuning. + # Match projection names directly: the auto q/k/v/o regex lists mlp|ffn as path + # segments, so a bare "mlp" substring wrongly enabled expert LoRA for attn-only tuning. targets_mlp_proj = any( name in target_modules for name in ("gate_proj", "up_proj", "down_proj", "gate_up_proj") ) From c6a611a2869ba67b90244c1f76c7c37a25deef9d Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sat, 4 Jul 2026 13:14:57 +0000 Subject: [PATCH 4/9] Detect MoE expert LoRA via mlp path segment, not proj names The auto-generated target regex always lists every projection leaf (q/k/v/o and gate/up/down), so keying detection on a proj name mis-fired: it enabled expert LoRA for attention-only regexes and dropped the mlp/ffn path regexes. Key on the mlp/ffn/feed_forward/experts path segment instead, which is present only when the MLP/experts are actually targeted. Add a regression test for the attention-only case. --- tests/test_moe_lora_targets.py | 14 ++++++++++++++ unsloth/models/_utils.py | 11 ++++++----- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/tests/test_moe_lora_targets.py b/tests/test_moe_lora_targets.py index 994d39f261..470cf8daf2 100644 --- a/tests/test_moe_lora_targets.py +++ b/tests/test_moe_lora_targets.py @@ -49,3 +49,17 @@ def test_explicit_dotted_module_target_does_not_discover_moe_parameters(): ) is None ) + + +@pytest.mark.parametrize( + "target_modules", + [ + # Attention-only auto-regex lists every projection leaf (incl. gate/up/down) + # but its path segment is attention-only, so experts must NOT be targeted. + r"(?:\bmodel\.layers\.[\d]{1,}\.(?:self_attn|attention|attn|mixer)\.(?:q_proj|k_proj|v_proj|o_proj|gate_proj|up_proj|down_proj))", + ".*self_attn.*proj", + ], +) +def test_attention_only_regex_does_not_discover_moe_parameters(target_modules): + from unsloth.models._utils import get_moe_target_parameters + assert get_moe_target_parameters(_FakeMoeModel(), target_modules) is None diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py index de03a66043..cf050cdbf3 100644 --- a/unsloth/models/_utils.py +++ b/unsloth/models/_utils.py @@ -3507,12 +3507,13 @@ def _moe_target_set_from_string(target_modules: str) -> set[str]: return {target_modules} is_regex = re.search(r"[*+?()[\]{}|\\^$]", target_modules) is not None - # Match projection names directly: the auto q/k/v/o regex lists mlp|ffn as path - # segments, so a bare "mlp" substring wrongly enabled expert LoRA for attn-only tuning. - targets_mlp_proj = any( - name in target_modules for name in ("gate_proj", "up_proj", "down_proj", "gate_up_proj") + # The auto regex always lists every projection leaf (q/k/v/o and gate/up/down), + # so key detection on the mlp/ffn/experts path segment, absent from an + # attention-only regex, not on the proj names. + targets_mlp_path = any( + tag in target_modules for tag in ("mlp", "ffn", "feed_forward", "experts") ) - if is_regex and targets_mlp_proj: + if is_regex and targets_mlp_path and "proj" in target_modules: return set(_MOE_BROAD_MLP_TARGETS) return set() From cdbb3dcaa4a640d1b4c749725b0ee1c94430188a Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sun, 5 Jul 2026 01:49:21 +0000 Subject: [PATCH 5/9] Scope expert LoRA targets to the leaves a regex names An mlp path alternative with attention-only leaves, for example (mlp|self_attn).(q_proj|o_proj), no longer enables expert LoRA, and a regex naming a single expert leaf such as .*experts.*down_proj now targets only that projection instead of the whole broad set. Generic mlp projections (.*mlp.*proj) and the auto regex mlp tag block keep the broad set for fused-expert models whose leaves are plain Parameters. --- tests/test_moe_lora_targets.py | 27 +++++++++++++++++++++++++++ unsloth/models/_utils.py | 20 ++++++++++++++++---- 2 files changed, 43 insertions(+), 4 deletions(-) diff --git a/tests/test_moe_lora_targets.py b/tests/test_moe_lora_targets.py index 470cf8daf2..ff318629b0 100644 --- a/tests/test_moe_lora_targets.py +++ b/tests/test_moe_lora_targets.py @@ -58,8 +58,35 @@ def test_explicit_dotted_module_target_does_not_discover_moe_parameters(): # but its path segment is attention-only, so experts must NOT be targeted. r"(?:\bmodel\.layers\.[\d]{1,}\.(?:self_attn|attention|attn|mixer)\.(?:q_proj|k_proj|v_proj|o_proj|gate_proj|up_proj|down_proj))", ".*self_attn.*proj", + # An mlp path alternative with attention-only leaves is still attention-only. + r"model\.layers\.\d+\.(?:mlp|self_attn)\.(?:q_proj|k_proj|v_proj|o_proj)", ], ) def test_attention_only_regex_does_not_discover_moe_parameters(target_modules): from unsloth.models._utils import get_moe_target_parameters assert get_moe_target_parameters(_FakeMoeModel(), target_modules) is None + + +def test_single_leaf_regex_targets_only_that_projection(): + from unsloth.models._utils import get_moe_target_parameters + assert get_moe_target_parameters(_FakeMoeModel(), ".*experts.*down_proj") == [ + "mlp.experts.down_proj", + ] + assert get_moe_target_parameters(_FakeMoeModel(), ".*mlp.*gate_proj") == [ + "mlp.experts.gate_up_proj", + ] + + +def test_auto_regex_mlp_tag_block_discovers_moe_on_fused_models(): + # get_peft_regex on a fused-expert model lists only attention Linears as + # leaves; the mlp tag block is the remaining signal of MLP finetune intent. + from unsloth.models._utils import get_moe_target_parameters + both_auto = ( + r"(?:\bmodel\.layers\.[\d]{1,}\." + r"(?:self_attn|attention|attn|mixer|mlp|feed_forward|ffn|dense|mixer)\." + r"(?:(?:q_proj|k_proj|v_proj|o_proj)))" + ) + assert get_moe_target_parameters(_FakeMoeModel(), both_auto) == [ + "mlp.experts.gate_up_proj", + "mlp.experts.down_proj", + ] diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py index cf050cdbf3..c95ea80031 100644 --- a/unsloth/models/_utils.py +++ b/unsloth/models/_utils.py @@ -3507,13 +3507,25 @@ def _moe_target_set_from_string(target_modules: str) -> set[str]: return {target_modules} is_regex = re.search(r"[*+?()[\]{}|\\^$]", target_modules) is not None - # The auto regex always lists every projection leaf (q/k/v/o and gate/up/down), - # so key detection on the mlp/ffn/experts path segment, absent from an - # attention-only regex, not on the proj names. + # Key detection on the mlp/ffn/experts path segment (absent from an + # attention-only regex), never on q/k/v/o leaves alone. targets_mlp_path = any( tag in target_modules for tag in ("mlp", "ffn", "feed_forward", "experts") ) - if is_regex and targets_mlp_path and "proj" in target_modules: + if not is_regex or not targets_mlp_path: + return set() + # Explicit expert leaves scope the target set to exactly those leaves. + named = {name for name in _MOE_BROAD_MLP_TARGETS if name in target_modules} + if named: + return named + # A generic projection under an mlp path (e.g. ".*mlp.*proj"): any proj + # occurrence that is not an attention leaf name. + if re.search(r"(? Date: Mon, 6 Jul 2026 07:53:13 +0000 Subject: [PATCH 6/9] Route explicit leaf list into MoE expert detection An attention-only explicit target_modules list routed through get_peft_regex for family scoping (e.g. FastVisionModel with vision layers off) yields a regex carrying the full mlp|feed_forward|ffn|dense component block even though its leaf group only names q/k/v/o_proj. Keying expert detection on that regex trained the experts for a language-only/attention-only request. Use the caller's original leaf list for detection; only the auto path uses the regex, where the mlp block is the sole MLP-intent signal on fused-expert models. --- tests/test_moe_lora_targets.py | 26 ++++++++++++++++++++++++++ unsloth/models/vision.py | 17 +++++++++++++++-- 2 files changed, 41 insertions(+), 2 deletions(-) diff --git a/tests/test_moe_lora_targets.py b/tests/test_moe_lora_targets.py index ff318629b0..911d288b0c 100644 --- a/tests/test_moe_lora_targets.py +++ b/tests/test_moe_lora_targets.py @@ -90,3 +90,29 @@ def test_auto_regex_mlp_tag_block_discovers_moe_on_fused_models(): "mlp.experts.gate_up_proj", "mlp.experts.down_proj", ] + + +def test_explicit_attention_only_list_does_not_discover_moe_parameters(): + # An explicit attention-only leaf list names no MLP projection, so experts + # must never be targeted. get_peft_model routes this ORIGINAL list (not the + # scoped regex) into detection precisely because family scoping makes + # get_peft_regex emit its full "mlp|feed_forward|ffn|dense" component block + # even for an attention-only request (see the regex below), which the + # string fallback cannot distinguish from the fused-expert auto regex. + from unsloth.models._utils import get_moe_target_parameters + attn_only_list = ["q_proj", "k_proj", "v_proj", "o_proj"] + assert get_moe_target_parameters(_FakeMoeModel(), attn_only_list) is None + assert get_moe_target_parameters(_FakeMoeModel(), tuple(attn_only_list)) is None + + # The regex get_peft_regex emits for that same attention-only list under a + # vision-off family scope carries the mlp component block, so the string + # path would wrongly enable experts -- hence detection must use the list. + scoped_regex = ( + r"(?:.*?(?:language|text).*?" + r"(?:self_attn|attention|attn|mixer|mlp|feed_forward|ffn|dense|mixer).*?" + r"(?:q_proj|k_proj|v_proj|o_proj))" + ) + assert get_moe_target_parameters(_FakeMoeModel(), scoped_regex) == [ + "mlp.experts.gate_up_proj", + "mlp.experts.down_proj", + ] diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py index 5ab55152db..0b21755297 100644 --- a/unsloth/models/vision.py +++ b/unsloth/models/vision.py @@ -1629,6 +1629,16 @@ def get_peft_model( ) else: _audio_kwargs = {} + # Remember the caller's ORIGINAL explicit leaf list for MoE expert + # detection. When an explicit list is routed through get_peft_regex for + # family scoping below, the generated regex carries get_peft_regex's full + # "mlp|feed_forward|ffn|dense" component block even when the caller named + # only attention leaves (q/k/v/o_proj). Keying expert detection on that + # regex would train the experts for an attention-only request. The + # original list carries the true leaf intent, so use it for MoE detection; + # only the auto (None / "all-linear") path relies on the regex, whose mlp + # block is the sole remaining MLP-intent signal on fused-expert models. + _moe_detect_target = target_modules if type(target_modules) in (list, tuple) else None if target_modules is None or target_modules == "all-linear": target_modules = get_peft_regex( model, @@ -1706,9 +1716,12 @@ def get_peft_model( loftq_config, lora_dropout, bias, init_lora_weights, model ) - # Auto-detect MoE models and populate target_parameters for expert layers + # Auto-detect MoE models and populate target_parameters for expert layers. + # For an explicit leaf list use the ORIGINAL list, not the scoped regex, so + # attention-only requests do not train experts via get_peft_regex's mlp block. if target_parameters is None: - target_parameters = get_moe_target_parameters(model, target_modules) + _moe_targets = _moe_detect_target if _moe_detect_target is not None else target_modules + target_parameters = get_moe_target_parameters(model, _moe_targets) if finetune_last_n_layers is not None and layers_to_transform is None: _total_layers = _get_total_transformer_layers(model) From 9d98b36ca9e058ca59852cc46735aacb2f009f04 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 6 Jul 2026 07:54:01 +0000 Subject: [PATCH 7/9] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/test_moe_lora_targets.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_moe_lora_targets.py b/tests/test_moe_lora_targets.py index 911d288b0c..b42e020d6a 100644 --- a/tests/test_moe_lora_targets.py +++ b/tests/test_moe_lora_targets.py @@ -100,6 +100,7 @@ def test_explicit_attention_only_list_does_not_discover_moe_parameters(): # even for an attention-only request (see the regex below), which the # string fallback cannot distinguish from the fused-expert auto regex. from unsloth.models._utils import get_moe_target_parameters + attn_only_list = ["q_proj", "k_proj", "v_proj", "o_proj"] assert get_moe_target_parameters(_FakeMoeModel(), attn_only_list) is None assert get_moe_target_parameters(_FakeMoeModel(), tuple(attn_only_list)) is None From 787d3b999144d8a0045391e3b7a16faa5db28486 Mon Sep 17 00:00:00 2001 From: danielhanchen Date: Mon, 6 Jul 2026 08:13:13 +0000 Subject: [PATCH 8/9] Respect finetune_mlp_modules and finetune_language_layers scope for MoE expert detection When an explicit leaf list that names MLP projections (gate_proj/up_proj/down_proj) is routed through get_peft_regex under finetune_mlp_modules=False, the scoped regex correctly drops the MLP leaves, but MoE expert detection was still keyed on the original list and re-added mlp.experts.* via target_parameters, training the experts the caller had frozen. Same gap for finetune_language_layers=False on vision-only runs. Prefer the original list only when MLP and language families are both in scope (preserving the attention-only fix); otherwise honor the scoped result so the frozen family is respected. Factored the choice into _select_moe_detection_targets with unit tests over the full selection matrix. --- tests/test_moe_lora_targets.py | 109 +++++++++++++++++++++++++++++++++ unsloth/models/_utils.py | 30 +++++++++ unsloth/models/vision.py | 16 ++++- 3 files changed, 152 insertions(+), 3 deletions(-) diff --git a/tests/test_moe_lora_targets.py b/tests/test_moe_lora_targets.py index b42e020d6a..087047ffe7 100644 --- a/tests/test_moe_lora_targets.py +++ b/tests/test_moe_lora_targets.py @@ -117,3 +117,112 @@ def test_explicit_attention_only_list_does_not_discover_moe_parameters(): "mlp.experts.gate_up_proj", "mlp.experts.down_proj", ] + + +def test_frozen_mlp_full_list_does_not_discover_moe_parameters(): + # Regression: an explicit list that names MLP leaves together with + # finetune_mlp_modules=False must NOT train experts. get_peft_regex scopes + # the MLP leaves out (its emitted regex carries no mlp tag block), so + # detection has to key on that SCOPED regex -- keying on the original list + # would let its gate/up/down leaves silently re-enable the frozen experts. + from unsloth.models._utils import ( + _select_moe_detection_targets, + get_moe_target_parameters, + ) + + original_list = [ + "q_proj", "k_proj", "v_proj", "o_proj", + "gate_proj", "up_proj", "down_proj", + ] + # Representative of what get_peft_regex emits for that list under + # finetune_mlp_modules=False: attention-only path, no mlp component block. + scoped_regex = ( + r"(?:.*?(?:language|text).*?" + r"(?:self_attn|attention|attn|mixer).*?" + r"(?:q_proj|k_proj|v_proj|o_proj))" + ) + selected = _select_moe_detection_targets( + original_list, + scoped_regex, + finetune_mlp_modules = False, + finetune_language_layers = True, + ) + assert selected is scoped_regex + assert get_moe_target_parameters(_FakeMoeModel(), selected) is None + + +def test_frozen_language_full_list_does_not_discover_moe_parameters(): + # Vision-only request (finetune_language_layers=False) with a full leaf list + # must not reach the language-model experts either. + from unsloth.models._utils import ( + _select_moe_detection_targets, + get_moe_target_parameters, + ) + + original_list = ["q_proj", "gate_proj", "up_proj", "down_proj"] + scoped_regex = ( + r"(?:.*?(?:vision|visual|image).*?" + r"(?:self_attn|attention|attn|mixer).*?" + r"(?:q_proj|k_proj|v_proj|o_proj))" + ) + selected = _select_moe_detection_targets( + original_list, + scoped_regex, + finetune_mlp_modules = True, + finetune_language_layers = False, + ) + assert selected is scoped_regex + assert get_moe_target_parameters(_FakeMoeModel(), selected) is None + + +def test_in_scope_mlp_full_list_still_discovers_moe_parameters(): + # With MLP and language both in scope, an explicit list that names MLP + # leaves SHOULD enable the experts (unchanged behavior): the original list + # is preferred and carries the gate/up/down intent. + from unsloth.models._utils import ( + _select_moe_detection_targets, + get_moe_target_parameters, + ) + + original_list = [ + "q_proj", "k_proj", "v_proj", "o_proj", + "gate_proj", "up_proj", "down_proj", + ] + scoped_regex = r".*self_attn.*proj" # unused: original list is preferred + selected = _select_moe_detection_targets( + original_list, + scoped_regex, + finetune_mlp_modules = True, + finetune_language_layers = True, + ) + assert selected is original_list + assert get_moe_target_parameters(_FakeMoeModel(), selected) == [ + "mlp.experts.gate_up_proj", + "mlp.experts.down_proj", + ] + + +def test_attention_only_list_prefers_original_when_in_scope(): + # The case the PR originally fixed: an attention-only list routed through + # get_peft_regex under a family scope (e.g. vision-off) still keeps experts + # off, because with MLP+language in scope detection uses the original + # attention-only list rather than the regex's spurious mlp component block. + from unsloth.models._utils import ( + _select_moe_detection_targets, + get_moe_target_parameters, + ) + + attn_only_list = ["q_proj", "k_proj", "v_proj", "o_proj"] + scoped_regex = ( # carries the spurious mlp block get_peft_regex always adds + r"(?:.*?(?:language|text).*?" + r"(?:self_attn|attention|attn|mixer|mlp|feed_forward|ffn|dense).*?" + r"(?:q_proj|k_proj|v_proj|o_proj))" + ) + selected = _select_moe_detection_targets( + attn_only_list, + scoped_regex, + finetune_mlp_modules = True, + finetune_language_layers = True, + ) + assert selected is attn_only_list + assert get_moe_target_parameters(_FakeMoeModel(), selected) is None diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py index c95ea80031..b076ba11f5 100644 --- a/unsloth/models/_utils.py +++ b/unsloth/models/_utils.py @@ -85,6 +85,7 @@ "hf_login", "is_moe_model", "get_moe_target_parameters", + "_select_moe_detection_targets", "make_fast_generate_wrapper", "_mark_unsloth_disable_data_parallel", "_patch_transformers_trainer_data_parallel", @@ -3611,6 +3612,35 @@ def get_moe_target_parameters(model, target_modules = None) -> Optional[List[str return None +def _select_moe_detection_targets( + original_target_modules, + scoped_target_modules, + finetune_mlp_modules = True, + finetune_language_layers = True, +): + """Pick what get_moe_target_parameters keys expert detection on. + + Prefer the caller's ORIGINAL explicit leaf list over the scoped regex so an + attention-only request is not pushed into the experts by get_peft_regex's + ``mlp|feed_forward|ffn|dense`` component block (which the string fallback + cannot tell apart from a fused-expert auto regex). + + But only when the MLP and language families are BOTH still in scope. If the + caller scoped MLP or language OFF (``finetune_mlp_modules=False`` or + ``finetune_language_layers=False``) the scoped regex already drops the MoE + experts, and reusing the original list -- which may still name gate/up/down + leaves -- would wrongly re-introduce them. In that case honor the scoped + result so the frozen-MLP / vision-only request is respected. + """ + if ( + original_target_modules is not None + and finetune_mlp_modules + and finetune_language_layers + ): + return original_target_modules + return scoped_target_modules + + def make_fast_generate_wrapper(original_generate): """ Creates a wrapper around model.generate that checks for incorrect diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py index 0b21755297..f1fa6fe188 100644 --- a/unsloth/models/vision.py +++ b/unsloth/models/vision.py @@ -37,6 +37,7 @@ _get_text_only_config, _is_family_text_decoder, _apply_text_only_key_mapping, + _select_moe_detection_targets, set_task_config_attr, ) from ._utils import * @@ -1717,10 +1718,19 @@ def get_peft_model( ) # Auto-detect MoE models and populate target_parameters for expert layers. - # For an explicit leaf list use the ORIGINAL list, not the scoped regex, so - # attention-only requests do not train experts via get_peft_regex's mlp block. + # Prefer the caller's ORIGINAL explicit leaf list over the scoped regex so an + # attention-only request does not train experts via get_peft_regex's mlp block, + # but only when MLP and language families are both still in scope. If the caller + # scoped MLP or language OFF (finetune_mlp_modules / finetune_language_layers + # False), the scoped regex already dropped the experts, so honor it instead of + # re-introducing the original list's gate/up/down leaves. if target_parameters is None: - _moe_targets = _moe_detect_target if _moe_detect_target is not None else target_modules + _moe_targets = _select_moe_detection_targets( + _moe_detect_target, + target_modules, + finetune_mlp_modules = finetune_mlp_modules, + finetune_language_layers = finetune_language_layers, + ) target_parameters = get_moe_target_parameters(model, _moe_targets) if finetune_last_n_layers is not None and layers_to_transform is None: From c8d58b5f09e4097d43ad383abc1f23ca3fdd8573 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 6 Jul 2026 08:13:49 +0000 Subject: [PATCH 9/9] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/test_moe_lora_targets.py | 18 ++++++++++++++---- unsloth/models/_utils.py | 6 +----- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/tests/test_moe_lora_targets.py b/tests/test_moe_lora_targets.py index 087047ffe7..7f9b9a0485 100644 --- a/tests/test_moe_lora_targets.py +++ b/tests/test_moe_lora_targets.py @@ -131,8 +131,13 @@ def test_frozen_mlp_full_list_does_not_discover_moe_parameters(): ) original_list = [ - "q_proj", "k_proj", "v_proj", "o_proj", - "gate_proj", "up_proj", "down_proj", + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj", ] # Representative of what get_peft_regex emits for that list under # finetune_mlp_modules=False: attention-only path, no mlp component block. @@ -185,8 +190,13 @@ def test_in_scope_mlp_full_list_still_discovers_moe_parameters(): ) original_list = [ - "q_proj", "k_proj", "v_proj", "o_proj", - "gate_proj", "up_proj", "down_proj", + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj", ] scoped_regex = r".*self_attn.*proj" # unused: original list is preferred selected = _select_moe_detection_targets( diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py index b076ba11f5..c8b85fd44b 100644 --- a/unsloth/models/_utils.py +++ b/unsloth/models/_utils.py @@ -3632,11 +3632,7 @@ def _select_moe_detection_targets( leaves -- would wrongly re-introduce them. In that case honor the scoped result so the frozen-MLP / vision-only request is respected. """ - if ( - original_target_modules is not None - and finetune_mlp_modules - and finetune_language_layers - ): + if original_target_modules is not None and finetune_mlp_modules and finetune_language_layers: return original_target_modules return scoped_target_modules