Scope MoE expert LoRA detection to actual MLP projection targets

tests/test_moe_lora_targets.py

@@ -49,3 +49,71 @@ def test_explicit_dotted_module_target_does_not_discover_moe_parameters():
         )
         is None
     )
+
+
+@pytest.mark.parametrize(
+    "target_modules",
+    [
+        # Attention-only auto-regex lists every projection leaf (incl. gate/up/down)
+        # but its path segment is attention-only, so experts must NOT be targeted.
+        r"(?:\bmodel\.layers\.[\d]{1,}\.(?:self_attn|attention|attn|mixer)\.(?:q_proj|k_proj|v_proj|o_proj|gate_proj|up_proj|down_proj))",
+        ".*self_attn.*proj",
+        # An mlp path alternative with attention-only leaves is still attention-only.
+        r"model\.layers\.\d+\.(?:mlp|self_attn)\.(?:q_proj|k_proj|v_proj|o_proj)",
+    ],
+)
+def test_attention_only_regex_does_not_discover_moe_parameters(target_modules):
+    from unsloth.models._utils import get_moe_target_parameters
+    assert get_moe_target_parameters(_FakeMoeModel(), target_modules) is None
+
+
+def test_single_leaf_regex_targets_only_that_projection():
+    from unsloth.models._utils import get_moe_target_parameters
+    assert get_moe_target_parameters(_FakeMoeModel(), ".*experts.*down_proj") == [
+        "mlp.experts.down_proj",
+    ]
+    assert get_moe_target_parameters(_FakeMoeModel(), ".*mlp.*gate_proj") == [
+        "mlp.experts.gate_up_proj",
+    ]
+
+
+def test_auto_regex_mlp_tag_block_discovers_moe_on_fused_models():
+    # get_peft_regex on a fused-expert model lists only attention Linears as
+    # leaves; the mlp tag block is the remaining signal of MLP finetune intent.
+    from unsloth.models._utils import get_moe_target_parameters
+    both_auto = (
+        r"(?:\bmodel\.layers\.[\d]{1,}\."
+        r"(?:self_attn|attention|attn|mixer|mlp|feed_forward|ffn|dense|mixer)\."
+        r"(?:(?:q_proj|k_proj|v_proj|o_proj)))"
+    )
+    assert get_moe_target_parameters(_FakeMoeModel(), both_auto) == [
+        "mlp.experts.gate_up_proj",
+        "mlp.experts.down_proj",
+    ]
+
+
+def test_explicit_attention_only_list_does_not_discover_moe_parameters():
+    # An explicit attention-only leaf list names no MLP projection, so experts
+    # must never be targeted. get_peft_model routes this ORIGINAL list (not the
+    # scoped regex) into detection precisely because family scoping makes
+    # get_peft_regex emit its full "mlp|feed_forward|ffn|dense" component block
+    # even for an attention-only request (see the regex below), which the
+    # string fallback cannot distinguish from the fused-expert auto regex.
+    from unsloth.models._utils import get_moe_target_parameters
+
+    attn_only_list = ["q_proj", "k_proj", "v_proj", "o_proj"]
+    assert get_moe_target_parameters(_FakeMoeModel(), attn_only_list) is None
+    assert get_moe_target_parameters(_FakeMoeModel(), tuple(attn_only_list)) is None
+
+    # The regex get_peft_regex emits for that same attention-only list under a
+    # vision-off family scope carries the mlp component block, so the string
+    # path would wrongly enable experts -- hence detection must use the list.
+    scoped_regex = (
+        r"(?:.*?(?:language|text).*?"
+        r"(?:self_attn|attention|attn|mixer|mlp|feed_forward|ffn|dense|mixer).*?"
+        r"(?:q_proj|k_proj|v_proj|o_proj))"
+    )
+    assert get_moe_target_parameters(_FakeMoeModel(), scoped_regex) == [
+        "mlp.experts.gate_up_proj",
+        "mlp.experts.down_proj",
+    ]

unsloth/models/_utils.py

@@ -3507,8 +3507,25 @@ def _moe_target_set_from_string(target_modules: str) -> set[str]:
         return {target_modules}
 
     is_regex = re.search(r"[*+?()[\]{}|\\^$]", target_modules) is not None
-    targets_mlp = "mlp" in target_modules or "ffn" in target_modules
-    if is_regex and "proj" in target_modules and targets_mlp:
+    # Key detection on the mlp/ffn/experts path segment (absent from an
+    # attention-only regex), never on q/k/v/o leaves alone.
+    targets_mlp_path = any(
+        tag in target_modules for tag in ("mlp", "ffn", "feed_forward", "experts")
+    )
+    if not is_regex or not targets_mlp_path:
+        return set()
+    # Explicit expert leaves scope the target set to exactly those leaves.
+    named = {name for name in _MOE_BROAD_MLP_TARGETS if name in target_modules}
+    if named:
+        return named
+    # A generic projection under an mlp path (e.g. ".*mlp.*proj"): any proj
+    # occurrence that is not an attention leaf name.
+    if re.search(r"(?<![qkvo]_)(?<!out_)(?<!in_)proj", target_modules):
+        return set(_MOE_BROAD_MLP_TARGETS)
+    # The auto regex on fused-expert models lists only attention Linears as
+    # leaves; its mlp tag block is the remaining MLP-intent signal. A regex
+    # like "(mlp|self_attn).(q_proj|o_proj)" has neither and stays attention-only.
+    if "mlp|feed_forward|ffn|dense" in target_modules:
         return set(_MOE_BROAD_MLP_TARGETS)
 
     return set()

unsloth/models/vision.py

@@ -1629,6 +1629,16 @@ def get_peft_model(
             )
         else:
             _audio_kwargs = {}
+        # Remember the caller's ORIGINAL explicit leaf list for MoE expert
+        # detection. When an explicit list is routed through get_peft_regex for
+        # family scoping below, the generated regex carries get_peft_regex's full
+        # "mlp|feed_forward|ffn|dense" component block even when the caller named
+        # only attention leaves (q/k/v/o_proj). Keying expert detection on that
+        # regex would train the experts for an attention-only request. The
+        # original list carries the true leaf intent, so use it for MoE detection;
+        # only the auto (None / "all-linear") path relies on the regex, whose mlp
+        # block is the sole remaining MLP-intent signal on fused-expert models.
+        _moe_detect_target = target_modules if type(target_modules) in (list, tuple) else None
         if target_modules is None or target_modules == "all-linear":
             target_modules = get_peft_regex(
                 model,
@@ -1706,9 +1716,12 @@ def get_peft_model(
             loftq_config, lora_dropout, bias, init_lora_weights, model
         )
 
-        # Auto-detect MoE models and populate target_parameters for expert layers
+        # Auto-detect MoE models and populate target_parameters for expert layers.
+        # For an explicit leaf list use the ORIGINAL list, not the scoped regex, so
+        # attention-only requests do not train experts via get_peft_regex's mlp block.
         if target_parameters is None:
-            target_parameters = get_moe_target_parameters(model, target_modules)
+            _moe_targets = _moe_detect_target if _moe_detect_target is not None else target_modules
+            target_parameters = get_moe_target_parameters(model, _moe_targets)
 
         if finetune_last_n_layers is not None and layers_to_transform is None:
             _total_layers = _get_total_transformer_layers(model)