Scope MoE expert LoRA detection to actual MLP projection targets

tests/test_moe_lora_targets.py

@@ -49,3 +49,44 @@ def test_explicit_dotted_module_target_does_not_discover_moe_parameters():
         )
         is None
     )
+
+
+@pytest.mark.parametrize(
+    "target_modules",
+    [
+        # Attention-only auto-regex lists every projection leaf (incl. gate/up/down)
+        # but its path segment is attention-only, so experts must NOT be targeted.
+        r"(?:\bmodel\.layers\.[\d]{1,}\.(?:self_attn|attention|attn|mixer)\.(?:q_proj|k_proj|v_proj|o_proj|gate_proj|up_proj|down_proj))",
+        ".*self_attn.*proj",
+        # An mlp path alternative with attention-only leaves is still attention-only.
+        r"model\.layers\.\d+\.(?:mlp|self_attn)\.(?:q_proj|k_proj|v_proj|o_proj)",
+    ],
+)
+def test_attention_only_regex_does_not_discover_moe_parameters(target_modules):
+    from unsloth.models._utils import get_moe_target_parameters
+    assert get_moe_target_parameters(_FakeMoeModel(), target_modules) is None
+
+
+def test_single_leaf_regex_targets_only_that_projection():
+    from unsloth.models._utils import get_moe_target_parameters
+    assert get_moe_target_parameters(_FakeMoeModel(), ".*experts.*down_proj") == [
+        "mlp.experts.down_proj",
+    ]
+    assert get_moe_target_parameters(_FakeMoeModel(), ".*mlp.*gate_proj") == [
+        "mlp.experts.gate_up_proj",
+    ]
+
+
+def test_auto_regex_mlp_tag_block_discovers_moe_on_fused_models():
+    # get_peft_regex on a fused-expert model lists only attention Linears as
+    # leaves; the mlp tag block is the remaining signal of MLP finetune intent.
+    from unsloth.models._utils import get_moe_target_parameters
+    both_auto = (
+        r"(?:\bmodel\.layers\.[\d]{1,}\."
+        r"(?:self_attn|attention|attn|mixer|mlp|feed_forward|ffn|dense|mixer)\."
+        r"(?:(?:q_proj|k_proj|v_proj|o_proj)))"
+    )
+    assert get_moe_target_parameters(_FakeMoeModel(), both_auto) == [
+        "mlp.experts.gate_up_proj",
+        "mlp.experts.down_proj",
+    ]

unsloth/models/_utils.py

@@ -3507,8 +3507,25 @@ def _moe_target_set_from_string(target_modules: str) -> set[str]:
         return {target_modules}
 
     is_regex = re.search(r"[*+?()[\]{}|\\^$]", target_modules) is not None
-    targets_mlp = "mlp" in target_modules or "ffn" in target_modules
-    if is_regex and "proj" in target_modules and targets_mlp:
+    # Key detection on the mlp/ffn/experts path segment (absent from an
+    # attention-only regex), never on q/k/v/o leaves alone.
+    targets_mlp_path = any(
+        tag in target_modules for tag in ("mlp", "ffn", "feed_forward", "experts")
+    )
+    if not is_regex or not targets_mlp_path:
+        return set()
+    # Explicit expert leaves scope the target set to exactly those leaves.
+    named = {name for name in _MOE_BROAD_MLP_TARGETS if name in target_modules}
+    if named:
+        return named
+    # A generic projection under an mlp path (e.g. ".*mlp.*proj"): any proj
+    # occurrence that is not an attention leaf name.
+    if re.search(r"(?<![qkvo]_)(?<!out_)(?<!in_)proj", target_modules):
+        return set(_MOE_BROAD_MLP_TARGETS)
+    # The auto regex on fused-expert models lists only attention Linears as
+    # leaves; its mlp tag block is the remaining MLP-intent signal. A regex
+    # like "(mlp|self_attn).(q_proj|o_proj)" has neither and stays attention-only.
+    if "mlp|feed_forward|ffn|dense" in target_modules:
         return set(_MOE_BROAD_MLP_TARGETS)
 
     return set()