Add unsloth eval CLI command

pyproject.toml

@@ -106,6 +106,10 @@ windows = [
 base = [
     "unsloth[huggingface]",
 ]
+eval = [
+    "unsloth[huggingface]",
+    "lm_eval>=0.4.4",
+]
 cu118only = [
     "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.22.post7%2Bcu118-cp39-cp39-manylinux2014_x86_64.whl ; python_version=='3.9' and ('linux' in sys_platform)",
     "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.22.post7%2Bcu118-cp310-cp310-manylinux2014_x86_64.whl ; python_version=='3.10' and ('linux' in sys_platform)",

studio/install_python_stack.py

@@ -1922,6 +1922,26 @@ def pip_install(
             temp_req.unlink(missing_ok = True)
 
 
+def _install_eval_extra(*, package_name: str, local_repo: str) -> None:
+    """Install pyproject.toml's [eval] extra (lm-eval-harness for `unsloth eval`)."""
+    if NO_TORCH or package_name != "unsloth":
+        return
+    _progress("eval extra")
+    if local_repo:
+        pip_install(
+            "Installing unsloth[eval] extra",
+            "--no-cache-dir",
+            "-e",
+            f"{local_repo}[eval]",
+        )
+    else:
+        pip_install(
+            "Installing unsloth[eval] extra",
+            "--no-cache-dir",
+            "unsloth[eval]",
+        )
+
+
 def download_file(url: str, dest: Path) -> None:
     """Download a file using urllib (no curl dependency)."""
     urllib.request.urlretrieve(url, dest)
@@ -1977,6 +1997,8 @@ def install_python_stack() -> int:
         base_total += 1  # ROCm torch check (line 1526) -- all non-macOS platforms
         if not IS_WINDOWS:
             base_total += 2  # flash-attn (line 1620) + ROCm torch final (line 1705) -- Linux only
+    if not NO_TORCH and package_name == "unsloth":
+        base_total += 1  # pyproject.toml [eval] extra
     _TOTAL = (base_total - 1) if skip_base else base_total
 
     # 1. Try uv for faster installs (before pip upgrade -- uv venvs don't
@@ -2197,6 +2219,8 @@ def _win_amd_smi_has_gpu(stdout: str) -> bool:
                 "Manual install may be required. See: https://docs.unsloth.ai/get-started/install-and-update/amd",
             )
 
+    _install_eval_extra(package_name = package_name, local_repo = local_repo)
+
     # 3. Extra dependencies
     _progress("unsloth extras")
     pip_install(

unsloth_cli/__init__.py

@@ -13,6 +13,7 @@
 from unsloth_cli.commands.chat import chat
 from unsloth_cli.commands.connect import connect_app
 from unsloth_cli.commands.export import export, list_checkpoints
+from unsloth_cli.commands.eval import evaluate as eval_command
 from unsloth_cli.commands.studio import (
     run as studio_run,
     studio_app,
@@ -76,6 +77,7 @@ def main(
 app.command()(inference)
 app.command()(chat)
 app.command()(export)
+app.command("eval")(eval_command)
 app.command("list-checkpoints")(list_checkpoints)
 app.add_typer(studio_app, name = "studio", help = "Unsloth Studio commands.")
 app.add_typer(