From e0172b5fa2d914e6d388f25c7bf4f9d750542896 Mon Sep 17 00:00:00 2001 From: VectorPeak <73048950+VectorPeak@users.noreply.github.com> Date: Fri, 3 Jul 2026 17:45:10 +0800 Subject: [PATCH 1/4] Fix continuous batching config serialization --- .../generation/configuration_utils.py | 16 ++++++++++ tests/generation/test_configuration_utils.py | 32 ++++++++++++++++++- 2 files changed, 47 insertions(+), 1 deletion(-) diff --git a/src/transformers/generation/configuration_utils.py b/src/transformers/generation/configuration_utils.py index 641201ef384f..af4238dc401f 100644 --- a/src/transformers/generation/configuration_utils.py +++ b/src/transformers/generation/configuration_utils.py @@ -458,6 +458,8 @@ def __init__(self, **kwargs): self.disable_compile = kwargs.pop("disable_compile", None) self.continuous_batching_config = kwargs.pop("continuous_batching_config", None) + if isinstance(self.continuous_batching_config, dict): + self.continuous_batching_config = ContinuousBatchingConfig(**self.continuous_batching_config) # Deprecated (moved to the Hub). TODO remove for v5 self.low_memory = kwargs.pop("low_memory", None) @@ -1798,6 +1800,11 @@ class ContinuousBatchingConfig: max_cached_graphs: int | None = None def __post_init__(self): + if isinstance(self.varlen_compile_config, dict): + self.varlen_compile_config = CompileConfig(**self.varlen_compile_config) + if isinstance(self.decode_compile_config, dict): + self.decode_compile_config = CompileConfig(**self.decode_compile_config) + # Only turn off graph mixing support if TP is on graph_mixing_supported = os.environ.get("NCCL_GRAPH_MIXING_SUPPORT", "1") == "1" distributed = int(os.environ.get("WORLD_SIZE", "1")) > 1 @@ -1836,3 +1843,12 @@ def cuda_graph_booleans(self) -> tuple[bool, bool]: def fallback_max_blocks_per_request(self) -> int: """Fallback if no user-hint is given and decode path is available.""" return 32 + + def to_dict(self) -> dict[str, Any]: + """Serializes this instance to a Python dictionary.""" + output = copy.deepcopy(self.__dict__) + if self.varlen_compile_config is not None: + output["varlen_compile_config"] = self.varlen_compile_config.to_dict() + if self.decode_compile_config is not None: + output["decode_compile_config"] = self.decode_compile_config.to_dict() + return output diff --git a/tests/generation/test_configuration_utils.py b/tests/generation/test_configuration_utils.py index 36ddf4844d54..a06f9b73f829 100644 --- a/tests/generation/test_configuration_utils.py +++ b/tests/generation/test_configuration_utils.py @@ -22,7 +22,14 @@ from huggingface_hub import create_pull_request from parameterized import parameterized -from transformers import AutoConfig, GenerationConfig, WatermarkingConfig, is_torch_available +from transformers import ( + AutoConfig, + CompileConfig, + ContinuousBatchingConfig, + GenerationConfig, + WatermarkingConfig, + is_torch_available, +) from transformers import logging as transformers_logging @@ -755,6 +762,29 @@ def test_serialize_generation_watermarking_config(self): self.assertEqual(watermark.seeding_scheme, seeding_scheme) self.assertEqual(watermark.context_width, context_width) + def test_serialize_generation_continuous_batching_config(self): + """Tests that GenerationConfig serializes ContinuousBatchingConfig parameters.""" + continuous_batching_config = ContinuousBatchingConfig( + block_size=128, + default_compile_level=2, + varlen_compile_config=CompileConfig(dynamic=True), + decode_compile_config=CompileConfig(mode="default"), + ) + generation_config = GenerationConfig(continuous_batching_config=continuous_batching_config) + + with tempfile.TemporaryDirectory("test-generation-config") as tmp_dir: + generation_config.save_pretrained(tmp_dir) + new_config = GenerationConfig.from_pretrained(tmp_dir) + + self.assertIsInstance(new_config.continuous_batching_config, ContinuousBatchingConfig) + self.assertEqual(new_config.continuous_batching_config.block_size, 128) + self.assertEqual(new_config.continuous_batching_config.default_compile_level, 2) + self.assertIsInstance(new_config.continuous_batching_config.varlen_compile_config, CompileConfig) + self.assertTrue(new_config.continuous_batching_config.varlen_compile_config.dynamic) + self.assertNotIn("_compile_all_devices", new_config.continuous_batching_config.varlen_compile_config.to_dict()) + self.assertIsInstance(new_config.continuous_batching_config.decode_compile_config, CompileConfig) + self.assertEqual(new_config.continuous_batching_config.decode_compile_config.mode, "default") + @is_staging_test class ConfigPushToHubTester(unittest.TestCase): From dac3f938745c830df34c291dca0b577b09f727ce Mon Sep 17 00:00:00 2001 From: VectorPeak <73048950+VectorPeak@users.noreply.github.com> Date: Fri, 3 Jul 2026 20:20:34 +0800 Subject: [PATCH 2/4] Use generic dataclass serialization fallback --- src/transformers/generation/configuration_utils.py | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/src/transformers/generation/configuration_utils.py b/src/transformers/generation/configuration_utils.py index af4238dc401f..fccc1a48da64 100644 --- a/src/transformers/generation/configuration_utils.py +++ b/src/transformers/generation/configuration_utils.py @@ -18,7 +18,7 @@ import os from abc import ABC, abstractmethod from collections.abc import Callable -from dataclasses import dataclass, is_dataclass +from dataclasses import dataclass, fields, is_dataclass from typing import TYPE_CHECKING, Any, Optional, Union from .. import __version__ @@ -1211,6 +1211,7 @@ def convert_dataclass_to_dict(obj): # Some of our dataclasses have a custom `to_dict()` method, and we prefer it if hasattr(obj, "to_dict"): return obj.to_dict() + return {field.name: convert_dataclass_to_dict(getattr(obj, field.name)) for field in fields(obj)} else: return obj @@ -1843,12 +1844,3 @@ def cuda_graph_booleans(self) -> tuple[bool, bool]: def fallback_max_blocks_per_request(self) -> int: """Fallback if no user-hint is given and decode path is available.""" return 32 - - def to_dict(self) -> dict[str, Any]: - """Serializes this instance to a Python dictionary.""" - output = copy.deepcopy(self.__dict__) - if self.varlen_compile_config is not None: - output["varlen_compile_config"] = self.varlen_compile_config.to_dict() - if self.decode_compile_config is not None: - output["decode_compile_config"] = self.decode_compile_config.to_dict() - return output From f72cf6da60244f896dc6863def7a5cc541400931 Mon Sep 17 00:00:00 2001 From: VectorPeak <73048950+VectorPeak@users.noreply.github.com> Date: Tue, 7 Jul 2026 11:15:26 +0800 Subject: [PATCH 3/4] Keep continuous batching config separate from GenerationConfig --- .../generation/configuration_utils.py | 8 ++--- tests/generation/test_configuration_utils.py | 33 +++++++++++-------- 2 files changed, 22 insertions(+), 19 deletions(-) diff --git a/src/transformers/generation/configuration_utils.py b/src/transformers/generation/configuration_utils.py index fccc1a48da64..21c3fa442e49 100644 --- a/src/transformers/generation/configuration_utils.py +++ b/src/transformers/generation/configuration_utils.py @@ -18,7 +18,7 @@ import os from abc import ABC, abstractmethod from collections.abc import Callable -from dataclasses import dataclass, fields, is_dataclass +from dataclasses import dataclass, is_dataclass from typing import TYPE_CHECKING, Any, Optional, Union from .. import __version__ @@ -457,9 +457,8 @@ def __init__(self, **kwargs): self.compile_config = kwargs.pop("compile_config", None) self.disable_compile = kwargs.pop("disable_compile", None) - self.continuous_batching_config = kwargs.pop("continuous_batching_config", None) - if isinstance(self.continuous_batching_config, dict): - self.continuous_batching_config = ContinuousBatchingConfig(**self.continuous_batching_config) + # Continuous batching config is runtime/hardware-specific and lives beside `GenerationConfig`. + kwargs.pop("continuous_batching_config", None) # Deprecated (moved to the Hub). TODO remove for v5 self.low_memory = kwargs.pop("low_memory", None) @@ -1211,7 +1210,6 @@ def convert_dataclass_to_dict(obj): # Some of our dataclasses have a custom `to_dict()` method, and we prefer it if hasattr(obj, "to_dict"): return obj.to_dict() - return {field.name: convert_dataclass_to_dict(getattr(obj, field.name)) for field in fields(obj)} else: return obj diff --git a/tests/generation/test_configuration_utils.py b/tests/generation/test_configuration_utils.py index a06f9b73f829..ea1f1fc012d4 100644 --- a/tests/generation/test_configuration_utils.py +++ b/tests/generation/test_configuration_utils.py @@ -18,6 +18,7 @@ import tempfile import unittest import warnings +from dataclasses import asdict from huggingface_hub import create_pull_request from parameterized import parameterized @@ -762,28 +763,32 @@ def test_serialize_generation_watermarking_config(self): self.assertEqual(watermark.seeding_scheme, seeding_scheme) self.assertEqual(watermark.context_width, context_width) - def test_serialize_generation_continuous_batching_config(self): - """Tests that GenerationConfig serializes ContinuousBatchingConfig parameters.""" + def test_continuous_batching_config_from_dict(self): + """Tests that ContinuousBatchingConfig round-trips nested CompileConfig parameters.""" continuous_batching_config = ContinuousBatchingConfig( block_size=128, default_compile_level=2, varlen_compile_config=CompileConfig(dynamic=True), decode_compile_config=CompileConfig(mode="default"), ) + new_config = ContinuousBatchingConfig(**asdict(continuous_batching_config)) + + self.assertEqual(new_config.block_size, 128) + self.assertEqual(new_config.default_compile_level, 2) + self.assertIsInstance(new_config.varlen_compile_config, CompileConfig) + self.assertTrue(new_config.varlen_compile_config.dynamic) + self.assertNotIn("_compile_all_devices", new_config.varlen_compile_config.to_dict()) + self.assertIsInstance(new_config.decode_compile_config, CompileConfig) + self.assertEqual(new_config.decode_compile_config.mode, "default") + + def test_generation_config_does_not_serialize_continuous_batching_config(self): + """Tests that ContinuousBatchingConfig lives beside GenerationConfig.""" + continuous_batching_config = ContinuousBatchingConfig(block_size=128) generation_config = GenerationConfig(continuous_batching_config=continuous_batching_config) - with tempfile.TemporaryDirectory("test-generation-config") as tmp_dir: - generation_config.save_pretrained(tmp_dir) - new_config = GenerationConfig.from_pretrained(tmp_dir) - - self.assertIsInstance(new_config.continuous_batching_config, ContinuousBatchingConfig) - self.assertEqual(new_config.continuous_batching_config.block_size, 128) - self.assertEqual(new_config.continuous_batching_config.default_compile_level, 2) - self.assertIsInstance(new_config.continuous_batching_config.varlen_compile_config, CompileConfig) - self.assertTrue(new_config.continuous_batching_config.varlen_compile_config.dynamic) - self.assertNotIn("_compile_all_devices", new_config.continuous_batching_config.varlen_compile_config.to_dict()) - self.assertIsInstance(new_config.continuous_batching_config.decode_compile_config, CompileConfig) - self.assertEqual(new_config.continuous_batching_config.decode_compile_config.mode, "default") + self.assertFalse(hasattr(generation_config, "continuous_batching_config")) + self.assertNotIn("continuous_batching_config", generation_config.to_dict()) + self.assertNotIn("continuous_batching_config", generation_config.to_json_string()) @is_staging_test From ba89a1c9738bd66f801634ea0a55d12bda7762bc Mon Sep 17 00:00:00 2001 From: VectorPeak <73048950+VectorPeak@users.noreply.github.com> Date: Tue, 7 Jul 2026 11:21:17 +0800 Subject: [PATCH 4/4] Remove GenerationConfig continuous batching fallback --- .../generation/continuous_batching/continuous_api.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/transformers/generation/continuous_batching/continuous_api.py b/src/transformers/generation/continuous_batching/continuous_api.py index 1c405a287a18..882416880345 100644 --- a/src/transformers/generation/continuous_batching/continuous_api.py +++ b/src/transformers/generation/continuous_batching/continuous_api.py @@ -1104,10 +1104,7 @@ def init_continuous_batching( # Retrieve continuous batching config, or create it if none is provided if continuous_batching_config is None: - if isinstance(getattr(gen_config, "continuous_batching_config", None), ContinuousBatchingConfig): - continuous_batching_config = gen_config.continuous_batching_config - else: - continuous_batching_config = ContinuousBatchingConfig() + continuous_batching_config = ContinuousBatchingConfig() # Create and return the manager return ContinuousBatchingManager(