Skip to content

Commit f5a67a4

Browse files
authored
🚨 Pass tp_plan from lm_heads (#47253)
* pass tp_plan from lm_heads * add a small test to avoid regressiosn on TP * world size must divide vocab size * early raise for incompatible vocabs * test our test
1 parent cd7456e commit f5a67a4

4 files changed

Lines changed: 70 additions & 13 deletions

File tree

‎src/transformers/integrations/tensor_parallel.py‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -674,6 +674,10 @@ def shard_tensor(
674674
) -> torch.Tensor:
675675
raise NotImplementedError
676676

677+
def validate_module(self, module: nn.Module, device_mesh, layer_name: str = ""):
678+
"""Raise if the module cannot be sharded with this style on the given mesh."""
679+
pass
680+
677681
def prepare_module_tp(self, module: nn.Module, device_mesh, **kwargs) -> nn.Module:
678682
distribute_module(
679683
module,
@@ -719,6 +723,16 @@ def __init__(self, gather_output: bool = False, **kwargs):
719723
super().__init__(**kwargs)
720724
self.gather_output = gather_output
721725

726+
def validate_module(self, module: nn.Module, device_mesh, layer_name: str = ""):
727+
out_features = getattr(module, "out_features", None)
728+
if self.gather_output and out_features is not None and out_features % device_mesh.size() != 0:
729+
raise ValueError(
730+
f"`{layer_name}` ({type(module).__name__} with out_features={out_features}) is sharded with "
731+
f"'colwise_gather_output', which requires out_features to be divisible by the number of ranks "
732+
f"({device_mesh.size()}) to all-gather equal-size shards. Resize the weight (e.g. "
733+
f"`model.resize_token_embeddings` for LM heads) or override this module's entry in the tp_plan."
734+
)
735+
722736
def _prepare_input_fn(self, mod, inputs, device_mesh):
723737
input_tensor = inputs[0] if inputs else inputs
724738
return all_reduce_backward(input_tensor, device_mesh)
@@ -1512,6 +1526,7 @@ def add_tensor_parallel_hooks_to_module(
15121526
"""
15131527
if current_module_plan is not None:
15141528
tp_layer = ALL_PARALLEL_STYLES[current_module_plan]
1529+
tp_layer.validate_module(module, device_mesh, layer_name)
15151530
try:
15161531
tp_layer.prepare_module_tp(module, device_mesh, config=model.config)
15171532
except NotImplementedError as e:

‎src/transformers/modeling_utils.py‎

Lines changed: 13 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1268,6 +1268,9 @@ class PreTrainedModel(nn.Module, EmbeddingAccessMixin, ModuleUtilsMixin, PushToH
12681268
# models, this attribute is currently defined in respective model code. For base models, it comes from
12691269
# `config.base_model_pp_plan` during `post_init`.
12701270
_pp_plan: dict[str, tuple[str, str]] = None
1271+
# An expert parallel plan used instead of `_tp_plan` when expert parallelism is enabled. For base models, it comes
1272+
# from `config.base_model_ep_plan` during `post_init`.
1273+
_ep_plan: dict[str, str] = None
12711274
# FSDP2 sharding plan of the form `{"layers.*": "free_full_weight"}`. For top-level models, this attribute is
12721275
# defined on the head class (e.g. `*ForCausalLM`). For base models, it comes from `config.base_model_fsdp_plan`
12731276
# during `post_init`.
@@ -1415,15 +1418,18 @@ def post_init(self):
14151418
"""
14161419
# Attach the different parallel plans and tied weight keys to the top-most model, so that everything is
14171420
# easily available.
1418-
self._tp_plan, self._ep_plan, self._pp_plan, self._fsdp_plan = {}, {}, {}, {}
1421+
# Start from the class-level plans (e.g. `{"lm_head": "colwise_rep"}` on `...ForCausalLM` classes), copying
1422+
# them as they are mutated below and would otherwise contaminate the class attribute shared by all instances
1423+
self._tp_plan = dict(self._tp_plan or {})
1424+
self._ep_plan = dict(self._ep_plan or {})
1425+
self._pp_plan = dict(self._pp_plan or {})
1426+
self._fsdp_plan = dict(self._fsdp_plan or {})
14191427
# If current model is a base model, attach `base_model_tp_plan` and `base_model_pp_plan` from config
14201428
if self.base_model is self:
1421-
self._pp_plan = self.config.base_model_pp_plan.copy() if self.config.base_model_pp_plan is not None else {}
1422-
self._tp_plan = self.config.base_model_tp_plan.copy() if self.config.base_model_tp_plan is not None else {}
1423-
self._ep_plan = self.config.base_model_ep_plan.copy() if self.config.base_model_ep_plan is not None else {}
1424-
self._fsdp_plan = (
1425-
self.config.base_model_fsdp_plan.copy() if self.config.base_model_fsdp_plan is not None else {}
1426-
)
1429+
self._pp_plan.update(self.config.base_model_pp_plan or {})
1430+
self._tp_plan.update(self.config.base_model_tp_plan or {})
1431+
self._ep_plan.update(self.config.base_model_ep_plan or {})
1432+
self._fsdp_plan.update(self.config.base_model_fsdp_plan or {})
14271433
# Current submodel should register its tied weights
14281434
self.all_tied_weights_keys = self.get_expanded_tied_weights_keys(all_submodels=False)
14291435
# Current submodel should register its `_keep_in_fp32_modules`

‎tests/tensor_parallel/test_tensor_parallel.py‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
PackedColwiseParallel,
2626
PackedRowwiseParallel,
2727
RowwiseParallel,
28+
add_tensor_parallel_hooks_to_module,
2829
get_packed_weights,
2930
repack_weights,
3031
)
@@ -166,6 +167,17 @@ def test_tp_plan_none_handling(self):
166167
model.tp_plan = {"model.layers.*.self_attn.q_proj": "colwise"}
167168
self.assertEqual(model.tp_plan, {"model.layers.*.self_attn.q_proj": "colwise"})
168169

170+
def test_post_init_keeps_class_level_plans(self):
171+
"""Class-level plans (e.g. `lm_head` on ForCausalLM classes) must survive post_init alongside the base model plan."""
172+
model_id = "hf-internal-testing/tiny-random-LlamaForCausalLM"
173+
model = AutoModelForCausalLM.from_pretrained(model_id, dtype="auto")
174+
175+
self.assertIn("lm_head", model._tp_plan)
176+
self.assertIn("model.layers.*.self_attn.q_proj", model._tp_plan)
177+
self.assertIn("lm_head", model._pp_plan)
178+
# The merge must not have mutated the class attribute shared by all instances
179+
self.assertEqual(set(type(model)._tp_plan), {"lm_head"})
180+
169181

170182
@is_tensor_parallel_test
171183
class TestTensorParallelLayer(TestCasePlus):
@@ -181,6 +193,21 @@ def size(self):
181193
def get_local_rank(self):
182194
return self.rank
183195

196+
def test_colwise_gather_output_rejects_indivisible_out_features(self):
197+
device_mesh = self.MockDeviceMesh(world_size=2, rank=0)
198+
199+
with self.assertRaises(ValueError) as context:
200+
add_tensor_parallel_hooks_to_module(
201+
model=SimpleNamespace(config=None),
202+
module=torch.nn.Linear(8, 99),
203+
current_module_plan="colwise_gather_output",
204+
layer_name="lm_head",
205+
device_mesh=device_mesh,
206+
)
207+
208+
self.assertIn("lm_head", str(context.exception))
209+
self.assertIn("divisible", str(context.exception))
210+
184211
def test_colwise_get_expected_sharded_shape(self):
185212
world_size = 3
186213
size = 10 # not divisible by world_size to test edge case

‎tests/test_tensor_parallel_mixin.py‎

Lines changed: 15 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -491,6 +491,15 @@ def _get_tp_model_class(self):
491491
return self.model_tester.causal_lm_class
492492
return self.all_model_classes[0]
493493

494+
def _get_tp_config(self):
495+
"""Tiny config with `vocab_size` rounded up to a multiple of the world size, as sharded dims (typically `lm_head`) have to be split across ranks."""
496+
config = self.model_tester.get_config()
497+
text_config = config.get_text_config()
498+
remainder = text_config.vocab_size % self.tensor_parallel_size
499+
if remainder:
500+
text_config.vocab_size += self.tensor_parallel_size - remainder
501+
return config
502+
494503
def _skip_if_not_supported(self, expert_parallel: bool = False):
495504
"""Check and skip the test if tensor/expert parallel is not supported for this model/environment."""
496505
parallelism = "Expert" if expert_parallel else "Tensor"
@@ -539,7 +548,7 @@ def _skip_if_not_supported(self, expert_parallel: bool = False):
539548
def test_tp_forward(self):
540549
self._skip_if_not_supported()
541550

542-
config = self.model_tester.get_config()
551+
config = self._get_tp_config()
543552
model_class = self._get_tp_model_class()
544553
atol = self.tensor_parallel_atol
545554
rtol = self.tensor_parallel_rtol
@@ -555,7 +564,7 @@ def test_tp_forward(self):
555564
def test_tp_backward(self):
556565
self._skip_if_not_supported()
557566

558-
config = self.model_tester.get_config()
567+
config = self._get_tp_config()
559568
model_class = self._get_tp_model_class()
560569
atol = self.tensor_parallel_atol
561570
rtol = self.tensor_parallel_rtol
@@ -572,7 +581,7 @@ def test_tp_generation(self):
572581
# Test TP generation: unfused checkpoint → conversion mapping (if needed) → TP sharding → model → generate
573582
self._skip_if_not_supported()
574583

575-
config = self.model_tester.get_config()
584+
config = self._get_tp_config()
576585

577586
model_class = self._get_tp_model_class()
578587
atol = self.tensor_parallel_atol
@@ -594,7 +603,7 @@ def test_tp_generation_quantized(self):
594603
if not is_torchao_available():
595604
self.skipTest("Test requires torchao")
596605

597-
config = self.model_tester.get_config()
606+
config = self._get_tp_config()
598607
model_class = self._get_tp_model_class()
599608
max_new_tokens = 25
600609

@@ -611,7 +620,7 @@ def test_tp_generation_quantized(self):
611620
def test_ep_forward(self):
612621
self._skip_if_not_supported(expert_parallel=True)
613622

614-
config = self.model_tester.get_config()
623+
config = self._get_tp_config()
615624
model_class = self._get_tp_model_class()
616625
atol = self.tensor_parallel_atol
617626
rtol = self.tensor_parallel_rtol
@@ -627,7 +636,7 @@ def test_ep_forward(self):
627636
def test_ep_backward(self):
628637
self._skip_if_not_supported(expert_parallel=True)
629638

630-
config = self.model_tester.get_config()
639+
config = self._get_tp_config()
631640
model_class = self._get_tp_model_class()
632641
atol = self.tensor_parallel_atol
633642
rtol = self.tensor_parallel_rtol

0 commit comments

Comments
 (0)