Skip to content

Commit e035e45

Browse files
committed
Fix assisted decoding for models with EncoderDecoder cache & OlmoHybrid (#47361)
* fix attribute error * fix * fix * minicpm
1 parent 0eb5e03 commit e035e45

7 files changed

Lines changed: 15 additions & 2 deletions

File tree

‎src/transformers/cache_utils.py‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1510,7 +1510,7 @@ def get_mask_sizes(self, query_length: int, layer_idx: int) -> tuple[int, int]:
15101510

15111511
return self.layers[layer_idx].get_mask_sizes(query_length)
15121512

1513-
def get_query_offset(self, layer_idx: int) -> int:
1513+
def get_query_offset(self, layer_idx: int = 0) -> int:
15141514
"""Returns the current offset of the query for the given `layer_idx`. It's always equal to the cache length, i.e.
15151515
`get_seq_length(layer_idx)`, except for MTP layers.
15161516
"""
@@ -2002,6 +2002,9 @@ def is_sliding(self):
20022002
def is_compileable(self) -> bool:
20032003
return self.self_attention_cache.is_compileable
20042004

2005+
def activate_past_recording(self):
2006+
self.self_attention_cache.activate_past_recording()
2007+
20052008
def get_max_cache_shape(self, layer_idx: int = 0) -> int:
20062009
logger.warning_once(
20072010
"`get_max_cache_shape` is deprecated, and will be removed in version 5.16. Please use `get_max_length` instead"
@@ -2014,7 +2017,7 @@ def get_max_cache_shape(self, layer_idx: int = 0) -> int:
20142017

20152018

20162019
class MtpCache(DynamicCache):
2017-
def get_query_offset(self, layer_idx=0):
2020+
def get_query_offset(self, layer_idx: int = 0):
20182021
# Queries of MTP depth k run k+1 tokens ahead of the main_model, i.e. they have an offset of k+1
20192022
mtp_offset = layer_idx + 1
20202023
return super().get_query_offset(layer_idx) + mtp_offset

‎src/transformers/models/lfm2_vl/modeling_lfm2_vl.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,7 @@ class Lfm2VlPreTrainedModel(PreTrainedModel):
8686
_can_compile_fullgraph = False
8787
_supports_flex_attn = True
8888
_supports_attention_backend = True
89+
_is_stateful = True
8990

9091

9192
@auto_docstring(

‎src/transformers/models/lfm2_vl/modular_lfm2_vl.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,7 @@ def pixel_unshuffle(self, hidden_states: torch.Tensor):
7676
class Lfm2VlPreTrainedModel(LlavaPreTrainedModel):
7777
_can_compile_fullgraph = False
7878
base_model_prefix = "model"
79+
_is_stateful = True
7980

8081

8182
class Lfm2VlCausalLMOutputWithPast(LlavaCausalLMOutputWithPast):

‎src/transformers/models/minicpmv4_6/modeling_minicpmv4_6.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -584,6 +584,7 @@ class MiniCPMV4_6PreTrainedModel(PreTrainedModel):
584584
"MiniCPMV4_6VisionEncoderLayer",
585585
"MiniCPMV4_6ViTWindowAttentionMerger",
586586
]
587+
_is_stateful = True
587588

588589

589590
@auto_docstring(

‎src/transformers/models/minicpmv4_6/modular_minicpmv4_6.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -536,6 +536,7 @@ class MiniCPMV4_6PreTrainedModel(PreTrainedModel):
536536
"MiniCPMV4_6VisionEncoderLayer",
537537
"MiniCPMV4_6ViTWindowAttentionMerger",
538538
]
539+
_is_stateful = True
539540

540541

541542
class MiniCPMV4_6Model(Lfm2VlModel):

‎src/transformers/models/olmo_hybrid/modeling_olmo_hybrid.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -158,6 +158,9 @@ def has_previous_state(self):
158158
"""We have a previous state if the last linear (conv) layer was already updated."""
159159
return self.conv_states_q[self.last_linear_layer] is not None
160160

161+
def get_query_offset(self, layer_idx: int = 0) -> int:
162+
return self.get_seq_length(layer_idx=layer_idx)
163+
161164

162165
class OlmoHybridRMSNormGated(nn.Module):
163166
def __init__(self, hidden_size, eps=1e-6, **kwargs):

‎src/transformers/models/olmo_hybrid/modular_olmo_hybrid.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -291,6 +291,9 @@ def has_previous_state(self):
291291
"""We have a previous state if the last linear (conv) layer was already updated."""
292292
return self.conv_states_q[self.last_linear_layer] is not None
293293

294+
def get_query_offset(self, layer_idx: int = 0) -> int:
295+
return self.get_seq_length(layer_idx=layer_idx)
296+
294297

295298
class OlmoHybridRMSNormGated(Qwen3NextRMSNormGated):
296299
pass

0 commit comments

Comments
 (0)