Skip to content

Commit 8f23330

Browse files
CyrilvallezArthurZucker
authored andcommitted
Small improvement (#47348)
fix
1 parent 76613e6 commit 8f23330

3 files changed

Lines changed: 8 additions & 6 deletions

File tree

‎src/transformers/models/inkling/configuration_inkling.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -183,6 +183,7 @@ class InklingVisionConfig(PreTrainedConfig):
183183
initializer_range: float = 0.02
184184

185185

186+
@strict
186187
class InklingConfig(PreTrainedConfig):
187188
"""Top-level multimodal config (`InklingMMConfig` in the SGLang source)."""
188189

‎src/transformers/models/inkling/modeling_inkling.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -557,7 +557,6 @@ def __init__(self, config: InklingTextConfig, layer_idx: int):
557557
self.input_layernorm = InklingRMSNorm(config.hidden_size, config.rms_norm_eps)
558558
self.post_attention_layernorm = InklingRMSNorm(config.hidden_size, config.rms_norm_eps)
559559
self.layer_type = config.layer_types[layer_idx]
560-
self.attention_type = "full_attention" if self.layer_type == "hybrid" else "sliding_attention"
561560
self.attn_sconv = InklingShortConvolution(
562561
config.hidden_size, config.conv_kernel_size, layer_idx=layer_idx, conv_idx=2
563562
)
@@ -706,10 +705,11 @@ def forward(
706705
}
707706

708707
hidden_states = inputs_embeds
709-
for decoder_layer in self.layers:
708+
for i, decoder_layer in enumerate(self.layers):
709+
attention_type = "full_attention" if self.config.layer_types[i] == "hybrid" else "sliding_attention"
710710
hidden_states = decoder_layer(
711711
hidden_states,
712-
attention_mask=causal_mask_mapping[decoder_layer.attention_type],
712+
attention_mask=causal_mask_mapping[attention_type],
713713
conv_mask=causal_mask_mapping["linear_attention"],
714714
past_key_values=past_key_values,
715715
**kwargs,

‎src/transformers/models/inkling/modular_inkling.py‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -216,6 +216,7 @@ class InklingVisionConfig(PreTrainedConfig):
216216
initializer_range: float = 0.02
217217

218218

219+
@strict
219220
class InklingConfig(PreTrainedConfig):
220221
"""Top-level multimodal config (`InklingMMConfig` in the SGLang source)."""
221222

@@ -652,7 +653,6 @@ def __init__(self, config: InklingTextConfig, layer_idx: int):
652653
self.input_layernorm = InklingRMSNorm(config.hidden_size, config.rms_norm_eps)
653654
self.post_attention_layernorm = InklingRMSNorm(config.hidden_size, config.rms_norm_eps)
654655
self.layer_type = config.layer_types[layer_idx]
655-
self.attention_type = "full_attention" if self.layer_type == "hybrid" else "sliding_attention"
656656
self.attn_sconv = InklingShortConvolution(
657657
config.hidden_size, config.conv_kernel_size, layer_idx=layer_idx, conv_idx=2
658658
)
@@ -801,10 +801,11 @@ def forward(
801801
}
802802

803803
hidden_states = inputs_embeds
804-
for decoder_layer in self.layers:
804+
for i, decoder_layer in enumerate(self.layers):
805+
attention_type = "full_attention" if self.config.layer_types[i] == "hybrid" else "sliding_attention"
805806
hidden_states = decoder_layer(
806807
hidden_states,
807-
attention_mask=causal_mask_mapping[decoder_layer.attention_type],
808+
attention_mask=causal_mask_mapping[attention_type],
808809
conv_mask=causal_mask_mapping["linear_attention"],
809810
past_key_values=past_key_values,
810811
**kwargs,

0 commit comments

Comments
 (0)