from transformers import AutoModelForCausalLM, AutoTokenizer, SinkCache
from transformers.trainer_utils import set_seed
# Load the model and tokenizer
# model_name = "meta-llama/Llama-2-7b-hf" # <-- this works!
# model_name = "mistralai/Mistral-7B-v0.1" # <-- this works!
model_name = "Qwen/Qwen1.5-1.8B" # <-- this doesn't work!
tokenizer = AutoTokenizer.from_pretrained(model_name)
model = AutoModelForCausalLM.from_pretrained(model_name)
# Prepare the input
input_text = "The quick brown fox jumps over the lazy"
inputs = tokenizer(input_text, return_tensors="pt")
# Generate the output
set_seed(42)
sink_cache = SinkCache(window_length=50, num_sink_tokens=8)
output = model.generate(
**inputs,
use_cache=True,
past_key_values=sink_cache,
max_new_tokens=100,
do_sample=True,
top_p=0.9,
)
# Decode the output
output_text = tokenizer.decode(output[0])
print(output_text)Traceback (most recent call last):
File "/remote/ayuser/user/sinkcache_test.py", line 19, in <module>
output = model.generate(
File "/home/user/anaconda3/envs/bark/lib/python3.9/site-packages/torch/utils/_contextlib.py", line 115, in decorate_context
return func(*args, **kwargs)
File "/home/user/anaconda3/envs/bark/lib/python3.9/site-packages/transformers/generation/utils.py", line 1989, in generate
result = self._sample(
File "/home/user/anaconda3/envs/bark/lib/python3.9/site-packages/transformers/generation/utils.py", line 2932, in _sample
outputs = self(**model_inputs, return_dict=True)
File "/home/user/anaconda3/envs/bark/lib/python3.9/site-packages/torch/nn/modules/module.py", line 1518, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
File "/home/user/anaconda3/envs/bark/lib/python3.9/site-packages/torch/nn/modules/module.py", line 1527, in _call_impl
return forward_call(*args, **kwargs)
File "/home/user/anaconda3/envs/bark/lib/python3.9/site-packages/transformers/models/qwen2/modeling_qwen2.py", line 1054, in forward
outputs = self.model(
File "/home/user/anaconda3/envs/bark/lib/python3.9/site-packages/torch/nn/modules/module.py", line 1518, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
File "/home/user/anaconda3/envs/bark/lib/python3.9/site-packages/torch/nn/modules/module.py", line 1527, in _call_impl
return forward_call(*args, **kwargs)
File "/home/user/anaconda3/envs/bark/lib/python3.9/site-packages/transformers/models/qwen2/modeling_qwen2.py", line 856, in forward
layer_outputs = decoder_layer(
File "/home/user/anaconda3/envs/bark/lib/python3.9/site-packages/torch/nn/modules/module.py", line 1518, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
File "/home/user/anaconda3/envs/bark/lib/python3.9/site-packages/torch/nn/modules/module.py", line 1527, in _call_impl
return forward_call(*args, **kwargs)
File "/home/user/anaconda3/envs/bark/lib/python3.9/site-packages/transformers/models/qwen2/modeling_qwen2.py", line 596, in forward
hidden_states, self_attn_weights, present_key_value = self.self_attn(
File "/home/user/anaconda3/envs/bark/lib/python3.9/site-packages/torch/nn/modules/module.py", line 1518, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
File "/home/user/anaconda3/envs/bark/lib/python3.9/site-packages/torch/nn/modules/module.py", line 1527, in _call_impl
return forward_call(*args, **kwargs)
File "/home/user/anaconda3/envs/bark/lib/python3.9/site-packages/transformers/models/qwen2/modeling_qwen2.py", line 496, in forward
query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin, position_ids)
File "/home/user/anaconda3/envs/bark/lib/python3.9/site-packages/transformers/models/qwen2/modeling_qwen2.py", line 149, in apply_rotary_pos_emb
cos = cos[position_ids].unsqueeze(unsqueeze_dim)
IndexError: index 50 is out of bounds for dimension 0 with size 50
The generation would complete with no error.
System Info
transformersversion: 4.43.2Who can help?
@zucchini-nlp @gante
Information
Tasks
examplesfolder (such as GLUE/SQuAD, ...)Reproduction
Run the code below on CPU (using GPU hides the actual error behind a
RuntimeError: CUDA error: device-side assert triggered).*** Works fine in v4.42.4, error appears only in 4.43.0+ ***
*** Tested Llama-2, Mistral, and Qwen1.5 models. Issue only appears to affect Qwen1.5, but may impact other models that I didn't test. ***
This appears to be a separate issue from #31381
Traceback:
Expected behavior
The generation would complete with no error.