I tried to quantize Qwen2.5VL 3B model and got TypeError: 'NoneType' object is not subscriptable error.
Data from allenai/c4
qwen_dataset = []
for example in calibration_dataset:
message = [
{
"role": "user",
"content": [
{"type": "text", "text": example},
],
}
]
qwen_dataset.append(message)model_id = "Qwen/Qwen2.5-VL-3B-Instruct"
quant_config = QuantizeConfig(bits=4, group_size=128)
model = GPTQModel.load(model_id, quant_config)
model.quantize(qwen_dataset, batch_size=1)
---------------------------------------------------------------------------
TypeError Traceback (most recent call last)
Cell In[75], line 2
1 # increase `batch_size` to match gpu/vram specs to speed up quantization
----> 2 model.quantize(qwen_dataset, batch_size=1)
File ~/code/.venv/lib/python3.11/site-packages/gptqmodel/models/base.py:450, in BaseGPTQModel.quantize(self, calibration_dataset, calibration_dataset_concat_size, batch_size, calibration_enable_gpu_cache, tokenizer, logger_board, backend, buffered_fwd, auto_gc, adapter, adapter_calibration_dataset)
447 # prepare processor worker (looper)
448 module_looper = ModuleLooper(self, processors=processors)
--> 450 return module_looper.loop(
451 calibration_enable_gpu_cache=calibration_enable_gpu_cache,
452 buffered_fwd=buffered_fwd,
453 auto_gc=auto_gc,
454 backend=backend,
455 )
File ~/code/.venv/lib/python3.11/site-packages/torch/utils/_contextlib.py:116, in context_decorator.<locals>.decorate_context(*args, **kwargs)
113 @functools.wraps(func)
114 def decorate_context(*args, **kwargs):
115 with ctx_factory():
--> 116 return func(*args, **kwargs)
File ~/code/.venv/lib/python3.11/site-packages/gptqmodel/looper/module_looper.py:171, in ModuleLooper.loop(self, auto_gc, calibration_enable_gpu_cache, buffered_fwd, **kwargs)
167 processor.receive_input_cache(InputCache([], [], [], []))
169 continue
--> 171 input_cache = self.cache_inputs(layers=layers, auto_gc=auto_gc,
172 calibration_data=processor.calibration_dataset,
173 calibration_enable_gpu_cache=calibration_enable_gpu_cache)
174 processor.receive_input_cache(input_cache)
176 # release calibration_dataset
File ~/code/.venv/lib/python3.11/site-packages/gptqmodel/looper/module_looper.py:50, in ModuleLooper.cache_inputs(self, layers, auto_gc, calibration_data, calibration_enable_gpu_cache)
47 position_ids = []
48 layer_input_kwargs = []
---> 50 cur_layer_device = get_device(layers[0])
51 data_device = cur_layer_device if calibration_enable_gpu_cache else CPU
53 # TODO HookLinear add register_forward_pre_hook()
TypeError: 'NoneType' object is not subscriptable
I tried to quantize Qwen2.5VL 3B model and got
TypeError: 'NoneType' object is not subscriptableerror.Data from
allenai/c4--------------------------------------------------------------------------- TypeError Traceback (most recent call last) Cell In[75], line 2 1 # increase `batch_size` to match gpu/vram specs to speed up quantization ----> 2 model.quantize(qwen_dataset, batch_size=1) File ~/code/.venv/lib/python3.11/site-packages/gptqmodel/models/base.py:450, in BaseGPTQModel.quantize(self, calibration_dataset, calibration_dataset_concat_size, batch_size, calibration_enable_gpu_cache, tokenizer, logger_board, backend, buffered_fwd, auto_gc, adapter, adapter_calibration_dataset) 447 # prepare processor worker (looper) 448 module_looper = ModuleLooper(self, processors=processors) --> 450 return module_looper.loop( 451 calibration_enable_gpu_cache=calibration_enable_gpu_cache, 452 buffered_fwd=buffered_fwd, 453 auto_gc=auto_gc, 454 backend=backend, 455 ) File ~/code/.venv/lib/python3.11/site-packages/torch/utils/_contextlib.py:116, in context_decorator.<locals>.decorate_context(*args, **kwargs) 113 @functools.wraps(func) 114 def decorate_context(*args, **kwargs): 115 with ctx_factory(): --> 116 return func(*args, **kwargs) File ~/code/.venv/lib/python3.11/site-packages/gptqmodel/looper/module_looper.py:171, in ModuleLooper.loop(self, auto_gc, calibration_enable_gpu_cache, buffered_fwd, **kwargs) 167 processor.receive_input_cache(InputCache([], [], [], [])) 169 continue --> 171 input_cache = self.cache_inputs(layers=layers, auto_gc=auto_gc, 172 calibration_data=processor.calibration_dataset, 173 calibration_enable_gpu_cache=calibration_enable_gpu_cache) 174 processor.receive_input_cache(input_cache) 176 # release calibration_dataset File ~/code/.venv/lib/python3.11/site-packages/gptqmodel/looper/module_looper.py:50, in ModuleLooper.cache_inputs(self, layers, auto_gc, calibration_data, calibration_enable_gpu_cache) 47 position_ids = [] 48 layer_input_kwargs = [] ---> 50 cur_layer_device = get_device(layers[0]) 51 data_device = cur_layer_device if calibration_enable_gpu_cache else CPU 53 # TODO HookLinear add register_forward_pre_hook() TypeError: 'NoneType' object is not subscriptable