System Info
transformers==4.42.3
torch==2.3.0
numpy==1.26.4
gguf==0.6.0
Who can help?
@SunMarc
Information
Tasks
Reproduction
import os
from transformers import AutoModelForCausalLM
gguf_path = "path/to/llama3-8b.fp16.gguf" # or bf16
model_id = os.path.dirname(gguf_path)
gguf_file = os.path.basename(gguf_path)
model = AutoModelForCausalLM.from_pretrained(model_id, gguf_file=gguf_file)
Expected behavior
Besides quantization, only F32 is implemented. FP16 and BF16 are not yet supported.
fp16 error log:
Converting and de-quantizing GGUF tensors...: 0%| | 0/291 [00:00<?, ?it/s]
Traceback (most recent call last):
File "/data2/Penut/LLM-Backend/Testing.py", line 9, in <module>
model = AutoModelForCausalLM.from_pretrained(model_id, gguf_file=gguf_file)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/data2/Penut/.miniconda/envs/Py311/lib/python3.11/site-packages/transformers/models/auto/auto_factory.py", line 564, in from_pretrained
return model_class.from_pretrained(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/data2/Penut/.miniconda/envs/Py311/lib/python3.11/site-packages/transformers/modeling_utils.py", line 3583, in from_pretrained
state_dict = load_gguf_checkpoint(gguf_path, return_tensors=True)["tensors"]
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/data2/Penut/.miniconda/envs/Py311/lib/python3.11/site-packages/transformers/modeling_gguf_pytorch_utils.py", line 146, in load_gguf_checkpoint
weights = load_dequant_gguf_tensor(shape=shape, ggml_type=tensor.tensor_type, data=tensor.data)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/data2/Penut/.miniconda/envs/Py311/lib/python3.11/site-packages/transformers/integrations/ggml.py", line 507, in load_dequant_gguf_tensor
raise NotImplementedError(
NotImplementedError: ggml_type 1 not implemented - please raise an issue on huggingface transformers: https://github.com/huggingface/transformers/issues/new/choosebf16 error log:
Traceback (most recent call last):
File "/data2/Penut/LLM-Backend/Testing.py", line 9, in <module>
model = AutoModelForCausalLM.from_pretrained(model_id, gguf_file=gguf_file)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/data2/Penut/.miniconda/envs/Py311/lib/python3.11/site-packages/transformers/models/auto/auto_factory.py", line 524, in from_pretrained
config, kwargs = AutoConfig.from_pretrained(
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/data2/Penut/.miniconda/envs/Py311/lib/python3.11/site-packages/transformers/models/auto/configuration_auto.py", line 965, in from_pretrained
config_dict, unused_kwargs = PretrainedConfig.get_config_dict(pretrained_model_name_or_path, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/data2/Penut/.miniconda/envs/Py311/lib/python3.11/site-packages/transformers/configuration_utils.py", line 632, in get_config_dict
config_dict, kwargs = cls._get_config_dict(pretrained_model_name_or_path, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/data2/Penut/.miniconda/envs/Py311/lib/python3.11/site-packages/transformers/configuration_utils.py", line 719, in _get_config_dict
config_dict = load_gguf_checkpoint(resolved_config_file, return_tensors=False)["config"]
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/data2/Penut/.miniconda/envs/Py311/lib/python3.11/site-packages/transformers/modeling_gguf_pytorch_utils.py", line 81, in load_gguf_checkpoint
reader = GGUFReader(gguf_checkpoint_path)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/data2/Penut/.miniconda/envs/Py311/lib/python3.11/site-packages/gguf/gguf_reader.py", line 116, in __init__
self._build_tensors(offs, tensors_fields)
File "/data2/Penut/.miniconda/envs/Py311/lib/python3.11/site-packages/gguf/gguf_reader.py", line 239, in _build_tensors
ggml_type = GGMLQuantizationType(raw_dtype[0])
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/data2/Penut/.miniconda/envs/Py311/lib/python3.11/enum.py", line 714, in __call__
return cls.__new__(cls, value)
^^^^^^^^^^^^^^^^^^^^^^^
File "/data2/Penut/.miniconda/envs/Py311/lib/python3.11/enum.py", line 1137, in __new__
raise ve_exc
ValueError: 30 is not a valid GGMLQuantizationTypeI tried to add F16 to GGML_TYPES:
GGML_TYPES = {
"F32": 0,
"F16": 1,
# ...
}
def load_dequant_gguf_tensor(shape, ggml_type, data):
if ggml_type == GGML_TYPES["F32"]:
values = data
elif ggml_type == GGML_TYPES["F16"]:
values = data
# ...I'm not sure if this is correct, but after converting to hf, the PPL is over 1000.
System Info
Who can help?
@SunMarc
Information
Tasks
examplesfolder (such as GLUE/SQuAD, ...)Reproduction
Expected behavior
Besides quantization, only F32 is implemented. FP16 and BF16 are not yet supported.
fp16 error log:
Converting and de-quantizing GGUF tensors...: 0%| | 0/291 [00:00<?, ?it/s] Traceback (most recent call last): File "/data2/Penut/LLM-Backend/Testing.py", line 9, in <module> model = AutoModelForCausalLM.from_pretrained(model_id, gguf_file=gguf_file) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/data2/Penut/.miniconda/envs/Py311/lib/python3.11/site-packages/transformers/models/auto/auto_factory.py", line 564, in from_pretrained return model_class.from_pretrained( ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/data2/Penut/.miniconda/envs/Py311/lib/python3.11/site-packages/transformers/modeling_utils.py", line 3583, in from_pretrained state_dict = load_gguf_checkpoint(gguf_path, return_tensors=True)["tensors"] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/data2/Penut/.miniconda/envs/Py311/lib/python3.11/site-packages/transformers/modeling_gguf_pytorch_utils.py", line 146, in load_gguf_checkpoint weights = load_dequant_gguf_tensor(shape=shape, ggml_type=tensor.tensor_type, data=tensor.data) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/data2/Penut/.miniconda/envs/Py311/lib/python3.11/site-packages/transformers/integrations/ggml.py", line 507, in load_dequant_gguf_tensor raise NotImplementedError( NotImplementedError: ggml_type 1 not implemented - please raise an issue on huggingface transformers: https://github.com/huggingface/transformers/issues/new/choosebf16 error log:
Traceback (most recent call last): File "/data2/Penut/LLM-Backend/Testing.py", line 9, in <module> model = AutoModelForCausalLM.from_pretrained(model_id, gguf_file=gguf_file) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/data2/Penut/.miniconda/envs/Py311/lib/python3.11/site-packages/transformers/models/auto/auto_factory.py", line 524, in from_pretrained config, kwargs = AutoConfig.from_pretrained( ^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/data2/Penut/.miniconda/envs/Py311/lib/python3.11/site-packages/transformers/models/auto/configuration_auto.py", line 965, in from_pretrained config_dict, unused_kwargs = PretrainedConfig.get_config_dict(pretrained_model_name_or_path, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/data2/Penut/.miniconda/envs/Py311/lib/python3.11/site-packages/transformers/configuration_utils.py", line 632, in get_config_dict config_dict, kwargs = cls._get_config_dict(pretrained_model_name_or_path, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/data2/Penut/.miniconda/envs/Py311/lib/python3.11/site-packages/transformers/configuration_utils.py", line 719, in _get_config_dict config_dict = load_gguf_checkpoint(resolved_config_file, return_tensors=False)["config"] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/data2/Penut/.miniconda/envs/Py311/lib/python3.11/site-packages/transformers/modeling_gguf_pytorch_utils.py", line 81, in load_gguf_checkpoint reader = GGUFReader(gguf_checkpoint_path) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/data2/Penut/.miniconda/envs/Py311/lib/python3.11/site-packages/gguf/gguf_reader.py", line 116, in __init__ self._build_tensors(offs, tensors_fields) File "/data2/Penut/.miniconda/envs/Py311/lib/python3.11/site-packages/gguf/gguf_reader.py", line 239, in _build_tensors ggml_type = GGMLQuantizationType(raw_dtype[0]) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/data2/Penut/.miniconda/envs/Py311/lib/python3.11/enum.py", line 714, in __call__ return cls.__new__(cls, value) ^^^^^^^^^^^^^^^^^^^^^^^ File "/data2/Penut/.miniconda/envs/Py311/lib/python3.11/enum.py", line 1137, in __new__ raise ve_exc ValueError: 30 is not a valid GGMLQuantizationTypeI tried to add F16 to
GGML_TYPES:I'm not sure if this is correct, but after converting to hf, the PPL is over 1000.