from transformers import AutoTokenizer

model = "codellama/CodeLlama-7b-hf"

tokenizer = AutoTokenizer.from_pretrained(model, use_fast=False)
tokenizer_fast = AutoTokenizer.from_pretrained(model, use_fast=True)

print(tokenizer.encode("<s>\n", add_special_tokens=False))
# [1, 13]

print(tokenizer_fast.encode("<s>\n", add_special_tokens=False))
# [1, 29871, 13]

# the same issue occurs with any element of `tokenizer.all_special_tokens`, not just <s>

for special_token in tokenizer.all_special_tokens:
    print(special_token)
    print(tokenizer.encode(f"{special_token}\n", add_special_tokens=False))
    print(tokenizer_fast.encode(f"{special_token}\n", add_special_tokens=False))
    print()
    
# <s>
# [1, 13]
# [1, 29871, 13]

# </s>
# [2, 13]
# [2, 29871, 13]

# <unk>
# [0, 13]
# [0, 29871, 13]

# ▁<PRE>
# [32007, 13]
# [32007, 29871, 13]

# ▁<MID>
# [32009, 13]
# [32009, 29871, 13]

# ▁<SUF>
# [32008, 13]
# [32008, 29871, 13]

# ▁<EOT>
# [32010, 13]
# [32010, 29871, 13]

# assuming repo is cloned at ./codellama and 7b is downloaded

import sys
sys.path.append('codellama')

from llama.tokenizer import Tokenizer

tokenizer_facebookresearch = Tokenizer('codellama/CodeLlama-7b/tokenizer.model')

print(tokenizer_facebookresearch.encode('<PRE>\n', bos=False, eos=False))
# [32007, 13]