mirror of
https://github.com/ggerganov/llama.cpp
synced 2026-04-26 18:42:50 +02:00
c8ac02fa1b
* requirements : update transformers to 5.5.0 This commit updates the transformers dependency to version 5.5.0. The motivation for this is that transformers 5.5.0 includes support for Gemma4 and is required to be able to convert Gemma4 models. This is also causing issues for user of gguf-my-repo. Refs: https://huggingface.co/spaces/ggml-org/gguf-my-repo/discussions/202 * fix huggingface_hub version * set version of transformers to 5.5.0 * convert : add ty ignore directives to convert_hf_to_gguf.py This commit adds `ty: ignore` directives to transformers tokenizers field/methods to avoid type check errors. There might be better ways to handle this and perhaps this can be done in a follow up commit. The motivation for this is that it looks like in transformers 5.5.0 AutoTokenizer.from_pretrained can return generic tokenizer types or None and the type checker now produces an error when the conversion script accesses field like tokenizer.vocab. * convert : add ty ignore to suppress type check errors * convert : remove incorrect type ignores * convert : fix remaining python checks I was running a newer version of ty locally but I've switched to version 0.0.26 which is what CI uses and I was then able to reproduce the errors. Sorry about the noise. * update transformers version to 5.5.1
47 lines
2.0 KiB
Python
47 lines
2.0 KiB
Python
import time
|
|
import argparse
|
|
|
|
from transformers import AutoTokenizer
|
|
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("dir_tokenizer", help="directory containing 'tokenizer.model' file")
|
|
parser.add_argument("--fname-tok", help="path to a text file to tokenize", required=True)
|
|
args = parser.parse_args()
|
|
|
|
dir_tokenizer = args.dir_tokenizer
|
|
fname_tok = args.fname_tok
|
|
|
|
tokenizer = AutoTokenizer.from_pretrained(dir_tokenizer)
|
|
|
|
print('tokenizing file: ', fname_tok) # noqa: NP100
|
|
fname_out = fname_tok + '.tok'
|
|
with open(fname_tok, 'r', encoding='utf-8') as f:
|
|
lines = f.readlines()
|
|
s = ''.join(lines)
|
|
t_start = time.time()
|
|
res = tokenizer.encode(s, add_special_tokens=False) # ty: ignore[unresolved-attribute]
|
|
t_end = time.time()
|
|
print('\nmain : tokenized in', "{:.3f}".format(1000.0 * (t_end - t_start)), 'ms (py)') # noqa: NP100
|
|
with open(fname_out, 'w', encoding='utf-8') as f:
|
|
for x in res:
|
|
# LLaMA v3 for some reason strips the space for these tokens (and others)
|
|
# if x == 662:
|
|
# f.write(str(x) + ' \' ' + tokenizer.decode(x) + '\'\n')
|
|
# elif x == 1174:
|
|
# f.write(str(x) + ' \' ' + tokenizer.decode(x) + '\'\n')
|
|
# elif x == 2564:
|
|
# f.write(str(x) + ' \' ' + tokenizer.decode(x) + '\'\n')
|
|
# elif x == 758:
|
|
# f.write(str(x) + ' \' ' + tokenizer.decode(x) + '\'\n')
|
|
# elif x == 949:
|
|
# f.write(str(x) + ' \' ' + tokenizer.decode(x) + '\'\n')
|
|
# elif x == 5354:
|
|
# f.write(str(x) + ' \' ' + tokenizer.decode(x) + '\'\n')
|
|
# else:
|
|
# f.write(str(x) + ' \'' + tokenizer.decode(x) + '\'\n')
|
|
# f.write(str(x) + ' \'' + tokenizer.decode(x).strip() + '\'\n')
|
|
f.write(str(x) + '\n')
|
|
print('len(res): ', len(res)) # noqa: NP100
|
|
print('len(lines): ', len(lines)) # noqa: NP100
|
|
print('results written to: ', fname_out) # noqa: NP100
|