copied from hf-internal-testing/llama-tokenizer

special_tokens_map.json

@@ -0,0 +1 @@
+{"bos_token": {"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, "eos_token": {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, "unk_token": {"content": "<unk>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}}

tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347
+size 499723

tokenizer_config.json

@@ -0,0 +1 @@
+{"bos_token": {"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "__type": "AddedToken"}, "eos_token": {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "__type": "AddedToken"}, "unk_token": {"content": "<unk>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "__type": "AddedToken"}, "pad_token": null, "add_bos_token": true, "add_eos_token": false, "sp_model_kwargs": {}, "clean_up_tokenization_spaces": false, "use_default_system_prompt": true, "spaces_between_special_tokens": false, "legacy": null, "model_max_length": 2048, "special_tokens_map_file": "/Users/bartosztopolski/.cache/huggingface/transformers/ee67ed7a74d257d710101490b096d6cebf66288983f0a7055a61a858685c9810.8bc7f2690b1bfa4da931da97531f22e22d495ecd5c2d99afc6ac4638635fdfc0", "tokenizer_file": "/Users/bartosztopolski/.cache/huggingface/transformers/23a21ac434cda96043d4fdb0ef66a81a8cf8125f3db17ce18577f4723025c208.cca5f981dc3bf24a9360282a28a74b32877f11138f5da76ed5dce54119a5ee51", "name_or_path": "hf-internal-testing/llama-tokenizer", "tokenizer_class": "LlamaTokenizer"}