File size: 388 Bytes
9fbec62 |
1 2 3 4 5 6 7 8 9 10 11 12 |
{
"bos_token": "<BOS>",
"cls_token": "<BOS>",
"eos_token": "<EOS>",
"name_or_path": "/home/mulcar/bigscience/gpt3-sl-tok-large.60k-bytelevel",
"pad_token": "<PAD>",
"sep_token": "<EOS>",
"special_tokens_map_file": "/home/mulcar/bigscience/gpt3-sl-tok-large.60k-bytelevel/special_tokens_map.json",
"tokenizer_class": "PreTrainedTokenizerFast",
"model_max_length": 1024
}
|