File size: 871 Bytes
ab2c44d |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 |
{
"tokenizer_name": "AutoTokenizer",
"pretrained_model_name": "AutoModel",
"vocab": {
"vocab_size": 30522,
"model_max_length": 512,
"padding_side": "right",
"truncation_side": "right",
"special_tokens": {
"pad_token": "[PAD]",
"unk_token": "[UNK]",
"cls_token": "[CLS]",
"sep_token": "[SEP]",
"mask_token": "[MASK]"
},
"tokenizer_type": "WordPiece",
"lowercase": true,
"pad_token_id": 0,
"unk_token_id": 100,
"cls_token_id": 101,
"sep_token_id": 102,
"mask_token_id": 103
},
"normalization": {
"lowercase": true,
"strip_accents": true
},
"preprocessing": {
"do_lower_case": true,
"handle_chinese_chars": true
}
} |