Omartificial-Intelligence-Space commited on
Commit
5c8588f
·
verified ·
1 Parent(s): d18fc37

update tokenizer_config

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +6 -0
tokenizer_config.json CHANGED
@@ -55,6 +55,12 @@
55
  "sep_token": "</s>",
56
  "stride": 0,
57
  "tokenizer_class": "XLMRobertaTokenizer",
 
 
 
 
 
 
58
  "truncation_side": "right",
59
  "truncation_strategy": "longest_first",
60
  "unk_token": "<unk>"
 
55
  "sep_token": "</s>",
56
  "stride": 0,
57
  "tokenizer_class": "XLMRobertaTokenizer",
58
+ "do_lower_case": false,
59
+ "max_len": 512,
60
+ "model_max_length": 512,
61
+ "padding_side": "right",
62
+ "special_tokens_map_file": "special_tokens_map.json",
63
+ "use_fast": true
64
  "truncation_side": "right",
65
  "truncation_strategy": "longest_first",
66
  "unk_token": "<unk>"