xaviergillard commited on
Commit
27a92c4
·
verified ·
1 Parent(s): f429f4d

Upload tokenizer

Browse files
Files changed (2) hide show
  1. special_tokens_map.json +5 -1
  2. tokenizer_config.json +3 -0
special_tokens_map.json CHANGED
@@ -1 +1,5 @@
1
- {}
 
 
 
 
 
1
+ {
2
+ "eos_token": "[SEP]",
3
+ "mask_token": "[MASK]",
4
+ "pad_token": "[PAD]"
5
+ }
tokenizer_config.json CHANGED
@@ -42,6 +42,9 @@
42
  }
43
  },
44
  "clean_up_tokenization_spaces": true,
 
 
45
  "model_max_length": 1000000000000000019884624838656,
 
46
  "tokenizer_class": "PreTrainedTokenizerFast"
47
  }
 
42
  }
43
  },
44
  "clean_up_tokenization_spaces": true,
45
+ "eos_token": "[SEP]",
46
+ "mask_token": "[MASK]",
47
  "model_max_length": 1000000000000000019884624838656,
48
+ "pad_token": "[PAD]",
49
  "tokenizer_class": "PreTrainedTokenizerFast"
50
  }