FalconNet commited on
Commit
c0c980b
·
verified ·
1 Parent(s): 9c213ad

Upload tokenizer

Browse files
Files changed (2) hide show
  1. tokenizer.json +2 -2
  2. tokenizer_config.json +1 -1
tokenizer.json CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:1815ea80ab714ea1e76500b20468d3bdb4e3a993cf8d473a640a8a44929879e3
3
- size 11422185
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:fab42efe8d17406525a9154b728cf9e957629a8ed7ce997770efdd71128c6a1a
3
+ size 11422086
tokenizer_config.json CHANGED
@@ -209,7 +209,7 @@
209
  "errors": "replace",
210
  "model_max_length": 32768,
211
  "pad_token": "<|PAD_TOKEN|>",
212
- "padding_side": "right",
213
  "split_special_tokens": false,
214
  "tokenizer_class": "Qwen2Tokenizer",
215
  "unk_token": null
 
209
  "errors": "replace",
210
  "model_max_length": 32768,
211
  "pad_token": "<|PAD_TOKEN|>",
212
+ "padding_side": "left",
213
  "split_special_tokens": false,
214
  "tokenizer_class": "Qwen2Tokenizer",
215
  "unk_token": null