Hugging Face's logo

DuongTrongChi
/
bloom-560m-code-text-to-sql

PEFT
TensorBoard
Safetensors
trl
sft
Generated from Trainer
Model card Files Metrics Community
DuongTrongChi commited on
Commit
11d154a
·
verified ·
1 Parent(s): c6f9ae4

Upload tokenizer

Browse files
Files changed (6) hide show
  1. added_tokens.json +4 -0
  2. merges.txt +0 -0
  3. special_tokens_map.json +2 -2
  4. tokenizer.json +2 -2
  5. tokenizer_config.json +10 -25
  6. vocab.json +0 -0
added_tokens.json ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ {
2
+ "<|im_end|>": 50265,
3
+ "<|im_start|>": 50266
4
+ }
merges.txt ADDED
The diff for this file is too large to render. See raw diff
 
special_tokens_map.json CHANGED
@@ -19,9 +19,9 @@
19
  "eos_token": "<|im_end|>",
20
  "pad_token": "<|im_end|>",
21
  "unk_token": {
22
- "content": "<unk>",
23
  "lstrip": false,
24
- "normalized": false,
25
  "rstrip": false,
26
  "single_word": false
27
  }
 
19
  "eos_token": "<|im_end|>",
20
  "pad_token": "<|im_end|>",
21
  "unk_token": {
22
+ "content": "</s>",
23
  "lstrip": false,
24
+ "normalized": true,
25
  "rstrip": false,
26
  "single_word": false
27
  }
tokenizer.json CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:b544845929225ecf2a773b7d94b93f3ca8c94d635d11f43e472f6594bc96c85e
3
- size 14500847
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3c3bffcdc2e12923436360a7056a8655db5a1cfb4aa062359a40e1a93d60e33f
3
+ size 2109004
tokenizer_config.json CHANGED
@@ -1,18 +1,11 @@
1
  {
 
2
  "add_prefix_space": false,
3
  "added_tokens_decoder": {
4
- "0": {
5
- "content": "<unk>",
6
- "lstrip": false,
7
- "normalized": false,
8
- "rstrip": false,
9
- "single_word": false,
10
- "special": true
11
- },
12
  "1": {
13
- "content": "<s>",
14
  "lstrip": false,
15
- "normalized": false,
16
  "rstrip": false,
17
  "single_word": false,
18
  "special": true
@@ -20,20 +13,12 @@
20
  "2": {
21
  "content": "</s>",
22
  "lstrip": false,
23
- "normalized": false,
24
- "rstrip": false,
25
- "single_word": false,
26
- "special": true
27
- },
28
- "3": {
29
- "content": "<pad>",
30
- "lstrip": false,
31
- "normalized": false,
32
  "rstrip": false,
33
  "single_word": false,
34
  "special": true
35
  },
36
- "250680": {
37
  "content": "<|im_end|>",
38
  "lstrip": false,
39
  "normalized": false,
@@ -41,7 +26,7 @@
41
  "single_word": false,
42
  "special": true
43
  },
44
- "250681": {
45
  "content": "<|im_start|>",
46
  "lstrip": false,
47
  "normalized": false,
@@ -56,11 +41,11 @@
56
  ],
57
  "bos_token": "<|im_start|>",
58
  "chat_template": "{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
59
- "clean_up_tokenization_spaces": false,
60
  "eos_token": "<|im_end|>",
 
61
  "model_max_length": 1000000000000000019884624838656,
62
  "pad_token": "<|im_end|>",
63
- "padding_side": "right",
64
- "tokenizer_class": "BloomTokenizer",
65
- "unk_token": "<unk>"
66
  }
 
1
  {
2
+ "add_bos_token": true,
3
  "add_prefix_space": false,
4
  "added_tokens_decoder": {
 
 
 
 
 
 
 
 
5
  "1": {
6
+ "content": "<pad>",
7
  "lstrip": false,
8
+ "normalized": true,
9
  "rstrip": false,
10
  "single_word": false,
11
  "special": true
 
13
  "2": {
14
  "content": "</s>",
15
  "lstrip": false,
16
+ "normalized": true,
 
 
 
 
 
 
 
 
17
  "rstrip": false,
18
  "single_word": false,
19
  "special": true
20
  },
21
+ "50265": {
22
  "content": "<|im_end|>",
23
  "lstrip": false,
24
  "normalized": false,
 
26
  "single_word": false,
27
  "special": true
28
  },
29
+ "50266": {
30
  "content": "<|im_start|>",
31
  "lstrip": false,
32
  "normalized": false,
 
41
  ],
42
  "bos_token": "<|im_start|>",
43
  "chat_template": "{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
44
+ "clean_up_tokenization_spaces": true,
45
  "eos_token": "<|im_end|>",
46
+ "errors": "replace",
47
  "model_max_length": 1000000000000000019884624838656,
48
  "pad_token": "<|im_end|>",
49
+ "tokenizer_class": "GPT2Tokenizer",
50
+ "unk_token": "</s>"
 
51
  }
vocab.json ADDED
The diff for this file is too large to render. See raw diff