add tokenizer

special_tokens_map.json

@@ -1,4 +1,17 @@
 {
+  "additional_special_tokens": [
+    "<BRG>",
+    "N_00",
+    "N_01",
+    "N_02",
+    "N_03",
+    "N_04",
+    "N_05",
+    "N_06",
+    "N_07",
+    "N_08",
+    "N_09"
+  ],
   "bos_token": "<s>",
   "eos_token": "</s>",
   "pad_token": "<pad>",

tokenizer.json

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6c1abd73425d69d27b6933af4fa2a004568434169689b37d1314c6ca3a1d2a7f
-size 14500541
+oid sha256:aa8a458a0b4246af98eeba2630a5817264d60de06571c35bcc5b9927de0d968f
+size 14502533

tokenizer_config.json

@@ -1,5 +1,19 @@
 {
   "add_prefix_space": false,
+  "add_special_tokens": true,
+  "additional_special_tokens": [
+    "<BRG>",
+    "N_00",
+    "N_01",
+    "N_02",
+    "N_03",
+    "N_04",
+    "N_05",
+    "N_06",
+    "N_07",
+    "N_08",
+    "N_09"
+  ],
   "bos_token": "<s>",
   "eos_token": "</s>",
   "name_or_path": "bigscience/bloom-560m",