Upload tokenizer.json

tokenizer.json

@@ -929,6 +929,24 @@
       "rstrip": false,
       "normalized": false,
       "special": true
+    },
+    {
+      "id": 32100,
+      "content": "[PAD]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 32101,
+      "content": "hello_enthusiastic_morning",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": true,
+      "special": false
     }
   ],
   "normalizer": {
@@ -950,7 +968,8 @@
   "pre_tokenizer": {
     "type": "Metaspace",
     "replacement": "▁",
-    "add_prefix_space": true
+    "add_prefix_space": true,
+    "prepend_scheme": "always"
   },
   "post_processor": {
     "type": "TemplateProcessing",
@@ -1009,7 +1028,8 @@
   "decoder": {
     "type": "Metaspace",
     "replacement": "▁",
-    "add_prefix_space": true
+    "add_prefix_space": true,
+    "prepend_scheme": "always"
   },
   "model": {
     "type": "Unigram",
@@ -129415,6 +129435,7 @@
         "<extra_id_0>",
         0.0
       ]
-    ]
+    ],
+    "byte_fallback": false
   }
 }