add tokenizer

vocab.json

@@ -1 +1 @@
-{"[PAD]": 0, "[UNK]": 1, "á": 2, "r": 3, "с": 4, ")": 5, "я": 6, "j": 7, "…": 8, "s": 9, "ę": 10, "u": 11, "g": 12, "н": 13, "i": 14, "а": 15, "—": 16, "д": 17, "л": 18, "у": 19, "o": 20, "w": 21, "a": 22, "v": 23, "о": 24, "h": 25, "ф": 26, "&": 27, "f": 28, "ш": 29, "ч": 30, "é": 31, "b": 32, "(": 33, "ъ": 34, "y": 35, "q": 36, "б": 37, "–": 38, "’": 39, "ю": 40, "х": 41, "«": 42, "−": 43, "й": 44, "в": 45, " ": 46, "l": 47, "ц": 48, "р": 49, "d": 50, "‑": 51, "p": 52, "ы": 53, "z": 54, "п": 55, "к": 56, "м": 57, "m": 58, "г": 59, "ж": 60, "c": 61, "x": 62, "»": 63, "т": 64, "и": 65, "ō": 66, "t": 67, "ё": 68, "n": 69, "k": 70, "e": 71, "е": 72, "э": 73, "ö": 74, "з": 75, "ä": 76, "ь": 77, "щ": 78}
+{"[PAD]": 0, "[UNK]": 1, "я": 2, "е": 3, "z": 4, "o": 5, "’": 6, "(": 7, "г": 8, "э": 9, "ч": 10, "ю": 11, "y": 12, "ę": 13, "‑": 14, "é": 15, "á": 16, "и": 17, ")": 18, "…": 19, "−": 20, "м": 21, "k": 22, "ö": 23, "щ": 24, "с": 25, "j": 26, "ъ": 27, "ц": 28, "–": 29, "t": 30, "—": 31, "ш": 32, "ф": 33, "r": 34, "ь": 35, "e": 36, "к": 37, "n": 38, "l": 39, "f": 40, "q": 41, "g": 42, "»": 43, "й": 44, "v": 45, "&": 46, "m": 47, "d": 48, "i": 49, "ō": 50, "х": 51, "h": 52, "п": 53, "т": 54, "ы": 55, "c": 56, "ё": 57, "u": 58, "у": 59, "в": 60, "н": 61, "w": 62, "ж": 63, "р": 64, "p": 65, "x": 66, " ": 67, "д": 68, "а": 69, "ä": 70, "з": 71, "s": 72, "б": 73, "о": 74, "«": 75, "л": 76, "b": 77, "a": 78}