add tokenizer

vocab.json

@@ -1 +1 @@
-{"[PAD]": 0, "[UNK]": 1, "я": 2, "е": 3, "z": 4, "o": 5, "’": 6, "(": 7, "г": 8, "э": 9, "ч": 10, "ю": 11, "y": 12, "ę": 13, "‑": 14, "é": 15, "á": 16, "и": 17, ")": 18, "…": 19, "−": 20, "м": 21, "k": 22, "ö": 23, "щ": 24, "с": 25, "j": 26, "ъ": 27, "ц": 28, "–": 29, "t": 30, "—": 31, "ш": 32, "ф": 33, "r": 34, "ь": 35, "e": 36, "к": 37, "n": 38, "l": 39, "f": 40, "q": 41, "g": 42, "»": 43, "й": 44, "v": 45, "&": 46, "m": 47, "d": 48, "i": 49, "ō": 50, "х": 51, "h": 52, "п": 53, "т": 54, "ы": 55, "c": 56, "ё": 57, "u": 58, "у": 59, "в": 60, "н": 61, "w": 62, "ж": 63, "р": 64, "p": 65, "x": 66, " ": 67, "д": 68, "а": 69, "ä": 70, "з": 71, "s": 72, "б": 73, "о": 74, "«": 75, "л": 76, "b": 77, "a": 78}
+{"[PAD]": 0, "[UNK]": 1, "x": 2, "н": 3, "é": 4, "l": 5, "з": 6, "ю": 7, " ": 8, "s": 9, "e": 10, "и": 11, "й": 12, "n": 13, "…": 14, "ъ": 15, "—": 16, "о": 17, "ь": 18, "c": 19, "p": 20, "т": 21, "’": 22, "в": 23, "я": 24, "»": 25, "b": 26, "с": 27, "ф": 28, "r": 29, "ш": 30, "ö": 31, "–": 32, "g": 33, "«": 34, "j": 35, "л": 36, "у": 37, "ä": 38, "б": 39, "i": 40, "ц": 41, "а": 42, "f": 43, "t": 44, "a": 45, "k": 46, "ō": 47, "m": 48, "ы": 49, "u": 50, "(": 51, "щ": 52, "е": 53, "−": 54, "y": 55, "г": 56, "ж": 57, "á": 58, "ч": 59, "z": 60, "d": 61, "o": 62, "х": 63, "д": 64, "‑": 65, "v": 66, "п": 67, "w": 68, "к": 69, ")": 70, "ę": 71, "м": 72, "ё": 73, "&": 74, "э": 75, "h": 76, "q": 77, "р": 78}