add tokenizer

added_tokens.json

@@ -1 +1 @@
-{"<s>": 86, "</s>": 87}
+{"<s>": 79, "</s>": 80}

vocab.json

@@ -1 +1 @@
-{"[PAD]": 0, "[UNK]": 1, "в": 2, "é": 3, "ё": 4, "l": 5, "а": 6, "y": 7, "&": 8, "ö": 9, "р": 10, "ó": 11, "з": 12, "n": 13, "α": 14, "д": 15, "ō": 16, "t": 17, "к": 18, "о": 19, "п": 20, "k": 21, "ч": 22, "—": 23, "н": 24, "щ": 25, "−": 26, "(": 27, "w": 28, "z": 29, "x": 30, "d": 31, "‑": 32, "v": 33, "ф": 34, "ь": 35, "ц": 36, "и": 37, "б": 38, "h": 39, "х": 40, "ñ": 41, "ж": 42, "с": 43, "i": 44, "ъ": 45, ")": 46, "ю": 47, "g": 48, "m": 49, "ú": 50, "я": 51, "э": 52, "–": 53, "т": 54, "è": 55, "p": 56, "м": 57, "…": 58, "e": 59, "r": 60, "b": 61, " ": 62, "j": 63, "й": 64, "à": 65, "ä": 66, "ш": 67, "á": 68, "ы": 69, "’": 70, "г": 71, "f": 72, "c": 73, "s": 74, "»": 75, "«": 76, "q": 77, "a": 78, "í": 79, "u": 80, "е": 81, "ę": 82, "у": 83, "л": 84, "o": 85}
+{"[PAD]": 0, "[UNK]": 1, "á": 2, "r": 3, "с": 4, ")": 5, "я": 6, "j": 7, "…": 8, "s": 9, "ę": 10, "u": 11, "g": 12, "н": 13, "i": 14, "а": 15, "—": 16, "д": 17, "л": 18, "у": 19, "o": 20, "w": 21, "a": 22, "v": 23, "о": 24, "h": 25, "ф": 26, "&": 27, "f": 28, "ш": 29, "ч": 30, "é": 31, "b": 32, "(": 33, "ъ": 34, "y": 35, "q": 36, "б": 37, "–": 38, "’": 39, "ю": 40, "х": 41, "«": 42, "−": 43, "й": 44, "в": 45, " ": 46, "l": 47, "ц": 48, "р": 49, "d": 50, "‑": 51, "p": 52, "ы": 53, "z": 54, "п": 55, "к": 56, "м": 57, "m": 58, "г": 59, "ж": 60, "c": 61, "x": 62, "»": 63, "т": 64, "и": 65, "ō": 66, "t": 67, "ё": 68, "n": 69, "k": 70, "e": 71, "е": 72, "э": 73, "ö": 74, "з": 75, "ä": 76, "ь": 77, "щ": 78}