Upload tokenizer
a4530bf
verified
|
{ |
|
"<mask>": 395270, |
|
"ace_Arab": 395271, |
|
"ace_Latn": 395272, |
|
"acm_Arab": 395273, |
|
"acq_Arab": 395274, |
|
"aeb_Arab": 395275, |
|
"afr_Latn": 395276, |
|
"ajp_Arab": 395277, |
|
"aka_Latn": 395278, |
|
"als_Latn": 395279, |
|
"amh_Ethi": 395280, |
|
"apc_Arab": 395281, |
|
"arb_Arab": 395282, |
|
"ars_Arab": 395283, |
|
"ary_Arab": 395284, |
|
"arz_Arab": 395285, |
|
"asm_Beng": 395286, |
|
"ast_Latn": 395287, |
|
"awa_Deva": 395288, |
|
"ayr_Latn": 395289, |
|
"azb_Arab": 395290, |
|
"azj_Latn": 395291, |
|
"bak_Cyrl": 395292, |
|
"bam_Latn": 395293, |
|
"ban_Latn": 395294, |
|
"bel_Cyrl": 395295, |
|
"bem_Latn": 395296, |
|
"ben_Beng": 395297, |
|
"bho_Deva": 395298, |
|
"bjn_Arab": 395299, |
|
"bjn_Latn": 395300, |
|
"bod_Tibt": 395301, |
|
"bos_Latn": 395302, |
|
"bug_Latn": 395303, |
|
"bul_Cyrl": 395304, |
|
"bxr_Cyrl": 395305, |
|
"cat_Latn": 395306, |
|
"ceb_Latn": 395307, |
|
"ces_Latn": 395308, |
|
"che_Cyrl": 395309, |
|
"chv_Cyrl": 395310, |
|
"cjk_Latn": 395311, |
|
"ckb_Arab": 395312, |
|
"crh_Latn": 395313, |
|
"cym_Latn": 395314, |
|
"dan_Latn": 395315, |
|
"deu_Latn": 395316, |
|
"dik_Latn": 395317, |
|
"dyu_Latn": 395318, |
|
"dzo_Tibt": 395319, |
|
"ell_Grek": 395320, |
|
"eng_Latn": 395321, |
|
"epo_Latn": 395322, |
|
"est_Latn": 395323, |
|
"eus_Latn": 395324, |
|
"ewe_Latn": 395325, |
|
"fao_Latn": 395326, |
|
"fij_Latn": 395327, |
|
"fin_Latn": 395328, |
|
"fon_Latn": 395329, |
|
"fra_Latn": 395330, |
|
"fur_Latn": 395331, |
|
"fuv_Latn": 395332, |
|
"gaz_Latn": 395333, |
|
"gla_Latn": 395334, |
|
"gle_Latn": 395335, |
|
"glg_Latn": 395336, |
|
"grn_Latn": 395337, |
|
"guj_Gujr": 395338, |
|
"hat_Latn": 395339, |
|
"hau_Latn": 395340, |
|
"heb_Hebr": 395341, |
|
"hin_Deva": 395342, |
|
"hne_Deva": 395343, |
|
"hrv_Latn": 395344, |
|
"hun_Latn": 395345, |
|
"hye_Armn": 395346, |
|
"ibo_Latn": 395347, |
|
"ilo_Latn": 395348, |
|
"ind_Latn": 395349, |
|
"isl_Latn": 395350, |
|
"ita_Latn": 395351, |
|
"jav_Latn": 395352, |
|
"jpn_Jpan": 395353, |
|
"kab_Latn": 395354, |
|
"kac_Latn": 395355, |
|
"kam_Latn": 395356, |
|
"kan_Knda": 395357, |
|
"kas_Arab": 395358, |
|
"kas_Deva": 395359, |
|
"kat_Geor": 395360, |
|
"kaz_Cyrl": 395361, |
|
"kbp_Latn": 395362, |
|
"kea_Latn": 395363, |
|
"khk_Cyrl": 395364, |
|
"khm_Khmr": 395365, |
|
"kik_Latn": 395366, |
|
"kin_Latn": 395367, |
|
"kir_Cyrl": 395368, |
|
"kjh_Cyrl": 395369, |
|
"kmb_Latn": 395370, |
|
"kmr_Latn": 395371, |
|
"knc_Arab": 395372, |
|
"knc_Latn": 395373, |
|
"kon_Latn": 395374, |
|
"kor_Hang": 395375, |
|
"krc_Cyrl": 395376, |
|
"lao_Laoo": 395377, |
|
"lij_Latn": 395378, |
|
"lim_Latn": 395379, |
|
"lin_Latn": 395380, |
|
"lit_Latn": 395381, |
|
"lmo_Latn": 395382, |
|
"ltg_Latn": 395383, |
|
"ltz_Latn": 395384, |
|
"lua_Latn": 395385, |
|
"lug_Latn": 395386, |
|
"luo_Latn": 395387, |
|
"lus_Latn": 395388, |
|
"lvs_Latn": 395389, |
|
"mag_Deva": 395390, |
|
"mai_Deva": 395391, |
|
"mal_Mlym": 395392, |
|
"mar_Deva": 395393, |
|
"mhr_Cyrl": 395394, |
|
"min_Latn": 395395, |
|
"mkd_Cyrl": 395396, |
|
"mlt_Latn": 395397, |
|
"mni_Beng": 395398, |
|
"mos_Latn": 395399, |
|
"mri_Latn": 395400, |
|
"mya_Mymr": 395401, |
|
"myv_Cyrl": 395402, |
|
"nld_Latn": 395403, |
|
"nno_Latn": 395404, |
|
"nob_Latn": 395405, |
|
"npi_Deva": 395406, |
|
"nso_Latn": 395407, |
|
"nus_Latn": 395408, |
|
"nya_Latn": 395409, |
|
"oci_Latn": 395410, |
|
"ory_Orya": 395411, |
|
"pag_Latn": 395412, |
|
"pan_Guru": 395413, |
|
"pap_Latn": 395414, |
|
"pbt_Arab": 395415, |
|
"pes_Arab": 395416, |
|
"plt_Latn": 395417, |
|
"pol_Latn": 395418, |
|
"por_Latn": 395419, |
|
"prs_Arab": 395420, |
|
"quy_Latn": 395421, |
|
"ron_Latn": 395422, |
|
"run_Latn": 395423, |
|
"rus_Cyrl": 395424, |
|
"sag_Latn": 395425, |
|
"san_Deva": 395426, |
|
"sat_Beng": 395427, |
|
"scn_Latn": 395428, |
|
"shn_Mymr": 395429, |
|
"sin_Sinh": 395430, |
|
"slk_Latn": 395431, |
|
"slv_Latn": 395432, |
|
"smo_Latn": 395433, |
|
"sna_Latn": 395434, |
|
"snd_Arab": 395435, |
|
"som_Latn": 395436, |
|
"sot_Latn": 395437, |
|
"spa_Latn": 395438, |
|
"srd_Latn": 395439, |
|
"srp_Cyrl": 395440, |
|
"ssw_Latn": 395441, |
|
"sun_Latn": 395442, |
|
"swe_Latn": 395443, |
|
"swh_Latn": 395444, |
|
"szl_Latn": 395445, |
|
"tam_Taml": 395446, |
|
"taq_Latn": 395447, |
|
"taq_Tfng": 395448, |
|
"tat_Cyrl": 395449, |
|
"tel_Telu": 395450, |
|
"tgk_Cyrl": 395451, |
|
"tgl_Latn": 395452, |
|
"tha_Thai": 395453, |
|
"tir_Ethi": 395454, |
|
"tpi_Latn": 395455, |
|
"tsn_Latn": 395456, |
|
"tso_Latn": 395457, |
|
"tuk_Latn": 395458, |
|
"tum_Latn": 395459, |
|
"tur_Latn": 395460, |
|
"twi_Latn": 395461, |
|
"tyv_Cyrl": 395462, |
|
"tzm_Tfng": 395463, |
|
"uig_Arab": 395464, |
|
"ukr_Cyrl": 395465, |
|
"umb_Latn": 395466, |
|
"urd_Arab": 395467, |
|
"uzn_Latn": 395468, |
|
"vec_Latn": 395469, |
|
"vie_Latn": 395470, |
|
"war_Latn": 395471, |
|
"wol_Latn": 395472, |
|
"xho_Latn": 395473, |
|
"ydd_Hebr": 395474, |
|
"yor_Latn": 395475, |
|
"yue_Hant": 395476, |
|
"zho_Hans": 395477, |
|
"zho_Hant": 395478, |
|
"zsm_Latn": 395479, |
|
"zul_Latn": 395480 |
|
} |
|
|