Upload cak_latn_full tokenizer.

added_tokens.json

@@ -0,0 +1 @@
+{"[XXXXX549]": 32680, "[XXXXX572]": 32703, "[XXXXX378]": 32509, "[XXXXX367]": 32498, "[XXXXX4]": 32135, "[XXXXX322]": 32453, "[XXXXX558]": 32689, "[XXXXX339]": 32470, "[XXXXX561]": 32692, "[XXXXX212]": 32343, "[XXXXX75]": 32206, "[XXXXX408]": 32539, "[XXXXX529]": 32660, "[XXXXX380]": 32511, "[XXXXX314]": 32445, "[XXXXX284]": 32415, "[XXXXX337]": 32468, "[XXXXX318]": 32449, "[XXXXX261]": 32392, "[XXXXX199]": 32330, "[XXXXX253]": 32384, "[CLS]": 32127, "[XXXXX499]": 32630, "[XXXXX131]": 32262, "[XXXXX460]": 32591, "[XXXXX556]": 32687, "[XXXXX274]": 32405, "[XXXXX447]": 32578, "[XXXXX215]": 32346, "[XXXXX521]": 32652, "[XXXXX194]": 32325, "[XXXXX601]": 32732, "[XXXXX305]": 32436, "[XXXXX418]": 32549, "[XXXXX118]": 32249, "[XXXXX627]": 32758, "[XXXXX231]": 32362, "[XXXXX487]": 32618, "[XXXXX511]": 32642, "[XXXXX140]": 32271, "[XXXXX358]": 32489, "[XXXXX524]": 32655, "[XXXXX338]": 32469, "[XXXXX224]": 32355, "[XXXXX183]": 32314, "[XXXXX35]": 32166, "[XXXXX376]": 32507, "[XXXXX441]": 32572, "[XXXXX59]": 32190, "[XXXXX13]": 32144, "[XXXXX588]": 32719, "[XXXXX421]": 32552, "[XXXXX138]": 32269, "[XXXXX530]": 32661, "[XXXXX495]": 32626, "[XXXXX238]": 32369, "[XXXXX190]": 32321, "[XXXXX345]": 32476, "[XXXXX271]": 32402, "[XXXXX111]": 32242, "[XXXXX149]": 32280, "[XXXXX455]": 32586, "[XXXXX513]": 32644, "[XXXXX214]": 32345, "[XXXXX97]": 32228, "[XXXXX541]": 32672, "[XXXXX30]": 32161, "[XXXXX363]": 32494, "[XXXXX603]": 32734, "[XXXXX163]": 32294, "[XXXXX615]": 32746, "[XXXXX125]": 32256, "[XXXXX160]": 32291, "[XXXXX475]": 32606, "[XXXXX2]": 32133, "[XXXXX519]": 32650, "[XXXXX73]": 32204, "[XXXXX136]": 32267, "[XXXXX397]": 32528, "[XXXXX158]": 32289, "[XXXXX229]": 32360, "[XXXXX477]": 32608, "[XXXXX592]": 32723, "[XXXXX403]": 32534, "[XXXXX289]": 32420, "[XXXXX552]": 32683, "[XXXXX179]": 32310, "[XXXXX43]": 32174, "[XXXXX611]": 32742, "[XXXXX156]": 32287, "[XXXXX565]": 32696, "[XXXXX161]": 32292, "[XXXXX319]": 32450, "[XXXXX344]": 32475, "[XXXXX121]": 32252, "[XXXXX382]": 32513, "[XXXXX525]": 32656, "[XXXXX633]": 32764, "[XXXXX303]": 32434, "[XXXXX175]": 32306, "[XXXXX527]": 32658, "[XXXXX434]": 32565, "[XXXXX404]": 32535, "[XXXXX64]": 32195, "[XXXXX328]": 32459, "[XXXXX249]": 32380, "[XXXXX593]": 32724, "[XXXXX349]": 32480, "[XXXXX148]": 32279, "[XXXXX12]": 32143, "[XXXXX451]": 32582, "[XXXXX40]": 32171, "[XXXXX336]": 32467, "[XXXXX90]": 32221, "[XXXXX436]": 32567, "[XXXXX466]": 32597, "[XXXXX74]": 32205, "[XXXXX334]": 32465, "[XXXXX402]": 32533, "[XXXXX560]": 32691, "[XXXXX553]": 32684, "[XXXXX147]": 32278, "[XXXXX508]": 32639, "[XXXXX135]": 32266, "[XXXXX152]": 32283, "[XXXXX186]": 32317, "[XXXXX413]": 32544, "[XXXXX9]": 32140, "[XXXXX96]": 32227, "[XXXXX566]": 32697, "[XXXXX189]": 32320, "[XXXXX264]": 32395, "[XXXXX600]": 32731, "[XXXXX597]": 32728, "[XXXXX181]": 32312, "[XXXXX188]": 32319, "[XXXXX634]": 32765, "[XXXXX174]": 32305, "[XXXXX258]": 32389, "[XXXXX424]": 32555, "[XXXXX232]": 32363, "[XXXXX476]": 32607, "[XXXXX512]": 32643, "[XXXXX343]": 32474, "[XXXXX480]": 32611, "[XXXXX398]": 32529, "[XXXXX316]": 32447, "[XXXXX167]": 32298, "[XXXXX193]": 32324, "[XXXXX142]": 32273, "[XXXXX554]": 32685, "[XXXXX465]": 32596, "[XXXXX14]": 32145, "[XXXXX387]": 32518, "[XXXXX222]": 32353, "[XXXXX153]": 32284, "[XXXXX446]": 32577, "[XXXXX119]": 32250, "[XXXXX139]": 32270, "[XXXXX427]": 32558, "[XXXXX241]": 32372, "[XXXXX262]": 32393, "[XXXXX240]": 32371, "[XXXXX391]": 32522, "[XXXXX31]": 32162, "[XXXXX479]": 32610, "[XXXXX365]": 32496, "[XXXXX145]": 32276, "[XXXXX453]": 32584, "[XXXXX624]": 32755, "[XXXXX93]": 32224, "[XXXXX377]": 32508, "[XXXXX276]": 32407, "[XXXXX347]": 32478, "[XXXXX414]": 32545, "[XXXXX191]": 32322, "[XXXXX457]": 32588, "[XXXXX381]": 32512, "[XXXXX606]": 32737, "[XXXXX95]": 32226, "[XXXXX426]": 32557, "[XXXXX299]": 32430, "[XXXXX198]": 32329, "[XXXXX22]": 32153, "[XXXXX239]": 32370, "[XXXXX599]": 32730, "[XXXXX237]": 32368, "[XXXXX297]": 32428, "[XXXXX598]": 32729, "[XXXXX7]": 32138, "[XXXXX80]": 32211, "[XXXXX589]": 32720, "[XXXXX88]": 32219, "[XXXXX295]": 32426, "[XXXXX244]": 32375, "[XXXXX583]": 32714, "[XXXXX626]": 32757, "[XXXXX248]": 32379, "[XXXXX608]": 32739, "[XXXXX33]": 32164, "[XXXXX497]": 32628, "[XXXXX409]": 32540, "[XXXXX182]": 32313, "[XXXXX219]": 32350, "[XXXXX422]": 32553, "[XXXXX270]": 32401, "[XXXXX185]": 32316, "[XXXXX595]": 32726, "[XXXXX333]": 32464, "[XXXXX50]": 32181, "[XXXXX342]": 32473, "[XXXXX501]": 32632, "[XXXXX522]": 32653, "[XXXXX285]": 32416, "[XXXXX227]": 32358, "[XXXXX8]": 32139, "[XXXXX545]": 32676, "[XXXXX481]": 32612, "[XXXXX348]": 32479, "[XXXXX146]": 32277, "[XXXXX113]": 32244, "[XXXXX228]": 32359, "[XXXXX636]": 32767, "[XXXXX371]": 32502, "[XXXXX154]": 32285, "[XXXXX488]": 32619, "[XXXXX49]": 32180, "[XXXXX216]": 32347, "[XXXXX547]": 32678, "[XXXXX204]": 32335, "[XXXXX450]": 32581, "[XXXXX76]": 32207, "[XXXXX631]": 32762, "[XXXXX108]": 32239, "[XXXXX630]": 32761, "[XXXXX18]": 32149, "[XXXXX26]": 32157, "[XXXXX55]": 32186, "[XXXXX568]": 32699, "[XXXXX620]": 32751, "[XXXXX341]": 32472, "[XXXXX180]": 32311, "[XXXXX503]": 32634, "[XXXXX211]": 32342, "[XXXXX288]": 32419, "[MASK]": 32130, "[XXXXX395]": 32526, "[XXXXX582]": 32713, "[XXXXX29]": 32160, "[XXXXX92]": 32223, "[XXXXX67]": 32198, "[XXXXX85]": 32216, "[XXXXX393]": 32524, "[XXXXX575]": 32706, "[XXXXX157]": 32288, "[XXXXX410]": 32541, "[XXXXX196]": 32327, "[XXXXX53]": 32184, "[XXXXX122]": 32253, "[XXXXX245]": 32376, "[XXXXX471]": 32602, "[XXXXX110]": 32241, "[XXXXX440]": 32571, "[XXXXX370]": 32501, "[XXXXX412]": 32543, "[XXXXX94]": 32225, "[XXXXX411]": 32542, "[XXXXX69]": 32200, "[XXXXX133]": 32264, "[XXXXX586]": 32717, "[SEP]": 32128, "[XXXXX162]": 32293, "[XXXXX134]": 32265, "[XXXXX384]": 32515, "[XXXXX433]": 32564, "[XXXXX38]": 32169, "[XXXXX459]": 32590, "[XXXXX407]": 32538, "[XXXXX124]": 32255, "[XXXXX268]": 32399, "[XXXXX195]": 32326, "[XXXXX66]": 32197, "[XXXXX470]": 32601, "[XXXXX209]": 32340, "[XXXXX263]": 32394, "[XXXXX394]": 32525, "[XXXXX548]": 32679, "[XXXXX226]": 32357, "[XXXXX520]": 32651, "[XXXXX234]": 32365, "[XXXXX551]": 32682, "[XXXXX104]": 32235, "[XXXXX510]": 32641, "[XXXXX141]": 32272, "[XXXXX562]": 32693, "[XXXXX432]": 32563, "[XXXXX625]": 32756, "[XXXXX502]": 32633, "[XXXXX445]": 32576, "[XXXXX255]": 32386, "[XXXXX250]": 32381, "[XXXXX478]": 32609, "[XXXXX330]": 32461, "[XXXXX485]": 32616, "[XXXXX564]": 32695, "[XXXXX490]": 32621, "[XXXXX498]": 32629, "[XXXXX464]": 32595, "[XXXXX103]": 32234, "[XXXXX309]": 32440, "[XXXXX526]": 32657, "[XXXXX37]": 32168, "[XXXXX102]": 32233, "[XXXXX437]": 32568, "[XXXXX60]": 32191, "[XXXXX516]": 32647, "[XXXXX116]": 32247, "[XXXXX306]": 32437, "[XXXXX259]": 32390, "[XXXXX506]": 32637, "[XXXXX203]": 32334, "[XXXXX383]": 32514, "[XXXXX287]": 32418, "[XXXXX591]": 32722, "[XXXXX132]": 32263, "[XXXXX77]": 32208, "[XXXXX63]": 32194, "[XXXXX619]": 32750, "[XXXXX106]": 32237, "[XXXXX208]": 32339, "[XXXXX587]": 32718, "[XXXXX385]": 32516, "[XXXXX172]": 32303, "[XXXXX46]": 32177, "[XXXXX610]": 32741, "[XXXXX310]": 32441, "[XXXXX296]": 32427, "[XXXXX439]": 32570, "[XXXXX623]": 32754, "[XXXXX392]": 32523, "[XXXXX177]": 32308, "[XXXXX86]": 32217, "[XXXXX621]": 32752, "[XXXXX206]": 32337, "[XXXXX507]": 32638, "[XXXXX540]": 32671, "[XXXXX200]": 32331, "[XXXXX361]": 32492, "[XXXXX581]": 32712, "[XXXXX574]": 32705, "[XXXXX272]": 32403, "[XXXXX405]": 32536, "[XXXXX579]": 32710, "[XXXXX364]": 32495, "[XXXXX129]": 32260, "[XXXXX137]": 32268, "[XXXXX327]": 32458, "[XXXXX45]": 32176, "[XXXXX618]": 32749, "[XXXXX114]": 32245, "[XXXXX151]": 32282, "[XXXXX399]": 32530, "[XXXXX467]": 32598, "[XXXXX166]": 32297, "[XXXXX372]": 32503, "[XXXXX201]": 32332, "[XXXXX187]": 32318, "[XXXXX243]": 32374, "[XXXXX242]": 32373, "[XXXXX458]": 32589, "[XXXXX469]": 32600, "[XXXXX286]": 32417, "[XXXXX462]": 32593, "[XXXXX220]": 32351, "[XXXXX207]": 32338, "[XXXXX632]": 32763, "[XXXXX483]": 32614, "[XXXXX406]": 32537, "[XXXXX532]": 32663, "[XXXXX225]": 32356, "[XXXXX614]": 32745, "[XXXXX56]": 32187, "[XXXXX143]": 32274, "[XXXXX81]": 32212, "[XXXXX267]": 32398, "[XXXXX269]": 32400, "[XXXXX335]": 32466, "[XXXXX44]": 32175, "[XXXXX359]": 32490, "[XXXXX202]": 32333, "[XXXXX25]": 32156, "[XXXXX308]": 32439, "[XXXXX178]": 32309, "[XXXXX91]": 32222, "[XXXXX34]": 32165, "[XXXXX48]": 32179, "[XXXXX428]": 32559, "[XXXXX168]": 32299, "[XXXXX205]": 32336, "[XXXXX223]": 32354, "[XXXXX293]": 32424, "[XXXXX449]": 32580, "[XXXXX609]": 32740, "[XXXXX159]": 32290, "[XXXXX415]": 32546, "[XXXXX353]": 32484, "[XXXXX531]": 32662, "[XXXXX0]": 32131, "[XXXXX24]": 32155, "[XXXXX130]": 32261, "[XXXXX596]": 32727, "[XXXXX604]": 32735, "[XXXXX313]": 32444, "[XXXXX252]": 32383, "[XXXXX120]": 32251, "[XXXXX419]": 32550, "[XXXXX420]": 32551, "[XXXXX109]": 32240, "[XXXXX83]": 32214, "[XXXXX509]": 32640, "[XXXXX628]": 32759, "[XXXXX78]": 32209, "[XXXXX210]": 32341, "[XXXXX217]": 32348, "[XXXXX607]": 32738, "[XXXXX504]": 32635, "[XXXXX431]": 32562, "[XXXXX294]": 32425, "[XXXXX500]": 32631, "[XXXXX54]": 32185, "[XXXXX36]": 32167, "[XXXXX302]": 32433, "[XXXXX87]": 32218, "[XXXXX256]": 32387, "[XXXXX32]": 32163, "[XXXXX6]": 32137, "[XXXXX315]": 32446, "[XXXXX21]": 32152, "[XXXXX416]": 32547, "[XXXXX573]": 32704, "[XXXXX277]": 32408, "[XXXXX47]": 32178, "[XXXXX82]": 32213, "[XXXXX72]": 32203, "[XXXXX107]": 32238, "[XXXXX128]": 32259, "[XXXXX559]": 32690, "[XXXXX51]": 32182, "[XXXXX89]": 32220, "[XXXXX105]": 32236, "[XXXXX39]": 32170, "[XXXXX312]": 32443, "[XXXXX534]": 32665, "[XXXXX68]": 32199, "[XXXXX388]": 32519, "[XXXXX567]": 32698, "[XXXXX115]": 32246, "[XXXXX539]": 32670, "[XXXXX27]": 32158, "[XXXXX550]": 32681, "[XXXXX352]": 32483, "[XXXXX236]": 32367, "[XXXXX260]": 32391, "[XXXXX266]": 32397, "[XXXXX173]": 32304, "[XXXXX304]": 32435, "[XXXXX213]": 32344, "[XXXXX584]": 32715, "[XXXXX474]": 32605, "[XXXXX454]": 32585, "[XXXXX298]": 32429, "[XXXXX612]": 32743, "[XXXXX235]": 32366, "[XXXXX275]": 32406, "[XXXXX292]": 32423, "[XXXXX317]": 32448, "[XXXXX247]": 32378, "[XXXXX514]": 32645, "[XXXXX280]": 32411, "<pad>": 32129, "[XXXXX329]": 32460, "[XXXXX16]": 32147, "[XXXXX10]": 32141, "[XXXXX71]": 32202, "[XXXXX472]": 32603, "[XXXXX61]": 32192, "[XXXXX429]": 32560, "[XXXXX169]": 32300, "[XXXXX350]": 32481, "[XXXXX602]": 32733, "[XXXXX41]": 32172, "[XXXXX569]": 32700, "[XXXXX629]": 32760, "[XXXXX300]": 32431, "[XXXXX218]": 32349, "[XXXXX390]": 32521, "[XXXXX254]": 32385, "[XXXXX57]": 32188, "[XXXXX99]": 32230, "[XXXXX84]": 32215, "[XXXXX230]": 32361, "[XXXXX301]": 32432, "[XXXXX307]": 32438, "[XXXXX616]": 32747, "[XXXXX542]": 32673, "[XXXXX613]": 32744, "[XXXXX100]": 32231, "[XXXXX28]": 32159, "[XXXXX443]": 32574, "[XXXXX282]": 32413, "[XXXXX468]": 32599, "[XXXXX332]": 32463, "[XXXXX463]": 32594, "[XXXXX492]": 32623, "[XXXXX144]": 32275, "[XXXXX355]": 32486, "[XXXXX257]": 32388, "[XXXXX326]": 32457, "[XXXXX417]": 32548, "[XXXXX19]": 32150, "[XXXXX320]": 32451, "[XXXXX3]": 32134, "[XXXXX273]": 32404, "[XXXXX283]": 32414, "[XXXXX98]": 32229, "[XXXXX533]": 32664, "[XXXXX325]": 32456, "[XXXXX291]": 32422, "[XXXXX42]": 32173, "[XXXXX331]": 32462, "[XXXXX117]": 32248, "[XXXXX278]": 32409, "[XXXXX435]": 32566, "[XXXXX557]": 32688, "[XXXXX456]": 32587, "[XXXXX486]": 32617, "[XXXXX126]": 32257, "[XXXXX192]": 32323, "[XXXXX58]": 32189, "[XXXXX311]": 32442, "[XXXXX246]": 32377, "[XXXXX535]": 32666, "[XXXXX578]": 32709, "[XXXXX281]": 32412, "[XXXXX323]": 32454, "[XXXXX571]": 32702, "[XXXXX496]": 32627, "[XXXXX622]": 32753, "[XXXXX563]": 32694, "[XXXXX425]": 32556, "[XXXXX52]": 32183, "[XXXXX251]": 32382, "[XXXXX233]": 32364, "[XXXXX368]": 32499, "[XXXXX23]": 32154, "[XXXXX538]": 32669, "[XXXXX70]": 32201, "[XXXXX17]": 32148, "[XXXXX577]": 32708, "[XXXXX5]": 32136, "[XXXXX605]": 32736, "[XXXXX324]": 32455, "[XXXXX369]": 32500, "[XXXXX15]": 32146, "[XXXXX101]": 32232, "[XXXXX123]": 32254, "[XXXXX430]": 32561, "[XXXXX164]": 32295, "[XXXXX20]": 32151, "[XXXXX423]": 32554, "[XXXXX617]": 32748, "[XXXXX197]": 32328, "[XXXXX375]": 32506, "[XXXXX171]": 32302, "[XXXXX65]": 32196, "[XXXXX515]": 32646, "[XXXXX184]": 32315, "[XXXXX494]": 32625, "[XXXXX265]": 32396, "[XXXXX354]": 32485, "[XXXXX79]": 32210, "[XXXXX11]": 32142, "[XXXXX279]": 32410, "[XXXXX576]": 32707, "[XXXXX444]": 32575, "[XXXXX544]": 32675, "[XXXXX127]": 32258, "[XXXXX517]": 32648, "[XXXXX590]": 32721, "[XXXXX537]": 32668, "[XXXXX155]": 32286, "[XXXXX112]": 32243, "[XXXXX442]": 32573, "[XXXXX493]": 32624, "[XXXXX346]": 32477, "[XXXXX491]": 32622, "[XXXXX518]": 32649, "[XXXXX176]": 32307, "[XXXXX438]": 32569, "[XXXXX165]": 32296, "[XXXXX555]": 32686, "[XXXXX585]": 32716, "[XXXXX374]": 32505, "[XXXXX170]": 32301, "[XXXXX570]": 32701, "[XXXXX150]": 32281, "[XXXXX523]": 32654, "[XXXXX484]": 32615, "[XXXXX389]": 32520, "[XXXXX386]": 32517, "[XXXXX528]": 32659, "[XXXXX379]": 32510, "[XXXXX360]": 32491, "[XXXXX546]": 32677, "[XXXXX452]": 32583, "[XXXXX396]": 32527, "[XXXXX366]": 32497, "[XXXXX373]": 32504, "[XXXXX482]": 32613, "[XXXXX635]": 32766, "[XXXXX357]": 32488, "[XXXXX221]": 32352, "[XXXXX290]": 32421, "[XXXXX62]": 32193, "[XXXXX356]": 32487, "[XXXXX362]": 32493, "[XXXXX400]": 32531, "[XXXXX448]": 32579, "[XXXXX489]": 32620, "[XXXXX543]": 32674, "[XXXXX505]": 32636, "[XXXXX594]": 32725, "[XXXXX321]": 32452, "[XXXXX1]": 32132, "[XXXXX536]": 32667, "[XXXXX401]": 32532, "[XXXXX461]": 32592, "[XXXXX580]": 32711, "[XXXXX351]": 32482, "[XXXXX340]": 32471, "[XXXXX473]": 32604}

special_tokens_map.json

@@ -0,0 +1 @@
+{"bos_token": "[CLS]", "eos_token": "[SEP]", "unk_token": "<unk>", "sep_token": "[SEP]", "pad_token": "<pad>", "cls_token": "[CLS]", "mask_token": {"content": "[MASK]", "single_word": false, "lstrip": true, "rstrip": false, "normalized": false}, "additional_special_tokens": ["[XXXXX0]", "[XXXXX1]", "[XXXXX2]", "[XXXXX3]", "[XXXXX4]", "[XXXXX5]", "[XXXXX6]", "[XXXXX7]", "[XXXXX8]", "[XXXXX9]", "[XXXXX10]", "[XXXXX11]", "[XXXXX12]", "[XXXXX13]", "[XXXXX14]", "[XXXXX15]", "[XXXXX16]", "[XXXXX17]", "[XXXXX18]", "[XXXXX19]", "[XXXXX20]", "[XXXXX21]", "[XXXXX22]", "[XXXXX23]", "[XXXXX24]", "[XXXXX25]", "[XXXXX26]", "[XXXXX27]", "[XXXXX28]", "[XXXXX29]", "[XXXXX30]", "[XXXXX31]", "[XXXXX32]", "[XXXXX33]", "[XXXXX34]", "[XXXXX35]", "[XXXXX36]", "[XXXXX37]", "[XXXXX38]", "[XXXXX39]", "[XXXXX40]", "[XXXXX41]", "[XXXXX42]", "[XXXXX43]", "[XXXXX44]", "[XXXXX45]", "[XXXXX46]", "[XXXXX47]", "[XXXXX48]", "[XXXXX49]", "[XXXXX50]", "[XXXXX51]", "[XXXXX52]", "[XXXXX53]", "[XXXXX54]", "[XXXXX55]", "[XXXXX56]", "[XXXXX57]", "[XXXXX58]", "[XXXXX59]", "[XXXXX60]", "[XXXXX61]", "[XXXXX62]", "[XXXXX63]", "[XXXXX64]", "[XXXXX65]", "[XXXXX66]", "[XXXXX67]", "[XXXXX68]", "[XXXXX69]", "[XXXXX70]", "[XXXXX71]", "[XXXXX72]", "[XXXXX73]", "[XXXXX74]", "[XXXXX75]", "[XXXXX76]", "[XXXXX77]", "[XXXXX78]", "[XXXXX79]", "[XXXXX80]", "[XXXXX81]", "[XXXXX82]", "[XXXXX83]", "[XXXXX84]", "[XXXXX85]", "[XXXXX86]", "[XXXXX87]", "[XXXXX88]", "[XXXXX89]", "[XXXXX90]", "[XXXXX91]", "[XXXXX92]", "[XXXXX93]", "[XXXXX94]", "[XXXXX95]", "[XXXXX96]", "[XXXXX97]", "[XXXXX98]", "[XXXXX99]", "[XXXXX100]", "[XXXXX101]", "[XXXXX102]", "[XXXXX103]", "[XXXXX104]", "[XXXXX105]", "[XXXXX106]", "[XXXXX107]", "[XXXXX108]", "[XXXXX109]", "[XXXXX110]", "[XXXXX111]", "[XXXXX112]", "[XXXXX113]", "[XXXXX114]", "[XXXXX115]", "[XXXXX116]", "[XXXXX117]", "[XXXXX118]", "[XXXXX119]", "[XXXXX120]", "[XXXXX121]", "[XXXXX122]", "[XXXXX123]", "[XXXXX124]", "[XXXXX125]", "[XXXXX126]", "[XXXXX127]", "[XXXXX128]", "[XXXXX129]", "[XXXXX130]", "[XXXXX131]", "[XXXXX132]", "[XXXXX133]", "[XXXXX134]", "[XXXXX135]", "[XXXXX136]", "[XXXXX137]", "[XXXXX138]", "[XXXXX139]", "[XXXXX140]", "[XXXXX141]", "[XXXXX142]", "[XXXXX143]", "[XXXXX144]", "[XXXXX145]", "[XXXXX146]", "[XXXXX147]", "[XXXXX148]", "[XXXXX149]", "[XXXXX150]", "[XXXXX151]", "[XXXXX152]", "[XXXXX153]", "[XXXXX154]", "[XXXXX155]", "[XXXXX156]", "[XXXXX157]", "[XXXXX158]", "[XXXXX159]", "[XXXXX160]", "[XXXXX161]", "[XXXXX162]", "[XXXXX163]", "[XXXXX164]", "[XXXXX165]", "[XXXXX166]", "[XXXXX167]", "[XXXXX168]", "[XXXXX169]", "[XXXXX170]", "[XXXXX171]", "[XXXXX172]", "[XXXXX173]", "[XXXXX174]", "[XXXXX175]", "[XXXXX176]", "[XXXXX177]", "[XXXXX178]", "[XXXXX179]", "[XXXXX180]", "[XXXXX181]", "[XXXXX182]", "[XXXXX183]", "[XXXXX184]", "[XXXXX185]", "[XXXXX186]", "[XXXXX187]", "[XXXXX188]", "[XXXXX189]", "[XXXXX190]", "[XXXXX191]", "[XXXXX192]", "[XXXXX193]", "[XXXXX194]", "[XXXXX195]", "[XXXXX196]", "[XXXXX197]", "[XXXXX198]", "[XXXXX199]", "[XXXXX200]", "[XXXXX201]", "[XXXXX202]", "[XXXXX203]", "[XXXXX204]", "[XXXXX205]", "[XXXXX206]", "[XXXXX207]", "[XXXXX208]", "[XXXXX209]", "[XXXXX210]", "[XXXXX211]", "[XXXXX212]", "[XXXXX213]", "[XXXXX214]", "[XXXXX215]", "[XXXXX216]", "[XXXXX217]", "[XXXXX218]", "[XXXXX219]", "[XXXXX220]", "[XXXXX221]", "[XXXXX222]", "[XXXXX223]", "[XXXXX224]", "[XXXXX225]", "[XXXXX226]", "[XXXXX227]", "[XXXXX228]", "[XXXXX229]", "[XXXXX230]", "[XXXXX231]", "[XXXXX232]", "[XXXXX233]", "[XXXXX234]", "[XXXXX235]", "[XXXXX236]", "[XXXXX237]", "[XXXXX238]", "[XXXXX239]", "[XXXXX240]", "[XXXXX241]", "[XXXXX242]", "[XXXXX243]", "[XXXXX244]", "[XXXXX245]", "[XXXXX246]", "[XXXXX247]", "[XXXXX248]", "[XXXXX249]", "[XXXXX250]", "[XXXXX251]", "[XXXXX252]", "[XXXXX253]", "[XXXXX254]", "[XXXXX255]", "[XXXXX256]", "[XXXXX257]", "[XXXXX258]", "[XXXXX259]", "[XXXXX260]", "[XXXXX261]", "[XXXXX262]", "[XXXXX263]", "[XXXXX264]", "[XXXXX265]", "[XXXXX266]", "[XXXXX267]", "[XXXXX268]", "[XXXXX269]", "[XXXXX270]", "[XXXXX271]", "[XXXXX272]", "[XXXXX273]", "[XXXXX274]", "[XXXXX275]", "[XXXXX276]", "[XXXXX277]", "[XXXXX278]", "[XXXXX279]", "[XXXXX280]", "[XXXXX281]", "[XXXXX282]", "[XXXXX283]", "[XXXXX284]", "[XXXXX285]", "[XXXXX286]", "[XXXXX287]", "[XXXXX288]", "[XXXXX289]", "[XXXXX290]", "[XXXXX291]", "[XXXXX292]", "[XXXXX293]", "[XXXXX294]", "[XXXXX295]", "[XXXXX296]", "[XXXXX297]", "[XXXXX298]", "[XXXXX299]", "[XXXXX300]", "[XXXXX301]", "[XXXXX302]", "[XXXXX303]", "[XXXXX304]", "[XXXXX305]", "[XXXXX306]", "[XXXXX307]", "[XXXXX308]", "[XXXXX309]", "[XXXXX310]", "[XXXXX311]", "[XXXXX312]", "[XXXXX313]", "[XXXXX314]", "[XXXXX315]", "[XXXXX316]", "[XXXXX317]", "[XXXXX318]", "[XXXXX319]", "[XXXXX320]", "[XXXXX321]", "[XXXXX322]", "[XXXXX323]", "[XXXXX324]", "[XXXXX325]", "[XXXXX326]", "[XXXXX327]", "[XXXXX328]", "[XXXXX329]", "[XXXXX330]", "[XXXXX331]", "[XXXXX332]", "[XXXXX333]", "[XXXXX334]", "[XXXXX335]", "[XXXXX336]", "[XXXXX337]", "[XXXXX338]", "[XXXXX339]", "[XXXXX340]", "[XXXXX341]", "[XXXXX342]", "[XXXXX343]", "[XXXXX344]", "[XXXXX345]", "[XXXXX346]", "[XXXXX347]", "[XXXXX348]", "[XXXXX349]", "[XXXXX350]", "[XXXXX351]", "[XXXXX352]", "[XXXXX353]", "[XXXXX354]", "[XXXXX355]", "[XXXXX356]", "[XXXXX357]", "[XXXXX358]", "[XXXXX359]", "[XXXXX360]", "[XXXXX361]", "[XXXXX362]", "[XXXXX363]", "[XXXXX364]", "[XXXXX365]", "[XXXXX366]", "[XXXXX367]", "[XXXXX368]", "[XXXXX369]", "[XXXXX370]", "[XXXXX371]", "[XXXXX372]", "[XXXXX373]", "[XXXXX374]", "[XXXXX375]", "[XXXXX376]", "[XXXXX377]", "[XXXXX378]", "[XXXXX379]", "[XXXXX380]", "[XXXXX381]", "[XXXXX382]", "[XXXXX383]", "[XXXXX384]", "[XXXXX385]", "[XXXXX386]", "[XXXXX387]", "[XXXXX388]", "[XXXXX389]", "[XXXXX390]", "[XXXXX391]", "[XXXXX392]", "[XXXXX393]", "[XXXXX394]", "[XXXXX395]", "[XXXXX396]", "[XXXXX397]", "[XXXXX398]", "[XXXXX399]", "[XXXXX400]", "[XXXXX401]", "[XXXXX402]", "[XXXXX403]", "[XXXXX404]", "[XXXXX405]", "[XXXXX406]", "[XXXXX407]", "[XXXXX408]", "[XXXXX409]", "[XXXXX410]", "[XXXXX411]", "[XXXXX412]", "[XXXXX413]", "[XXXXX414]", "[XXXXX415]", "[XXXXX416]", "[XXXXX417]", "[XXXXX418]", "[XXXXX419]", "[XXXXX420]", "[XXXXX421]", "[XXXXX422]", "[XXXXX423]", "[XXXXX424]", "[XXXXX425]", "[XXXXX426]", "[XXXXX427]", "[XXXXX428]", "[XXXXX429]", "[XXXXX430]", "[XXXXX431]", "[XXXXX432]", "[XXXXX433]", "[XXXXX434]", "[XXXXX435]", "[XXXXX436]", "[XXXXX437]", "[XXXXX438]", "[XXXXX439]", "[XXXXX440]", "[XXXXX441]", "[XXXXX442]", "[XXXXX443]", "[XXXXX444]", "[XXXXX445]", "[XXXXX446]", "[XXXXX447]", "[XXXXX448]", "[XXXXX449]", "[XXXXX450]", "[XXXXX451]", "[XXXXX452]", "[XXXXX453]", "[XXXXX454]", "[XXXXX455]", "[XXXXX456]", "[XXXXX457]", "[XXXXX458]", "[XXXXX459]", "[XXXXX460]", "[XXXXX461]", "[XXXXX462]", "[XXXXX463]", "[XXXXX464]", "[XXXXX465]", "[XXXXX466]", "[XXXXX467]", "[XXXXX468]", "[XXXXX469]", "[XXXXX470]", "[XXXXX471]", "[XXXXX472]", "[XXXXX473]", "[XXXXX474]", "[XXXXX475]", "[XXXXX476]", "[XXXXX477]", "[XXXXX478]", "[XXXXX479]", "[XXXXX480]", "[XXXXX481]", "[XXXXX482]", "[XXXXX483]", "[XXXXX484]", "[XXXXX485]", "[XXXXX486]", "[XXXXX487]", "[XXXXX488]", "[XXXXX489]", "[XXXXX490]", "[XXXXX491]", "[XXXXX492]", "[XXXXX493]", "[XXXXX494]", "[XXXXX495]", "[XXXXX496]", "[XXXXX497]", "[XXXXX498]", "[XXXXX499]", "[XXXXX500]", "[XXXXX501]", "[XXXXX502]", "[XXXXX503]", "[XXXXX504]", "[XXXXX505]", "[XXXXX506]", "[XXXXX507]", "[XXXXX508]", "[XXXXX509]", "[XXXXX510]", "[XXXXX511]", "[XXXXX512]", "[XXXXX513]", "[XXXXX514]", "[XXXXX515]", "[XXXXX516]", "[XXXXX517]", "[XXXXX518]", "[XXXXX519]", "[XXXXX520]", "[XXXXX521]", "[XXXXX522]", "[XXXXX523]", "[XXXXX524]", "[XXXXX525]", "[XXXXX526]", "[XXXXX527]", "[XXXXX528]", "[XXXXX529]", "[XXXXX530]", "[XXXXX531]", "[XXXXX532]", "[XXXXX533]", "[XXXXX534]", "[XXXXX535]", "[XXXXX536]", "[XXXXX537]", "[XXXXX538]", "[XXXXX539]", "[XXXXX540]", "[XXXXX541]", "[XXXXX542]", "[XXXXX543]", "[XXXXX544]", "[XXXXX545]", "[XXXXX546]", "[XXXXX547]", "[XXXXX548]", "[XXXXX549]", "[XXXXX550]", "[XXXXX551]", "[XXXXX552]", "[XXXXX553]", "[XXXXX554]", "[XXXXX555]", "[XXXXX556]", "[XXXXX557]", "[XXXXX558]", "[XXXXX559]", "[XXXXX560]", "[XXXXX561]", "[XXXXX562]", "[XXXXX563]", "[XXXXX564]", "[XXXXX565]", "[XXXXX566]", "[XXXXX567]", "[XXXXX568]", "[XXXXX569]", "[XXXXX570]", "[XXXXX571]", "[XXXXX572]", "[XXXXX573]", "[XXXXX574]", "[XXXXX575]", "[XXXXX576]", "[XXXXX577]", "[XXXXX578]", "[XXXXX579]", "[XXXXX580]", "[XXXXX581]", "[XXXXX582]", "[XXXXX583]", "[XXXXX584]", "[XXXXX585]", "[XXXXX586]", "[XXXXX587]", "[XXXXX588]", "[XXXXX589]", "[XXXXX590]", "[XXXXX591]", "[XXXXX592]", "[XXXXX593]", "[XXXXX594]", "[XXXXX595]", "[XXXXX596]", "[XXXXX597]", "[XXXXX598]", "[XXXXX599]", "[XXXXX600]", "[XXXXX601]", "[XXXXX602]", "[XXXXX603]", "[XXXXX604]", "[XXXXX605]", "[XXXXX606]", "[XXXXX607]", "[XXXXX608]", "[XXXXX609]", "[XXXXX610]", "[XXXXX611]", "[XXXXX612]", "[XXXXX613]", "[XXXXX614]", "[XXXXX615]", "[XXXXX616]", "[XXXXX617]", "[XXXXX618]", "[XXXXX619]", "[XXXXX620]", "[XXXXX621]", "[XXXXX622]", "[XXXXX623]", "[XXXXX624]", "[XXXXX625]", "[XXXXX626]", "[XXXXX627]", "[XXXXX628]", "[XXXXX629]", "[XXXXX630]", "[XXXXX631]", "[XXXXX632]", "[XXXXX633]", "[XXXXX634]", "[XXXXX635]", "[XXXXX636]"]}

spiece.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6f3191c87a1f8e53e8c329dcb6e57bba9f7d3daceb4f42714999b7d6fc75bce0
+size 804070

tokenizer_config.json

@@ -0,0 +1 @@
+{"do_lower_case": false, "remove_space": true, "keep_accents": true, "bos_token": "[CLS]", "eos_token": "[SEP]", "unk_token": "<unk>", "sep_token": "[SEP]", "pad_token": "<pad>", "cls_token": "[CLS]", "mask_token": {"content": "[MASK]", "single_word": false, "lstrip": true, "rstrip": false, "normalized": false, "__type": "AddedToken"}, "sp_model_kwargs": {}, "name_or_path": "models/full/cak_latn_full", "model_input_names": ["input_ids", "attention_mask"], "special_tokens_map_file": "models/full/cak_latn_full/special_tokens_map.json", "tokenizer_class": "AlbertTokenizer"}