lang_bpe_500/tokens.txt

<blk> 0
<sos/eos> 1
<unk> 2
S 3
▁THE 4
T 5
E 6
▁A 7
ED 8
D 9
▁AND 10
▁TO 11
▁OF 12
ING 13
N 14
▁IN 15
Y 16
R 17
A 18
O 19
C 20
L 21
P 22
I 23
▁HE 24
M 25
▁I 26
▁S 27
▁F 28
▁WAS 29
U 30
▁ 31
▁THAT 32
▁IT 33
ER 34
G 35
AR 36
RE 37
LY 38
B 39
IN 40
OR 41
▁BE 42
▁FOR 43
▁HIS 44
' 45
F 46
▁P 47
▁RE 48
LE 49
EN 50
H 51
▁WITH 52
▁HAD 53
ON 54
▁YOU 55
▁AS 56
AL 57
W 58
IT 59
ST 60
▁HER 61
V 62
LL 63
▁T 64
▁SO 65
CE 66
▁C 67
▁W 68
TH 69
RI 70
▁NOT 71
▁ST 72
▁D 73
▁BUT 74
▁ON 75
▁SHE 76
TER 77
K 78
▁ME 79
CH 80
▁G 81
▁IS 82
▁DE 83
▁AT 84
▁B 85
IL 86
IC 87
UR 88
VE 89
ENT 90
IR 91
ES 92
▁E 93
OW 94
▁HIM 95
RO 96
▁NO 97
▁WE 98
RA 99
▁MY 100
▁ALL 101
▁HAVE 102
▁BY 103
NE 104
LA 105
▁THIS 106
▁WERE 107
AT 108
EL 109
▁WHICH 110
US 111
▁THEY 112
▁FROM 113
▁DO 114
▁WHO 115
CK 116
▁AN 117
▁LA 118
▁SAID 119
▁CON 120
▁TH 121
▁ONE 122
AN 123
▁OR 124
▁CA 125
SE 126
▁EX 127
▁GO 128
▁R 129
▁MAN 130
AD 131
PP 132
▁MO 133
AS 134
ATION 135
ION 136
▁SE 137
▁O 138
▁THERE 139
▁BO 140
TION 141
LO 142
▁MA 143
VER 144
UN 145
OL 146
▁WHEN 147
TE 148
▁CO 149
▁PRO 150
▁EN 151
▁SU 152
▁WOULD 153
ET 154
▁THEIR 155
▁PA 156
RY 157
▁WHAT 158
▁SH 159
▁UN 160
▁H 161
▁SP 162
▁CH 163
▁LI 164
ISH 165
▁OUT 166
IVE 167
▁BR 168
▁HA 169
▁IF 170
IS 171
ID 172
▁UP 173
▁FA 174
ANT 175
▁RO 176
▁BEEN 177
ARD 178
AND 179
IGHT 180
ATE 181
▁LO 182
▁K 183
▁ARE 184
EST 185
▁LE 186
▁WILL 187
UT 188
▁THEM 189
URE 190
MENT 191
AM 192
UL 193
▁NE 194
▁COULD 195
▁SEE 196
▁DIS 197
ANCE 198
THER 199
▁MI 200
MP 201
ENCE 202
AGE 203
▁WA 204
▁PO 205
▁SOME 206
ITY 207
KE 208
▁US 209
UND 210
GE 211
▁INTO 212
▁MORE 213
▁HO 214
CO 215
IM 216
ABLE 217
IF 218
LU 219
▁DID 220
IDE 221
OM 222
IG 223
▁TIME 224
▁OTHER 225
IE 226
UGH 227
▁YOUR 228
IST 229
▁LIKE 230
▁THAN 231
▁SA 232
▁KNOW 233
▁CR 234
▁BA 235
▁VERY 236
TEN 237
▁NOW 238
NESS 239
ERS 240
▁LITTLE 241
PER 242
IES 243
OUR 244
▁DAY 245
▁ABOUT 246
VI 247
▁PRE 248
TY 249
▁LOOK 250
FUL 251
▁VI 252
▁AFTER 253
▁OVER 254
▁HAS 255
▁CAN 256
BLE 257
ME 258
▁ANY 259
▁AGAIN 260
▁COM 261
X 262
Z 263
▁GREAT 264
▁UPON 265
▁HAND 266
OUS 267
▁WELL 268
▁THEN 269
▁OUR 270
DER 271
INE 272
▁TWO 273
▁ONLY 274
▁CAR 275
▁HU 276
RESS 277
▁GA 278
IOUS 279
▁GRA 280
▁SAY 281
▁EVERY 282
▁MAR 283
▁SHOULD 284
▁MADE 285
▁DOWN 286
▁EVEN 287
▁BEFORE 288
▁MISS 289
▁OLD 290
WARD 291
TI 292
QUI 293
▁GOOD 294
▁WOR 295
▁LONG 296
ANG 297
AK 298
LED 299
▁SUCH 300
ACT 301
MAN 302
LESS 303
▁BU 304
▁WAY 305
▁CAME 306
ALLY 307
▁OFF 308
▁UNDER 309
▁COMP 310
UM 311
▁NEVER 312
ATED 313
▁HOW 314
▁MISTER 315
▁COME 316
▁MUCH 317
▁WHERE 318
J 319
▁BACK 320
▁MEN 321
▁DON 322
▁DI 323
▁MUST 324
▁TAKE 325
IAN 326
▁HOUSE 327
▁JUST 328
▁THESE 329
▁TRA 330
▁IMP 331
▁OWN 332
▁THINK 333
▁THOUGHT 334
▁RI 335
▁CHA 336
▁FIRST 337
▁VA 338
▁MAKE 339
▁MAY 340
▁MIGHT 341
▁NA 342
▁HIMSELF 343
QUA 344
▁PART 345
▁THROUGH 346
▁PER 347
▁PLACE 348
IZ 349
▁WENT 350
▁MU 351
▁LET 352
▁NEW 353
▁GIVE 354
▁GET 355
▁PASS 356
▁LIFE 357
▁LOVE 358
▁EYES 359
▁WITHOUT 360
▁HEAD 361
IGN 362
▁THOSE 363
▁JO 364
▁MOST 365
QUE 366
▁AWAY 367
▁TURN 368
▁YOUNG 369
▁STILL 370
▁NIGHT 371
▁FOUND 372
▁TRU 373
▁NOTHING 374
▁WHILE 375
▁TELL 376
▁FACE 377
▁LAST 378
▁PEOPLE 379
HA 380
▁THOUGH 381
▁EAR 382
▁YEAR 383
▁COUNT 384
▁WORK 385
▁YET 386
▁THREE 387
▁RIGHT 388
▁MEAN 389
▁SHA 390
▁ROOM 391
▁JE 392
▁ASKED 393
▁SAW 394
▁PUT 395
▁CHILD 396
▁EM 397
▁SAME 398
▁HEART 399
▁EVER 400
▁WANT 401
OOK 402
▁ANOTHER 403
▁LEFT 404
UFF 405
▁ARM 406
▁SHALL 407
▁GIRL 408
▁FATHER 409
▁OPEN 410
▁TOOK 411
▁SEEMED 412
▁ALWAYS 413
▁JA 414
▁MOMENT 415
▁FRIEND 416
▁WHY 417
▁TOLD 418
▁MIND 419
▁LIGHT 420
▁JU 421
▁MOTHER 422
▁BECAUSE 423
▁HOME 424
▁YES 425
▁ANSWER 426
▁SOMETHING 427
▁NAME 428
▁FIND 429
▁WORLD 430
▁HIGH 431
▁RETURN 432
▁CERTAIN 433
▁APPEAR 434
▁KNEW 435
▁WOMAN 436
▁EACH 437
▁HAVING 438
▁FEW 439
▁BEGAN 440
▁PERSON 441
▁KIND 442
▁BETTER 443
▁FOLLOW 444
▁ENOUGH 445
▁PRESENT 446
▁VOICE 447
▁BELIEVE 448
▁WHITE 449
▁HUNDRED 450
▁HOPE 451
▁BETWEEN 452
▁MORNING 453
▁WALK 454
▁PRINCE 455
▁FELT 456
▁NI 457
▁CLOSE 458
SHIP 459
▁POWER 460
▁HERSELF 461
▁SIX 462
▁HALF 463
▁AMONG 464
▁HOWEVER 465
▁TALK 466
▁ALSO 467
▁WONDER 468
▁PERHAPS 469
▁REPLIED 470
▁COURSE 471
▁FULL 472
▁POINT 473
▁QUITE 474
▁SPEAK 475
▁MYSELF 476
▁TEA 477
▁MATTER 478
▁KEEP 479
▁SMALL 480
▁ALMO 481
▁TOGETHER 482
▁STRANGE 483
▁UNTIL 484
▁CONTINU 485
▁ANYTHING 486
▁GENERAL 487
▁NEXT 488
▁DARK 489
▁STOOD 490
▁BLACK 491
▁BROUGHT 492
▁LEAVE 493
▁LARGE 494
▁CONSIDER 495
▁WOOD 496
▁ALONG 497
▁HORSE 498
Q 499
#0 500
#1 501