inference code

.python-version

@@ -0,0 +1 @@
+3.12

app.py

@@ -1,7 +1,38 @@
 import gradio as gr
+import torch
+import numpy as np
+from synthesize import synthesize
 
-def greet(name):
-    return "Hello " + name + "!!"
 
-demo = gr.Interface(fn=greet, inputs="text", outputs="text")
+def text_to_speech(text, speaker_id, cfg_scale, num_sampling_steps):
+    audio, sample_rate = synthesize(
+        text=text,
+        duration_model_config="./train_duration_dit_s.yaml",
+        acoustic_model_config="./train_acoustic_dit_b.yaml",
+        duration_model_checkpoint="./duration_model_0120000.pt",
+        acoustic_model_checkpoint="./acoustic_model_0140000.pt",
+        speaker_id=speaker_id,
+        cfg_scale=cfg_scale,
+        num_sampling_steps=num_sampling_steps,
+    )
+    return (sample_rate, audio)
+
+
+speaker_ids = [str(i) for i in range(100)]
+sampling_steps = [100, 250, 500, 1000]
+
+demo = gr.Interface(
+    fn=text_to_speech,
+    inputs=[
+        gr.Textbox(label="Text"),
+        gr.Dropdown(choices=speaker_ids, label="Speaker ID", value="0"),
+        gr.Slider(minimum=0, maximum=10, value=4.0, label="CFG Scale"),
+        gr.Dropdown(choices=sampling_steps, label="Sampling Steps", value=100),
+    ],
+    outputs=gr.Audio(label="Generated Speech"),
+    title="Text to Speech with Diffusion Transformer",
+    description="Enter text, select a speaker ID (0-99), and adjust the CFG scale to generate speech.",
+    flagging_options=None,
+)
+
 demo.launch()

maps.json

@@ -0,0 +1,989 @@
+{
+  "speaker_id_to_idx": {
+    "3003": 0,
+    "2204": 1,
+    "3307": 2,
+    "8080": 3,
+    "5935": 4,
+    "3922": 5,
+    "7982": 6,
+    "3638": 7,
+    "3032": 8,
+    "8699": 9,
+    "3274": 10,
+    "6189": 11,
+    "100": 12,
+    "7481": 13,
+    "5570": 14,
+    "176": 15,
+    "1638": 16,
+    "1776": 17,
+    "5655": 18,
+    "534": 19,
+    "2401": 20,
+    "2853": 21,
+    "5400": 22,
+    "7484": 23,
+    "5290": 24,
+    "8193": 25,
+    "1027": 26,
+    "7134": 27,
+    "7240": 28,
+    "5712": 29,
+    "339": 30,
+    "5126": 31,
+    "3869": 32,
+    "7594": 33,
+    "4899": 34,
+    "5914": 35,
+    "7515": 36,
+    "922": 37,
+    "8142": 38,
+    "28": 39,
+    "459": 40,
+    "114": 41,
+    "6927": 42,
+    "5012": 43,
+    "2481": 44,
+    "1923": 45,
+    "5618": 46,
+    "7995": 47,
+    "5054": 48,
+    "8474": 49,
+    "8527": 50,
+    "2299": 51,
+    "8498": 52,
+    "1271": 53,
+    "4356": 54,
+    "8066": 55,
+    "9022": 56,
+    "7139": 57,
+    "318": 58,
+    "3185": 59,
+    "6104": 60,
+    "6683": 61,
+    "6877": 62,
+    "8194": 63,
+    "2531": 64,
+    "7789": 65,
+    "2971": 66,
+    "7754": 67,
+    "3994": 68,
+    "6555": 69,
+    "8506": 70,
+    "2053": 71,
+    "6426": 72,
+    "6694": 73,
+    "3370": 74,
+    "16": 75,
+    "6544": 76,
+    "5876": 77,
+    "5717": 78,
+    "93": 79,
+    "7145": 80,
+    "3615": 81,
+    "8848": 82,
+    "3361": 83,
+    "1463": 84,
+    "3923": 85,
+    "4957": 86,
+    "501": 87,
+    "6458": 88,
+    "8152": 89,
+    "8410": 90,
+    "1265": 91,
+    "7910": 92,
+    "4145": 93,
+    "5538": 94,
+    "4039": 95,
+    "6965": 96,
+    "7314": 97,
+    "2960": 98,
+    "1392": 99,
+    "2573": 100,
+    "6550": 101,
+    "7833": 102,
+    "7959": 103,
+    "1226": 104,
+    "7874": 105,
+    "2769": 106,
+    "2156": 107,
+    "8011": 108,
+    "6701": 109,
+    "4595": 110,
+    "6099": 111,
+    "7926": 112,
+    "8183": 113,
+    "7247": 114,
+    "3368": 115,
+    "7460": 116,
+    "7752": 117,
+    "8718": 118,
+    "8222": 119,
+    "7949": 120,
+    "4945": 121,
+    "8225": 122,
+    "5093": 123,
+    "5319": 124,
+    "2404": 125,
+    "7766": 126,
+    "4438": 127,
+    "724": 128,
+    "6341": 129,
+    "7704": 130,
+    "1422": 131,
+    "3483": 132,
+    "3549": 133,
+    "8772": 134,
+    "4535": 135,
+    "1313": 136,
+    "7981": 137,
+    "7245": 138,
+    "6499": 139,
+    "3157": 140,
+    "3717": 141,
+    "8786": 142,
+    "7416": 143,
+    "209": 144,
+    "8075": 145,
+    "3851": 146,
+    "4110": 147,
+    "7540": 148,
+    "3482": 149,
+    "6828": 150,
+    "335": 151,
+    "203": 152,
+    "7717": 153,
+    "7313": 154,
+    "3540": 155,
+    "3380": 156,
+    "4243": 157,
+    "688": 158,
+    "8725": 159,
+    "8820": 160,
+    "7994": 161,
+    "7128": 162,
+    "4098": 163,
+    "7933": 164,
+    "1446": 165,
+    "6567": 166,
+    "4586": 167,
+    "4973": 168,
+    "1195": 169,
+    "79": 170,
+    "3228": 171,
+    "7832": 172,
+    "1731": 173,
+    "6904": 174,
+    "8605": 175,
+    "637": 176,
+    "5985": 177,
+    "7495": 178,
+    "7553": 179,
+    "6006": 180,
+    "7825": 181,
+    "7739": 182,
+    "242": 183,
+    "5909": 184,
+    "225": 185,
+    "5261": 186,
+    "770": 187,
+    "4427": 188,
+    "731": 189,
+    "4116": 190,
+    "64": 191,
+    "7188": 192,
+    "1752": 193,
+    "6080": 194,
+    "2045": 195,
+    "5246": 196,
+    "816": 197,
+    "7276": 198,
+    "207": 199,
+    "3584": 200,
+    "7720": 201,
+    "5731": 202,
+    "3703": 203,
+    "1806": 204,
+    "6038": 205,
+    "1913": 206,
+    "6139": 207,
+    "7229": 208,
+    "2004": 209,
+    "3119": 210,
+    "8113": 211,
+    "8329": 212,
+    "6497": 213,
+    "8684": 214,
+    "6120": 215,
+    "6233": 216,
+    "7286": 217,
+    "5489": 218,
+    "2056": 219,
+    "2562": 220,
+    "3046": 221,
+    "2256": 222,
+    "7120": 223,
+    "6286": 224,
+    "5039": 225,
+    "98": 226,
+    "2827": 227,
+    "5660": 228,
+    "3914": 229,
+    "4148": 230,
+    "192": 231,
+    "3733": 232,
+    "4331": 233,
+    "258": 234,
+    "6308": 235,
+    "4363": 236,
+    "8776": 237,
+    "5635": 238,
+    "2074": 239,
+    "3105": 240,
+    "5007": 241,
+    "2592": 242,
+    "2774": 243,
+    "764": 244,
+    "7525": 245,
+    "2598": 246,
+    "5740": 247,
+    "7868": 248,
+    "8028": 249,
+    "3816": 250,
+    "3118": 251,
+    "8118": 252,
+    "3009": 253,
+    "7956": 254,
+    "5062": 255,
+    "3728": 256,
+    "1649": 257,
+    "2517": 258,
+    "56": 259,
+    "8050": 260,
+    "1678": 261,
+    "6300": 262,
+    "274": 263,
+    "7881": 264,
+    "3927": 265,
+    "6643": 266,
+    "1289": 267,
+    "8006": 268,
+    "175": 269,
+    "1777": 270,
+    "781": 271,
+    "6788": 272,
+    "699": 273,
+    "5583": 274,
+    "4138": 275,
+    "511": 276,
+    "6388": 277,
+    "2427": 278,
+    "8713": 279,
+    "7434": 280,
+    "288": 281,
+    "5293": 282,
+    "1046": 283,
+    "1482": 284,
+    "8825": 285,
+    "510": 286,
+    "8195": 287,
+    "5984": 288,
+    "409": 289,
+    "8643": 290,
+    "4744": 291,
+    "6371": 292,
+    "8591": 293,
+    "4434": 294,
+    "54": 295,
+    "126": 296,
+    "7258": 297,
+    "4289": 298,
+    "373": 299,
+    "7569": 300,
+    "2238": 301,
+    "5029": 302,
+    "8887": 303,
+    "7498": 304,
+    "7705": 305,
+    "7816": 306,
+    "7828": 307,
+    "2240": 308,
+    "1472": 309,
+    "2137": 310,
+    "7437": 311,
+    "4592": 312,
+    "718": 313,
+    "3972": 314,
+    "6032": 315,
+    "7867": 316,
+    "2775": 317,
+    "2638": 318,
+    "5239": 319,
+    "6696": 320,
+    "6763": 321,
+    "1903": 322,
+    "6098": 323,
+    "6937": 324,
+    "3230": 325,
+    "2589": 326,
+    "4967": 327,
+    "1283": 328,
+    "8824": 329,
+    "8163": 330,
+    "1335": 331,
+    "1165": 332,
+    "8534": 333,
+    "1789": 334,
+    "6538": 335,
+    "6395": 336,
+    "5448": 337,
+    "597": 338,
+    "3989": 339,
+    "6373": 340,
+    "6519": 341,
+    "6637": 342,
+    "6406": 343,
+    "5190": 344,
+    "2787": 345,
+    "3945": 346,
+    "8758": 347,
+    "359": 348,
+    "6269": 349,
+    "5684": 350,
+    "6981": 351,
+    "2582": 352,
+    "1093": 353,
+    "1826": 354,
+    "6317": 355,
+    "7000": 356,
+    "4860": 357,
+    "8388": 358,
+    "5724": 359,
+    "1121": 360,
+    "8404": 361,
+    "6575": 362,
+    "5975": 363,
+    "2494": 364,
+    "561": 365,
+    "6294": 366,
+    "7294": 367,
+    "6918": 368,
+    "5519": 369,
+    "2570": 370,
+    "5206": 371,
+    "2010": 372,
+    "8008": 373,
+    "2512": 374,
+    "2882": 375,
+    "3977": 376,
+    "6054": 377,
+    "1018": 378,
+    "1079": 379,
+    "3221": 380,
+    "3083": 381,
+    "2532": 382,
+    "2741": 383,
+    "3357": 384,
+    "1028": 385,
+    "1401": 386,
+    "806": 387,
+    "1054": 388,
+    "6188": 389,
+    "5622": 390,
+    "101": 391,
+    "6014": 392,
+    "4222": 393,
+    "5656": 394,
+    "5092": 395,
+    "303": 396,
+    "4854": 397,
+    "4519": 398,
+    "716": 399,
+    "3289": 400,
+    "2060": 401,
+    "850": 402,
+    "8138": 403,
+    "5389": 404,
+    "7140": 405,
+    "369": 406,
+    "667": 407,
+    "188": 408,
+    "1112": 409,
+    "1509": 410,
+    "2758": 411,
+    "4837": 412,
+    "2230": 413,
+    "2388": 414,
+    "205": 415,
+    "984": 416,
+    "1535": 417,
+    "500": 418,
+    "4856": 419,
+    "6510": 420,
+    "7318": 421,
+    "5809": 422,
+    "3224": 423,
+    "835": 424,
+    "7117": 425,
+    "337": 426,
+    "5802": 427,
+    "1348": 428,
+    "480": 429,
+    "5189": 430,
+    "3864": 431,
+    "2999": 432,
+    "4257": 433,
+    "7967": 434,
+    "5918": 435,
+    "8855": 436,
+    "598": 437,
+    "5333": 438,
+    "2364": 439,
+    "6505": 440,
+    "1851": 441,
+    "4848": 442,
+    "882": 443,
+    "580": 444,
+    "2269": 445,
+    "3967": 446,
+    "8573": 447,
+    "9023": 448,
+    "512": 449,
+    "1349": 450,
+    "4629": 451,
+    "1382": 452,
+    "6206": 453,
+    "7783": 454,
+    "3448": 455,
+    "6378": 456,
+    "3905": 457,
+    "6167": 458,
+    "7384": 459,
+    "666": 460,
+    "949": 461,
+    "6924": 462,
+    "4290": 463,
+    "4490": 464,
+    "8401": 465,
+    "7383": 466,
+    "4598": 467,
+    "6446": 468,
+    "329": 469,
+    "7809": 470,
+    "231": 471,
+    "30": 472,
+    "6215": 473,
+    "6686": 474,
+    "3654": 475,
+    "7939": 476,
+    "5810": 477,
+    "6574": 478,
+    "14": 479,
+    "7478": 480,
+    "8494": 481,
+    "7732": 482,
+    "7030": 483,
+    "8635": 484,
+    "296": 485,
+    "5968": 486,
+    "2012": 487,
+    "5401": 488,
+    "7316": 489,
+    "3258": 490,
+    "4839": 491,
+    "707": 492,
+    "1874": 493,
+    "3521": 494,
+    "7932": 495,
+    "6865": 496,
+    "7285": 497,
+    "2992": 498,
+    "8421": 499,
+    "4425": 500,
+    "3008": 501,
+    "6115": 502,
+    "7688": 503,
+    "968": 504,
+    "1724": 505,
+    "8266": 506,
+    "2194": 507,
+    "7520": 508,
+    "5604": 509,
+    "1571": 510,
+    "6160": 511,
+    "2294": 512,
+    "7647": 513,
+    "4064": 514,
+    "3171": 515,
+    "548": 516,
+    "2673": 517,
+    "8119": 518,
+    "6492": 519,
+    "8396": 520,
+    "3446": 521,
+    "7335": 522,
+    "7169": 523,
+    "224": 524,
+    "8097": 525,
+    "3852": 526,
+    "920": 527,
+    "2368": 528,
+    "664": 529,
+    "4800": 530,
+    "7241": 531,
+    "278": 532,
+    "1769": 533,
+    "4731": 534,
+    "953": 535,
+    "3825": 536,
+    "8742": 537,
+    "2688": 538,
+    "1603": 539,
+    "3645": 540,
+    "5133": 541,
+    "4278": 542,
+    "1513": 543,
+    "7938": 544,
+    "4054": 545,
+    "272": 546,
+    "7802": 547,
+    "1859": 548,
+    "7069": 549,
+    "8687": 550,
+    "4719": 551,
+    "2751": 552,
+    "7085": 553,
+    "5266": 554,
+    "1050": 555,
+    "6060": 556,
+    "5386": 557,
+    "6782": 558,
+    "5868": 559,
+    "353": 560,
+    "227": 561,
+    "204": 562,
+    "479": 563,
+    "5002": 564,
+    "5154": 565,
+    "1025": 566,
+    "7051": 567,
+    "7095": 568,
+    "6235": 569,
+    "3294": 570,
+    "6993": 571,
+    "3835": 572,
+    "2393": 573,
+    "1779": 574,
+    "948": 575,
+    "8464": 576,
+    "6359": 577,
+    "1343": 578,
+    "8791": 579,
+    "8176": 580,
+    "1748": 581,
+    "1825": 582,
+    "249": 583,
+    "5337": 584,
+    "7909": 585,
+    "8677": 586,
+    "22": 587,
+    "3082": 588,
+    "119": 589,
+    "6727": 590,
+    "3513": 591,
+    "7991": 592,
+    "4734": 593,
+    "5606": 594,
+    "4260": 595,
+    "4226": 596,
+    "7398": 597,
+    "3551": 598,
+    "4806": 599,
+    "868": 600,
+    "5883": 601,
+    "7011": 602,
+    "1053": 603,
+    "112": 604,
+    "1379": 605,
+    "166": 606,
+    "1212": 607,
+    "594": 608,
+    "589": 609,
+    "454": 610,
+    "925": 611,
+    "497": 612,
+    "1322": 613,
+    "7777": 614,
+    "70": 615,
+    "7297": 616,
+    "159": 617,
+    "899": 618,
+    "7061": 619,
+    "2618": 620,
+    "7555": 621,
+    "8459": 622,
+    "1365": 623,
+    "8879": 624,
+    "2113": 625,
+    "1066": 626,
+    "1445": 627,
+    "2229": 628,
+    "8057": 629,
+    "345": 630,
+    "612": 631,
+    "1958": 632,
+    "8545": 633,
+    "1448": 634,
+    "3328": 635,
+    "2929": 636,
+    "7962": 637,
+    "1387": 638,
+    "6494": 639,
+    "3876": 640,
+    "380": 641,
+    "5723": 642,
+    "6553": 643,
+    "3114": 644,
+    "398": 645,
+    "1885": 646,
+    "2709": 647,
+    "2920": 648,
+    "576": 649,
+    "1160": 650,
+    "246": 651,
+    "4335": 652,
+    "7558": 653,
+    "5115": 654,
+    "4133": 655,
+    "1641": 656,
+    "6956": 657,
+    "5139": 658,
+    "1264": 659,
+    "1259": 660,
+    "1705": 661,
+    "1961": 662,
+    "1012": 663,
+    "4057": 664,
+    "6895": 665,
+    "8347": 666,
+    "1337": 667,
+    "7395": 668,
+    "5672": 669,
+    "8479": 670,
+    "8190": 671,
+    "4381": 672,
+    "8771": 673,
+    "1060": 674,
+    "6258": 675,
+    "2654": 676,
+    "5123": 677,
+    "5304": 678,
+    "2790": 679,
+    "2254": 680,
+    "3493": 681,
+    "3180": 682,
+    "6352": 683,
+    "1987": 684,
+    "3094": 685,
+    "9026": 686,
+    "1222": 687,
+    "4013": 688,
+    "340": 689,
+    "2785": 690,
+    "3072": 691,
+    "6339": 692,
+    "836": 693,
+    "3330": 694,
+    "475": 695,
+    "7945": 696,
+    "6082": 697,
+    "4111": 698,
+    "8705": 699,
+    "3781": 700,
+    "4010": 701,
+    "5746": 702,
+    "1827": 703,
+    "476": 704,
+    "7538": 705,
+    "8875": 706,
+    "2149": 707,
+    "2272": 708,
+    "6509": 709,
+    "2127": 710,
+    "7730": 711,
+    "2823": 712,
+    "3490": 713,
+    "1241": 714,
+    "698": 715,
+    "1460": 716,
+    "6620": 717,
+    "1845": 718,
+    "3889": 719,
+    "1383": 720,
+    "3630": 721,
+    "2473": 722,
+    "2319": 723,
+    "6288": 724,
+    "7837": 725,
+    "711": 726,
+    "5186": 727,
+    "4238": 728,
+    "1944": 729,
+    "3379": 730,
+    "2577": 731,
+    "451": 732,
+    "3389": 733,
+    "5242": 734,
+    "1849": 735,
+    "2110": 736,
+    "5147": 737,
+    "4807": 738,
+    "7957": 739,
+    "581": 740,
+    "4108": 741,
+    "1323": 742,
+    "1556": 743,
+    "8619": 744,
+    "783": 745,
+    "957": 746,
+    "3979": 747,
+    "1052": 748,
+    "439": 749,
+    "115": 750,
+    "3866": 751,
+    "6673": 752,
+    "1943": 753,
+    "6037": 754,
+    "6157": 755,
+    "1668": 756,
+    "240": 757,
+    "596": 758,
+    "1974": 759,
+    "8592": 760,
+    "1061": 761,
+    "81": 762,
+    "639": 763,
+    "1487": 764,
+    "954": 765,
+    "8300": 766,
+    "1425": 767,
+    "2162": 768,
+    "7090": 769,
+    "4733": 770,
+    "492": 771,
+    "2061": 772,
+    "7733": 773,
+    "1413": 774,
+    "208": 775,
+    "5727": 776,
+    "2146": 777,
+    "6518": 778,
+    "4590": 779,
+    "216": 780,
+    "2652": 781,
+    "5776": 782,
+    "6330": 783,
+    "5588": 784,
+    "2285": 785,
+    "7126": 786,
+    "5513": 787,
+    "979": 788,
+    "5940": 789,
+    "1740": 790,
+    "4044": 791,
+    "8228": 792,
+    "543": 793,
+    "17": 794,
+    "4358": 795,
+    "5157": 796,
+    "8575": 797,
+    "1417": 798,
+    "1224": 799,
+    "606": 800,
+    "2628": 801,
+    "3546": 802,
+    "5063": 803,
+    "2201": 804,
+    "1311": 805,
+    "593": 806,
+    "1547": 807,
+    "1629": 808,
+    "4152": 809,
+    "1914": 810,
+    "7445": 811,
+    "4246": 812,
+    "1607": 813,
+    "559": 814,
+    "1801": 815,
+    "122": 816,
+    "1639": 817,
+    "663": 818,
+    "323": 819,
+    "2411": 820,
+    "1182": 821,
+    "3025": 822,
+    "5767": 823,
+    "2397": 824,
+    "2093": 825,
+    "4495": 826,
+    "708": 827,
+    "986": 828,
+    "154": 829,
+    "1498": 830,
+    "829": 831,
+    "3792": 832,
+    "549": 833,
+    "815": 834,
+    "362": 835,
+    "1933": 836,
+    "3790": 837,
+    "1100": 838,
+    "1811": 839,
+    "2812": 840,
+    "55": 841,
+    "636": 842,
+    "8490": 843,
+    "2499": 844,
+    "2167": 845,
+    "803": 846,
+    "3537": 847,
+    "4236": 848,
+    "90": 849,
+    "217": 850,
+    "3340": 851,
+    "7475": 852,
+    "1336": 853,
+    "2581": 854,
+    "1316": 855,
+    "7665": 856,
+    "1001": 857,
+    "3001": 858,
+    "3738": 859,
+    "472": 860,
+    "1473": 861,
+    "3215": 862,
+    "487": 863,
+    "1290": 864,
+    "7339": 865,
+    "3686": 866,
+    "7342": 867,
+    "6075": 868,
+    "210": 869,
+    "1552": 870,
+    "38": 871,
+    "7657": 872,
+    "2816": 873,
+    "1734": 874,
+    "6690": 875,
+    "1390": 876,
+    "3070": 877,
+    "8722": 878,
+    "7518": 879,
+    "1754": 880,
+    "408": 881,
+    "1645": 882,
+    "2696": 883,
+    "830": 884,
+    "4846": 885,
+    "1175": 886,
+    "2498": 887,
+    "434": 888,
+    "1058": 889,
+    "2674": 890,
+    "583": 891,
+    "1296": 892,
+    "4681": 893,
+    "4926": 894,
+    "820": 895,
+    "6119": 896,
+    "1536": 897,
+    "3187": 898,
+    "157": 899,
+    "2039": 900,
+    "4433": 901,
+    "4071": 902,
+    "2085": 903
+  },
+  "phone_to_idx": {
+    "AA0": 0,
+    "AA1": 1,
+    "AA2": 2,
+    "AE0": 3,
+    "AE1": 4,
+    "AE2": 5,
+    "AH0": 6,
+    "AH1": 7,
+    "AH2": 8,
+    "AO0": 9,
+    "AO1": 10,
+    "AO2": 11,
+    "AW0": 12,
+    "AW1": 13,
+    "AW2": 14,
+    "AY0": 15,
+    "AY1": 16,
+    "AY2": 17,
+    "B": 18,
+    "CH": 19,
+    "D": 20,
+    "DH": 21,
+    "EH0": 22,
+    "EH1": 23,
+    "EH2": 24,
+    "EMPTY": 25,
+    "ER0": 26,
+    "ER1": 27,
+    "ER2": 28,
+    "EY0": 29,
+    "EY1": 30,
+    "EY2": 31,
+    "F": 32,
+    "G": 33,
+    "HH": 34,
+    "IH0": 35,
+    "IH1": 36,
+    "IH2": 37,
+    "IY0": 38,
+    "IY1": 39,
+    "IY2": 40,
+    "JH": 41,
+    "K": 42,
+    "L": 43,
+    "M": 44,
+    "N": 45,
+    "NG": 46,
+    "OW0": 47,
+    "OW1": 48,
+    "OW2": 49,
+    "OY0": 50,
+    "OY1": 51,
+    "OY2": 52,
+    "P": 53,
+    "R": 54,
+    "S": 55,
+    "SH": 56,
+    "SPN": 57,
+    "T": 58,
+    "TH": 59,
+    "UH0": 60,
+    "UH1": 61,
+    "UH2": 62,
+    "UW0": 63,
+    "UW1": 64,
+    "UW2": 65,
+    "V": 66,
+    "W": 67,
+    "Y": 68,
+    "Z": 69,
+    "ZH": 70,
+    "PAD": 71
+  },
+  "phone_kind_to_idx": {
+    "EMPTY": 0,
+    "WORD": 1,
+    "START": 2,
+    "END": 3,
+    "MIDDLE": 4
+  }
+}

models.py

@@ -0,0 +1,478 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+# --------------------------------------------------------
+# References:
+# GLIDE: https://github.com/openai/glide-text2im
+# MAE: https://github.com/facebookresearch/mae/blob/main/models_mae.py
+# --------------------------------------------------------
+
+import math
+
+import numpy as np
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from torch.nn.attention.flex_attention import BlockMask, flex_attention
+
+
+class Mlp(nn.Module):
+    """MLP as used in Vision Transformer, MLP-Mixer and related networks"""
+
+    def __init__(
+        self,
+        in_features,
+        hidden_features=None,
+        out_features=None,
+        act_layer=nn.SiLU,
+        norm_layer=None,
+        bias=True,
+    ):
+        super().__init__()
+        out_features = out_features or in_features
+        hidden_features = hidden_features or in_features
+        linear_layer = nn.Linear
+
+        self.fc1 = linear_layer(in_features, hidden_features, bias=bias)
+        self.act = act_layer()
+        self.norm = (
+            norm_layer(hidden_features) if norm_layer is not None else nn.Identity()
+        )
+        self.fc2 = linear_layer(hidden_features, out_features, bias=bias)
+
+    def forward(self, x):
+        x = self.fc1(x)
+        x = self.act(x)
+        x = self.norm(x)
+        x = self.fc2(x)
+        return x
+
+
+class RMSNorm(torch.nn.Module):
+    def __init__(self, dim: int, eps: float = 1e-5):
+        super().__init__()
+        self.eps = eps
+        self.weight = nn.Parameter(torch.ones(dim))
+
+    def _norm(self, x):
+        return x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + self.eps)
+
+    def forward(self, x):
+        output = self._norm(x.float()).type_as(x)
+        return output * self.weight
+
+
+class Attention(nn.Module):
+    def __init__(
+        self,
+        dim: int,
+        num_heads: int = 8,
+        qkv_bias: bool = False,
+    ) -> None:
+        super().__init__()
+        assert dim % num_heads == 0, "dim should be divisible by num_heads"
+        self.num_heads = num_heads
+        self.head_dim = dim // num_heads
+        self.scale = self.head_dim**-0.5
+
+        self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias)
+        self.proj = nn.Linear(dim, dim)
+        self.q_norm = RMSNorm(self.head_dim, eps=1e-5)
+        self.k_norm = RMSNorm(self.head_dim, eps=1e-5)
+
+    def forward(self, x: torch.Tensor, attn_mask=None) -> torch.Tensor:
+        B, N, C = x.shape
+        qkv = (
+            self.qkv(x)
+            .reshape(B, N, 3, self.num_heads, self.head_dim)
+            .permute(2, 0, 3, 1, 4)
+        )
+        q, k, v = qkv.unbind(0)
+
+        if isinstance(attn_mask, torch.Tensor) or attn_mask is None:
+            q = self.q_norm(q)
+            k = self.k_norm(k)
+            # v = v
+            x = F.scaled_dot_product_attention(
+                q,
+                k,
+                v,
+                attn_mask=attn_mask,
+            )
+        elif isinstance(attn_mask, BlockMask):
+            with torch.autocast(enabled=False, device_type="cuda"):
+                q = self.q_norm(q).half()
+                k = self.k_norm(k).half()
+                v = v.half()
+                x = flex_attention(q, k, v, block_mask=attn_mask)
+        x = x.transpose(1, 2).reshape(B, N, C)
+        x = self.proj(x)
+        return x
+
+
+def modulate(x, shift, scale):
+    return x * (1 + scale) + shift
+
+
+#################################################################################
+#               Embedding Layers for Timesteps and Class Labels                 #
+#################################################################################
+
+
+class TimestepEmbedder(nn.Module):
+    """
+    Embeds scalar timesteps into vector representations.
+    """
+
+    def __init__(self, hidden_size, frequency_embedding_size=256):
+        super().__init__()
+        self.mlp = nn.Sequential(
+            nn.Linear(frequency_embedding_size, hidden_size, bias=True),
+            nn.SiLU(),
+            nn.Linear(hidden_size, hidden_size, bias=True),
+        )
+        self.frequency_embedding_size = frequency_embedding_size
+
+    @staticmethod
+    def timestep_embedding(t, dim, max_period=10000):
+        """
+        Create sinusoidal timestep embeddings.
+        :param t: a 1-D Tensor of N indices, one per batch element.
+                          These may be fractional.
+        :param dim: the dimension of the output.
+        :param max_period: controls the minimum frequency of the embeddings.
+        :return: an (N, D) Tensor of positional embeddings.
+        """
+        # https://github.com/openai/glide-text2im/blob/main/glide_text2im/nn.py
+        half = dim // 2
+        freqs = torch.exp(
+            -math.log(max_period)
+            * torch.arange(start=0, end=half, dtype=torch.float32)
+            / half
+        ).to(device=t.device)
+        args = t[:, None].float() * freqs[None]
+        embedding = torch.cat([torch.cos(args), torch.sin(args)], dim=-1)
+        if dim % 2:
+            embedding = torch.cat(
+                [embedding, torch.zeros_like(embedding[:, :1])], dim=-1
+            )
+        return embedding
+
+    def forward(self, t):
+        t_freq = self.timestep_embedding(t, self.frequency_embedding_size)
+        t_emb = self.mlp(t_freq)
+        return t_emb
+
+
+class LabelEmbedder(nn.Module):
+    """
+    Embeds class labels into vector representations. Also handles label dropout for classifier-free guidance.
+    """
+
+    def __init__(self, num_classes, hidden_size, dropout_prob):
+        super().__init__()
+        # use last class as unconditional value
+        use_cfg_embedding = dropout_prob > 0
+        if use_cfg_embedding:
+            self.unconditional_value = num_classes - 1
+        self.speaker_id_table = nn.Embedding(num_classes, hidden_size)
+        self.phone_table = nn.Embedding(num_classes, hidden_size)
+        self.phone_kind_table = nn.Embedding(num_classes, hidden_size)
+        self.num_classes = num_classes
+        self.dropout_prob = dropout_prob
+
+    def token_drop(self, speaker_id, phone, phone_kind, force_drop_ids=None):
+        """
+        Drops labels to enable classifier-free guidance.
+        """
+        if force_drop_ids is None:
+            drop_ids = (
+                torch.rand(speaker_id.shape[0], device=speaker_id.device)
+                < self.dropout_prob
+            )
+        else:
+            drop_ids = force_drop_ids == 1
+        speaker_id = torch.where(
+            drop_ids[:, None], self.unconditional_value, speaker_id
+        )
+        phone = torch.where(drop_ids[:, None], self.unconditional_value, phone)
+        phone_kind = torch.where(
+            drop_ids[:, None], self.unconditional_value, phone_kind
+        )
+        return speaker_id, phone, phone_kind
+
+    def forward(self, speaker_id, phone, phone_kind, train, force_drop_ids=None):
+        use_dropout = self.dropout_prob > 0
+        if (train and use_dropout) or (force_drop_ids is not None):
+            speaker_id, phone, phone_kind = self.token_drop(
+                speaker_id, phone, phone_kind, force_drop_ids
+            )
+        speaker_id_embeddings = self.speaker_id_table(speaker_id)
+        phone_embeddings = self.phone_table(phone)
+        phone_kind_embeddings = self.phone_kind_table(phone_kind)
+        return speaker_id_embeddings, phone_embeddings, phone_kind_embeddings
+
+
+#################################################################################
+#                                 Core DiT Model                                #
+#################################################################################
+
+
+class DiTBlock(nn.Module):
+    """
+    A DiT block with adaptive layer norm zero (adaLN-Zero) conditioning.
+    """
+
+    def __init__(self, hidden_size, num_heads, mlp_ratio=4.0, **block_kwargs):
+        super().__init__()
+        self.norm1 = nn.LayerNorm(hidden_size, elementwise_affine=False, eps=1e-6)
+        self.attn = Attention(
+            hidden_size, num_heads=num_heads, qkv_bias=True, **block_kwargs
+        )
+        self.norm2 = nn.LayerNorm(hidden_size, elementwise_affine=False, eps=1e-6)
+        mlp_hidden_dim = int(hidden_size * mlp_ratio)
+        self.mlp = Mlp(
+            in_features=hidden_size,
+            hidden_features=mlp_hidden_dim,
+            act_layer=nn.SiLU,
+        )
+        self.adaLN_modulation = nn.Sequential(
+            nn.SiLU(),
+            nn.Linear(hidden_size, 6 * hidden_size, bias=True),
+        )
+
+    def forward(self, x, c, attn_mask=None):
+        shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = (
+            self.adaLN_modulation(c).chunk(6, dim=-1)
+        )
+        x = x + gate_msa * self.attn(
+            modulate(self.norm1(x), shift_msa, scale_msa), attn_mask=attn_mask
+        )
+        x = x + gate_mlp * self.mlp(modulate(self.norm2(x), shift_mlp, scale_mlp))
+        return x
+
+
+class FinalLayer(nn.Module):
+    """
+    The final layer of DiT.
+    """
+
+    def __init__(self, hidden_size, patch_size, out_channels):
+        super().__init__()
+        self.norm_final = nn.LayerNorm(hidden_size, elementwise_affine=False, eps=1e-6)
+        self.linear = nn.Linear(
+            hidden_size, patch_size * patch_size * out_channels, bias=True
+        )
+        self.adaLN_modulation = nn.Sequential(
+            nn.SiLU(),
+            nn.Linear(hidden_size, 2 * hidden_size, bias=True),
+        )
+
+    def forward(self, x, c):
+        shift, scale = self.adaLN_modulation(c).chunk(2, dim=-1)
+        x = modulate(self.norm_final(x), shift, scale)
+        x = self.linear(x)
+        return x
+
+
+class DiT(nn.Module):
+    """
+    Diffusion model with a Transformer backbone.
+    """
+
+    def __init__(
+        self,
+        input_size=256,
+        in_channels=1024,
+        hidden_size=1152,
+        depth=28,
+        num_heads=16,
+        mlp_ratio=4.0,
+        class_dropout_prob=0.1,
+        learn_sigma=True,
+        embedding_vocab_size=1024,
+    ):
+        super().__init__()
+        self.input_size = input_size
+        self.learn_sigma = learn_sigma
+        self.in_channels = in_channels
+        self.hidden_size = hidden_size
+        self.out_channels = in_channels * 2 if learn_sigma else in_channels
+        self.num_heads = num_heads
+
+        self.x_embedder = nn.Linear(in_channels, hidden_size, bias=True)
+        self.t_embedder = TimestepEmbedder(hidden_size)
+        self.y_embedder = LabelEmbedder(
+            embedding_vocab_size, hidden_size, class_dropout_prob
+        )
+        # Will use fixed sin-cos embedding:
+        self.register_buffer("pos_embed", torch.zeros(1, self.input_size, hidden_size))
+
+        self.blocks = nn.ModuleList(
+            [
+                DiTBlock(hidden_size, num_heads, mlp_ratio=mlp_ratio)
+                for _ in range(depth)
+            ]
+        )
+        self.final_layer = FinalLayer(hidden_size, 1, self.out_channels)
+        self.initialize_weights()
+
+    def initialize_weights(self):
+        # Initialize transformer layers:
+        def _basic_init(module):
+            if isinstance(module, nn.Linear):
+                torch.nn.init.xavier_uniform_(module.weight)
+                if module.bias is not None:
+                    nn.init.constant_(module.bias, 0)
+
+        self.apply(_basic_init)
+
+        # Initialize (and freeze) pos_embed by sin-cos embedding:
+        pos_embed = get_1d_sincos_pos_embed(self.pos_embed.shape[-1], self.input_size)
+        self.pos_embed.data.copy_(torch.from_numpy(pos_embed).float().unsqueeze(0))
+
+        # Initialize patch_embed like nn.Linear (instead of nn.Conv2d):
+        w = self.x_embedder.weight.data
+        nn.init.xavier_uniform_(w.view([w.shape[0], -1]))
+        nn.init.constant_(self.x_embedder.bias, 0)
+
+        # Initialize label embedding table:
+        scale = 1.0 / math.sqrt(self.hidden_size)
+        nn.init.trunc_normal_(self.y_embedder.speaker_id_table.weight, std=scale)
+        nn.init.trunc_normal_(self.y_embedder.phone_table.weight, std=scale)
+
+        # Initialize timestep embedding MLP:
+        nn.init.trunc_normal_(self.t_embedder.mlp[0].weight, std=scale)
+        nn.init.trunc_normal_(self.t_embedder.mlp[2].weight, std=scale)
+
+        # Zero-out adaLN modulation layers in DiT blocks:
+        for block in self.blocks:
+            nn.init.constant_(block.adaLN_modulation[-1].weight, 0)
+            nn.init.constant_(block.adaLN_modulation[-1].bias, 0)
+
+        # Zero-out output layers:
+        nn.init.constant_(self.final_layer.adaLN_modulation[-1].weight, 0)
+        nn.init.constant_(self.final_layer.adaLN_modulation[-1].bias, 0)
+        nn.init.constant_(self.final_layer.linear.weight, 0)
+        nn.init.constant_(self.final_layer.linear.bias, 0)
+
+    def forward(self, x, t, speaker_id, phone, phone_kind, attn_mask=None):
+        """
+        Forward pass of DiT.
+        x: (N, C, L) tensor of spatial inputs
+        t: (N,) tensor of diffusion timesteps
+        speaker_id: (N,) tensor of speaker IDs
+        phone: (N, L) tensor of phone labels
+        phone_kind: (N, L) tensor of phone kinds
+        """
+        # (N, D), (N, L, D)
+        speaker_id_embedding, phone_embedding, phone_kind_embedding = self.y_embedder(
+            speaker_id, phone, phone_kind, self.training
+        )
+        t = self.t_embedder(t)  # (N, D)
+        c = t  # (N, D)
+        c = (
+            c[:, None, :]
+            + speaker_id_embedding
+            + phone_embedding
+            + phone_kind_embedding
+        )  # (N, L, D)
+
+        x = x.transpose(-1, -2)  # Swap last two dimensions
+        x = self.x_embedder(x) + self.pos_embed[:, : x.shape[1], :]  # (N, L, D)
+        for block in self.blocks:
+            x = block(x, c, attn_mask=attn_mask)  # (N, L, D)
+        x = self.final_layer(x, c)  # (N, L, 2 * out_channels)
+        x = x.transpose(-1, -2)  # Swap last two dimensions
+        return x
+
+    def forward_with_cfg(
+        self, x, t, speaker_id, phone, phone_kind, cfg_scale, attn_mask=None
+    ):
+        """
+        Forward pass of DiT, but also batches the unconditional forward pass for classifier-free guidance.
+        """
+        # https://github.com/openai/glide-text2im/blob/main/notebooks/text2im.ipynb
+        half = x[: len(x) // 2]
+        combined = torch.cat([half, half], dim=0)
+        model_out = self.forward(
+            combined, t, speaker_id, phone, phone_kind, attn_mask=attn_mask
+        )
+        # For exact reproducibility reasons, we apply classifier-free guidance on only
+        # three channels by default. The standard approach to cfg applies it to all channels.
+        # This can be done by uncommenting the following line and commenting-out the line following that.
+        eps, rest = model_out[:, : self.in_channels], model_out[:, self.in_channels :]
+        # eps, rest = model_out[:, :3], model_out[:, 3:]
+        cond_eps, uncond_eps = torch.split(eps, len(eps) // 2, dim=0)
+        half_eps = uncond_eps + cfg_scale * (cond_eps - uncond_eps)
+        eps = torch.cat([half_eps, half_eps], dim=0)
+        return torch.cat([eps, rest], dim=1)
+
+
+#################################################################################
+#                   Sine/Cosine Positional Embedding Functions                  #
+#################################################################################
+# https://github.com/facebookresearch/mae/blob/main/util/pos_embed.py
+
+
+def get_1d_sincos_pos_embed(embed_dim, length, cls_token=False, extra_tokens=0):
+    """
+    length: int of the length
+    return:
+    pos_embed: [length, embed_dim] or [1+length, embed_dim] (w/ or w/o cls_token)
+    """
+    grid = np.arange(length, dtype=np.float32)
+    pos_embed = get_1d_sincos_pos_embed_from_grid(embed_dim, grid)
+    if cls_token and extra_tokens > 0:
+        pos_embed = np.concatenate(
+            [np.zeros([extra_tokens, embed_dim]), pos_embed], axis=0
+        )
+    return pos_embed
+
+
+def get_1d_sincos_pos_embed_from_grid(embed_dim, pos):
+    """
+    embed_dim: output dimension for each position
+    pos: a list of positions to be encoded: size (M,)
+    out: (M, D)
+    """
+    assert embed_dim % 2 == 0
+    omega = np.arange(embed_dim // 2, dtype=np.float64)
+    omega /= embed_dim / 2.0
+    omega = 1.0 / 10000**omega  # (D/2,)
+
+    pos = pos.reshape(-1)  # (M,)
+    out = np.einsum("m,d->md", pos, omega)  # (M, D/2), outer product
+
+    emb_sin = np.sin(out)  # (M, D/2)
+    emb_cos = np.cos(out)  # (M, D/2)
+
+    emb = np.concatenate([emb_sin, emb_cos], axis=1)  # (M, D)
+    return emb
+
+
+#################################################################################
+#################################################################################
+#                                   DiT Configs                                  #
+#################################################################################
+
+
+def DiT_XL(**kwargs):
+    return DiT(depth=28, hidden_size=1152, num_heads=16, **kwargs)
+
+
+def DiT_L(**kwargs):
+    return DiT(depth=24, hidden_size=1024, num_heads=16, **kwargs)
+
+
+def DiT_B(**kwargs):
+    return DiT(depth=12, hidden_size=768, num_heads=12, **kwargs)
+
+
+def DiT_S(**kwargs):
+    return DiT(depth=6, hidden_size=256, num_heads=4, **kwargs)
+
+
+DiT_models = {"DiT-XL": DiT_XL, "DiT-L": DiT_L, "DiT-B": DiT_B, "DiT-S": DiT_S}

nltk_data/taggers/averaged_perceptron_tagger_eng/averaged_perceptron_tagger_eng.classes.json

@@ -0,0 +1 @@
+[".", "(", ")", ":", "''", "EX", "JJS", "WRB", "VBG", "VBP", "NN", "SYM", "VB", "UH", "NNPS", "NNP", "``", "$", "NNS", "JJR", "MD", "RP", "VBD", "DT", "POS", "RBR", ",", "VBZ", "PDT", "VBN", "WP$", "WDT", "WP", "PRP$", "CD", "IN", "#", "CC", "RB", "FW", "RBS", "PRP", "LS", "JJ", "TO"]

nltk_data/taggers/averaged_perceptron_tagger_eng/averaged_perceptron_tagger_eng.tagdict.json

@@ -0,0 +1 @@
+{"four": "CD", "facilities": "NNS", "controversial": "JJ", "Until": "IN", "whose": "WP$", "under": "IN", "pact": "NN", "regional": "JJ", "GE": "NNP", "every": "DT", "GM": "NNP", "Moon": "NNP", "school": "NN", "companies": "NNS", "Nasdaq": "NNP", "Paul": "NNP", "Pinkerton": "NNP", "leaders": "NNS", "guidelines": "NNS", "Sales": "NNS", "machines": "NNS", "pace": "NN", "spokesman": "NN", "new": "JJ", "ever": "RB", "men": "NNS", "here": "RB", "protection": "NN", "studio": "NN", "active": "JJ", "100": "CD", "ventures": "NNS", "items": "NNS", "employees": "NNS", "credit": "NN", "analysts": "NNS", "criticism": "NN", "golden": "JJ", "Group": "NNP", "campaign": "NN", "St.": "NNP", "replace": "VB", "Also": "RB", "Health": "NNP", "costly": "JJ", "unit": "NN", "swings": "NNS", "would": "MD", "century": "NN", "June": "NNP", "music": "NN", "asset": "NN", "N.J.": "NNP", "until": "IN", "PaineWebber": "NNP", "Breeden": "NNP", "Ministry": "NNP", "successful": "JJ", "phone": "NN", "90": "CD", "circumstances": "NNS", "me": "PRP", "1990": "CD", "1993": "CD", "1992": "CD", "word": "NN", "1994": "CD", "rights": "NNS", "movies": "NNS", "already": "RB", "my": "PRP$", "example": "NN", "estate": "NN", "psyllium": "NN", "Hurricane": "NNP", "10,000": "CD", "Digital": "NNP", "totaled": "VBD", "recovery": "NN", "Journal": "NNP", "thousands": "NNS", "machine": "NN", "how": "WRB", "Jack": "NNP", "interview": "NN", "resignation": "NN", "minority": "NN", "L.": "NNP", "after": "IN", "modest": "JJ", "president": "NN", "law": "NN", "effective": "JJ", "Maxwell": "NNP", "Telerate": "NNP", "Another": "DT", "Trust": "NNP", "order": "NN", "operations": "NNS", "office": "NN", "expects": "VBZ", "presence": "NN", "His": "PRP$", "personal": "JJ", "expectations": "NNS", "Here": "RB", "production": "NN", "400": "CD", "Judge": "NNP", "weeks": "NNS", "Spain": "NNP", "eventually": "RB", "them": "PRP", "weakness": "NN", "Thomas": "NNP", "effects": "NNS", "they": "PRP", "schools": "NNS", "bank": "NN", "represents": "VBZ", "Indeed": "RB", "each": "DT", "went": "VBD", "bond": "NN", "financial": "JJ", "fairly": "RB", "series": "NN", "substantially": "RB", "lawyers": "NNS", "by": "IN", "network": "NN", "Chancellor": "NNP", "William": "NNP", "Icahn": "NNP", "size": "NN", "University": "NNP", "N.Y.": "NNP", "enormous": "JJ", "Monday": "NNP", "September": "NNP", "National": "NNP", "days": "NNS", "appeals": "NNS", "economists": "NNS", "another": "DT", "electronic": "JJ", "Congress": "NNP", "lawsuits": "NNS", "rates": "NNS", "too": "RB", "percentage": "NN", "ceiling": "NN", "took": "VBD", "budget": "NN", "acquisition": "NN", "fashion": "NN", "Chicago": "NNP", "Cray": "NNP", "talking": "VBG", "seed": "NN", "Instead": "RB", "dozen": "NN", "Then": "RB", "strength": "NN", "responsible": "JJ", "-": ":", "practices": "NNS", "Minister": "NNP", "They": "PRP", "Bank": "NNP", "unsecured": "JJ", "Jones": "NNP", "shall": "MD", "involving": "VBG", "letter": "NN", "Mobil": "NNP", "medical": "JJ", "competitors": "NNS", "consumer": "NN", "Its": "PRP$", "came": "VBD", "Union": "NNP", "meetings": "NNS", "ending": "VBG", "specialists": "NNS", "judges": "NNS", "Mixte": "NNP", "representing": "VBG", "exports": "NNS", "wide": "JJ", "13": "CD", "certificates": "NNS", "despite": "IN", "volatility": "NN", "countries": "NNS", "high-yield": "JJ", "Washington": "NNP", "bad": "JJ", "Qintex": "NNP", "movement": "NN", "secretary": "NN", "Gorbachev": "NNP", "discussions": "NNS", "John": "NNP", "said": "VBD", "capacity": "NN", "wage": "NN", "we": "PRP", "never": "RB", "terms": "NNS", "wo": "MD", "were": "VBD", "weak": "JJ", "however": "RB", "news": "NN", "debt": "NN", "Among": "IN", "country": "NN", "uncertainty": "NN", "against": "IN", "Thomson": "NNP", "players": "NNS", "Computer": "NNP", "games": "NNS", "faces": "VBZ", "tough": "JJ", "tons": "NNS", "Board": "NNP", "250": "CD", "line-item": "JJ", "conference": "NN", "C.": "NNP", "basis": "NN", "union": "NN", "three": "CD", "been": "VBN", "C$": "$", "commission": "NN", "beer": "NN", "interest": "NN", "life": "NN", "families": "NNS", "Conn.": "NNP", "Tokyo": "NNP", "drugs": "NNS", "Poland": "NNP", "Secretary": "NNP", "Co": "NNP", "publicly": "RB", "property": "NN", "Tuesday": "NNP", "seven": "CD", "On": "IN", "is": "VBZ", "it": "PRP", "expenses": "NNS", "player": "NN", "Bush": "NNP", "experts": "NNS", "in": "IN", "victims": "NNS", "if": "IN", "things": "NNS", "damages": "NNS", "big": "JJ", "President": "NNP", "several": "JJ", "independent": "JJ", "Institute": "NNP", "hand": "NN", "Angeles": "NNP", "Morris": "NNP", "ownership": "NN", "opportunity": "NN", "cycle": "NN", "RJR": "NNP", "programs": "NNS", "client": "NN", "the": "DT", "corporate": "JJ", "investments": "NNS", "agency": "NN", "just": "RB", "unemployment": "NN", "previous": "JJ", "adding": "VBG", "buyers": "NNS", "board": "NN", "Philip": "NNP", "has": "VBZ", "gave": "VBD", "Santa": "NNP", "James": "NNP", "possible": "JJ", "Chrysler": "NNP", "30": "CD", "highly": "RB", "55": "CD", "51": "CD", "50": "CD", "securities": "NNS", "offices": "NNS", "officer": "NN", "night": "NN", "security": "NN", "Pentagon": "NNP", "attorney": "NN", "old": "JJ", "people": "NNS", "Commission": "NNP", "election": "NN", "short-term": "JJ", "Lee": "NNP", "for": "IN", "comments": "NNS", "everything": "NN", "He": "PRP", "corn": "NN", "conventional": "JJ", "Georgia-Pacific": "NNP", "brokerage": "NN", "properties": "NNS", "dollars": "NNS", "months": "NNS", "magazine": "NN", "ensure": "VB", "afternoon": "NN", "efforts": "NNS", "Still": "RB", "slightly": "RB", "Fed": "NNP", "statements": "NNS", "facility": "NN", "civil": "JJ", "magazines": "NNS", "defendants": "NNS", "initial": "JJ", "legislation": "NN", "why": "WRB", "editor": "NN", "way": "NN", "NBC": "NNP", "was": "VBD", "war": "NN", "manufacturers": "NNS", "January": "NNP", "becoming": "VBG", "true": "JJ", "analyst": "NN", "counsel": "NN", "devices": "NNS", "County": "NNP", "Greenspan": "NNP", ".": ".", "Sir": "NNP", "evidence": "NN", "''": "''", "trip": "NN", "negotiations": "NNS", "LTV": "NNP", "Francisco": "NNP", "floor": "NN", "stake": "NN", "generally": "RB", "role": "NN", "models": "NNS", "Hunt": "NNP", "fell": "VBD", "authorities": "NNS", "'m": "VBP", "Mass.": "NNP", "weekend": "NN", "billion": "CD", "reorganization": "NN", "Estate": "NNP", "Charles": "NNP", "time": "NN", "serious": "JJ", "Moscow": "NNP", "profits": "NNS", "chain": "NN", "global": "JJ", "alternatives": "NNS", "manager": "NN", "battle": "NN", "certainly": "RB", "Sept.": "NNP", "Columbia": "NNP", "environment": "NN", "finally": "RB", "must": "MD", "1991": "CD", "choice": "NN", "liability": "NN", "trouble": "NN", "Jersey": "NNP", "room": "NN", "did": "VBD", "proposals": "NNS", "standards": "NNS", "speculation": "NN", "George": "NNP", "Rey": "NNP", "says": "VBZ", "trend": "NN", "M.": "NNP", "adds": "VBZ", "shares": "NNS", "Ford": "NNP", "current": "JJ", "goes": "VBZ", "international": "JJ", "falling": "VBG", "Nov.": "NNP", "transportation": "NN", "genes": "NNS", "water": "NN", "baseball": "NN", "groups": "NNS", "Ltd": "NNP", "appears": "VBZ", "Warner": "NNP", "healthy": "JJ", "guilty": "JJ", "trial": "NN", "usually": "RB", "Inc": "NNP", "studies": "NNS", "When": "WRB", "crisis": "NN", "market": "NN", "Australia": "NNP", "August": "NNP", "positive": "JJ", "sports": "NNS", "francs": "NNS", "today": "NN", "``": "``", "October": "NNP", "These": "DT", "downturn": "NN", "cases": "NNS", "effort": "NN", "currency": "NN", "car": "NN", "abortion": "NN", "Pacific": "NNP", "believes": "VBZ", "districts": "NNS", "can": "MD", "Our": "PRP$", "heart": "NN", "subsidies": "NNS", "1.2": "CD", "requirements": "NNS", "Akzo": "NNP", "1": "CD", "fourth": "JJ", "H.": "NNP", "Why": "WRB", "economy": "NN", "product": "NN", "information": "NN", "may": "MD", "membership": "NN", "date": "NN", "man": "NN", "natural": "JJ", "commodity": "NN", "futures": "NNS", "truck": "NN", "exclusive": "JJ", "indeed": "RB", "LIN": "NNP", "Hong": "NNP", "years": "NNS", "brain": "NN", "managers": "NNS", "White": "NNP", "still": "RB", "group": "NN", "Lehman": "NNP", "policy": "NN", "main": "JJ", "nation": "NN", "She": "PRP", "not": "RB", "R.": "NNP", "now": "RB", "provision": "NN", "nor": "CC", "term": "NN", "attorneys": "NNS", "Stanley": "NNP", "quarter": "NN", "significantly": "RB", "begun": "VBN", "year": "NN", "Kong": "NNP", "shown": "VBN", "space": "NN", "looking": "VBG", "investigation": "NN", "Bloomingdale": "NNP", "Commerce": "NNP", "cars": "NNS", "million": "CD", "possibility": "NN", "language": "NN", "7\\/8": "CD", "thing": "NN", "revenue": "NN", "There": "EX", "directly": "RB", "corporations": "NNS", "Hollywood": "NNP", "tomorrow": "NN", "millions": "NNS", "city": "NN", "given": "VBN", "district": "NN", "trillion": "CD", "Dow": "NNP", "anyone": "NN", "2": "CD", "SEC": "NNP", "white": "JJ", "gives": "VBZ", "a": "DT", "mostly": "RB", "season": "NN", "probably": "RB", "surged": "VBD", "than": "IN", "Inc.": "NNP", "11": "CD", "10": "CD", "television": "NN", "12": "CD", "15": "CD", "14": "CD", "17": "CD", "16": "CD", "19": "CD", "18": "CD", "spokeswoman": "NN", "officials": "NNS", "venture": "NN", "amid": "IN", "and": "CC", "Court": "NNP", "investors": "NNS", "Marcos": "NNP", "Philadelphia": "NNP", "sells": "VBZ", "any": "DT", "equipment": "NN", "intends": "VBZ", "performance": "NN", "Du": "NNP", "200": "CD", "normal": "JJ", "price": "NN", "remarks": "NNS", "D.": "NNP", "especially": "RB", "sale": "NN", "ways": "NNS", "senior": "JJ", "typically": "RB", "laws": "NNS", "rating": "NN", "commitments": "NNS", "aggressive": "JJ", "We": "PRP", "written": "VBN", "crime": "NN", "going": "VBG", "black": "JJ", "congressional": "JJ", "contracts": "NNS", "nearly": "RB", "morning": "NN", "miles": "NNS", "where": "WRB", "college": "NN", "Grand": "NNP", "concern": "NN", "mortgage": "NN", "farmers": "NNS", "federal": "JJ", "representatives": "NNS", "materials": "NNS", "weapons": "NNS", "between": "IN", "Mitsubishi": "NNP", "jobs": "NNS", "Johnson": "NNP", "U.S.": "NNP", "26": "CD", "Each": "DT", "article": "NN", "cities": "NNS", "acquiring": "VBG", "many": "JJ", "region": "NN", "according": "VBG", "contract": "NN", "holders": "NNS", "comes": "VBZ", "among": "IN", "cancer": "NN", "150": "CD", "period": "NN", ",": ",", "60": "CD", "considering": "VBG", "unusual": "JJ", "Calif.": "NNP", "Electric": "NNP", "But": "CC", "Lynch": "NNP", "500": "CD", "engine": "NN", "direction": "NN", "Analysts": "NNS", "former": "JJ", "those": "DT", "paying": "VBG", "To": "TO", "these": "DT", "consultant": "NN", "Reagan": "NNP", "cash": "NN", "n't": "RB", "policies": "NNS", "newspaper": "NN", "situation": "NN", "trader": "NN", "then": "RB", "metric": "JJ", "telephone": "NN", "Peters": "NNP", "technology": "NN", "Israel": "NNP", "media": "NNS", "same": "JJ", "events": "NNS", "status": "NN", "oil": "NN", "I": "PRP", "IRS": "NNP", "Toyota": "NNP", "Coors": "NNP", "director": "NN", "largely": "RB", "constitutional": "JJ", "roughly": "RB", "mortgages": "NNS", "Rep.": "NNP", "without": "IN", "In": "IN", "researchers": "NNS", "If": "IN", "summer": "NN", "United": "NNP", "Service": "NNP", "being": "VBG", "money": "NN", "actions": "NNS", "Daniel": "NNP", "announcement": "NN", "death": "NN", "rose": "VBD", "seems": "VBZ", "improvement": "NN", "4": "CD", "Although": "IN", "pill": "NN", "real": "JJ", "rules": "NNS", "Sachs": "NNP", "Ortega": "NNP", "inflation": "NN", "traffic": "NN", "using": "VBG", "'ve": "VBP", "annually": "RB", "audience": "NN", "London": "NNP", "retailers": "NNS", "fully": "RB", "Moreover": "RB", "Since": "IN", "competition": "NN", "Dr.": "NNP", "New": "NNP", "gross": "JJ", "legal": "JJ", "conservative": "JJ", "critical": "JJ", "deficit": "NN", "provides": "VBZ", "football": "NN", "scientific": "JJ", "power": "NN", "leadership": "NN", "manufacturer": "NN", "on": "IN", "central": "JJ", "S.A.": "NNP", "of": "IN", "industry": "NN", "Trade": "NNP", "airline": "NN", "or": "CC", "road": "NN", "outlook": "NN", "coupon": "NN", "instruments": "NNS", "image": "NN", "parties": "NNS", "your": "PRP$", "area": "NN", "Engelken": "NNP", "Bartlett": "NNP", "trying": "VBG", "with": "IN", "Guber": "NNP", "volume": "NN", "fraud": "NN", "House": "NNP", "pulp": "NN", "gone": "VBN", "ad": "NN", "certain": "JJ", "am": "VBP", "sales": "NNS", "Thursday": "NNP", "an": "DT", "at": "IN", "film": "NN", "USX": "NNP", "4.5": "CD", "again": "RB", "event": "NN", "field": "NN", "5": "CD", "you": "PRP", "Ross": "NNP", "Las": "NNP", "poor": "JJ", "Jaguar": "NNP", "students": "NNS", "includes": "VBZ", "important": "JJ", "coverage": "NN", "stocks": "NNS", "US$": "$", "assets": "NNS", "wife": "NN", "directors": "NNS", "Street": "NNP", "minister": "NN", "Canada": "NNP", "founder": "NN", "dollar": "NN", "5\\/8": "CD", "month": "NN", "settlement": "NN", "decisions": "NNS", "children": "NNS", "Brown": "NNP", "to": "TO", "program": "NN", "health": "NN", "lawmakers": "NNS", "activities": "NNS", "woman": "NN", "far": "RB", "difference": "NN", "`": "``", "cable": "NN", "--": ":", "large": "JJ", "small": "JJ", "rate": "NN", "lawyer": "NN", "investment": "NN", "HUD": "NNP", "Korea": "NNP", "consumers": "NNS", "Paribas": "NNP", "version": "NN", "scientists": "NNS", "Ogilvy": "NNP", "Bethlehem": "NNP", "full": "JJ", "hours": "NNS", "strong": "JJ", "thrift": "NN", "prosecutors": "NNS", "ahead": "RB", "houses": "NNS", "losses": "NNS", "social": "JJ", "action": "NN", "options": "NNS", "via": "IN", "family": "NN", "S.": "NNP", "establish": "VB", "Europe": "NNP", "shareholders": "NNS", "Dinkins": "NNP", "eye": "NN", "takes": "VBZ", "11\\/16": "CD", "Hewlett-Packard": "NNP", "two": "CD", "Corp": "NNP", "6": "CD", "taken": "VBN", "markets": "NNS", "Manville": "NNP", "Intel": "NNP", "division": "NN", "company": "NN", "producing": "VBG", "town": "NN", "keeping": "VBG", "hour": "NN", "nine": "CD", "history": "NN", "purchases": "NNS", "IBM": "NNP", "adviser": "NN", "share": "NN", "numbers": "NNS", "Thompson": "NNP", "sharp": "JJ", "!": ".", "huge": "JJ", "court": "NN", "goal": "NN", "rather": "RB", "Carpenter": "NNP", "earnings": "NNS", "plant": "NN", "different": "JJ", "response": "NN", "acquisitions": "NNS", "Mexico": "NNP", ")": ")", "banks": "NNS", "What": "WP", "soon": "RB", "paper": "NN", "committee": "NN", "signs": "NNS", "its": "PRP$", "Texas": "NNP", "24": "CD", "25": "CD", "style": "NN", "27": "CD", "20": "CD", "21": "CD", "22": "CD", "23": "CD", "28": "CD", "29": "CD", "actually": "RB", "systems": "NNS", "governments": "NNS", "might": "MD", "Moody": "NNP", "someone": "NN", "seeking": "VBG", "food": "NN", "Michael": "NNP", "bigger": "JJR", "easily": "RB", "always": "RB", "week": "NN", "everyone": "NN", "generation": "NN", "house": "NN", "energy": "NN", "reduce": "VB", "idea": "NN", "slowdown": "NN", "Joseph": "NNP", "advertisers": "NNS", "operation": "NN", "beyond": "IN", "insurance": "NN", "really": "RB", "E.": "NNP", "since": "IN", "temporary": "JJ", "research": "NN", "safety": "NN", "7": "CD", "According": "VBG", "EC": "NNP", "extraordinary": "JJ", "reason": "NN", "members": "NNS", "producers": "NNS", "owners": "NNS", "benefits": "NNS", "Boston": "NNP", "computers": "NNS", "threat": "NN", "pilots": "NNS", "major": "JJ", "Hugo": "NNP", "number": "NN", "feet": "NNS", "done": "VBN", "fees": "NNS", "story": "NN", "statement": "NN", "option": "NN", "relationship": "NN", "part": "NN", "kind": "NN", "grew": "VBD", "toward": "IN", "outstanding": "JJ", "Douglas": "NNP", "It": "PRP", "substantial": "JJ", "orders": "NNS", "ratings": "NNS", "majority": "NN", "internal": "JJ", "Drexel": "NNP", "chairman": "NN", "With": "IN", "75": "CD", "shareholder": "NN", "significant": "JJ", "70": "CD", "services": "NNS", "The": "DT", "extremely": "RB", "dealers": "NNS", "OTC": "NNP", "traditional": "JJ", "three-month": "JJ", "institutions": "NNS", "sector": "NN", "particularly": "RB", "session": "NN", "businesses": "NNS", "Poor": "NNP", "regulations": "NNS", "merger": "NN", "equity": "NN", "8": "CD", "Prime": "NNP", "his": "PRP$", "gains": "NNS", "While": "IN", "5,000": "CD", "closely": "RB", "During": "IN", "during": "IN", "him": "PRP", "merchandise": "NN", "six-month": "JJ", "J.": "NNP", "common": "JJ", "activity": "NN", "wrote": "VBD", "Chairman": "NNP", "For": "IN", "France": "NNP", "culture": "NN", "defense": "NN", "are": "VBP", "jury": "NN", "2.5": "CD", "#": "#", "movie": "NN", "currently": "RB", "case": "NN", "various": "JJ", "Sony": "NNP", "conditions": "NNS", "available": "JJ", "recently": "RB", "creating": "VBG", "dividends": "NNS", "attention": "NN", "Florida": "NNP", "succeed": "VB", "opposition": "NN", "dividend": "NN", "last": "JJ", "ANC": "NNP", "annual": "JJ", "foreign": "JJ", "connection": "NN", "became": "VBD", "long-term": "JJ", "Compaq": "NNP", "reasons": "NNS", "loan": "NN", "community": "NN", "simply": "RB", "throughout": "IN", "political": "JJ", "earthquake": "NN", "whom": "WP", "reduction": "NN", "California": "NNP", "treatment": "NN", "partly": "RB", "gas": "NN", "priced": "VBN", "brokers": "NNS", "prices": "NNS", "plants": "NNS", "bill": "NN", "elections": "NNS", "33": "CD", "31": "CD", "City": "NNP", "pound": "NN", "Italy": "NNP", "voters": "NNS", "cents": "NNS", "itself": "PRP", "seen": "VBN", "Co.": "NNP", "underwriters": "NNS", "virtually": "RB", "widely": "RB", "grand": "JJ", "9": "CD", "products": "NNS", "relatively": "RB", "development": "NN", "currencies": "NNS", "Allianz": "NNP", "affairs": "NNS", "yesterday": "NN", "moment": "NN", "levels": "NNS", "{": "(", "recent": "JJ", "Miller": "NNP", "person": "NN", "organization": "NN", "one-year": "JJ", "competitive": "JJ", "Boren": "NNP", "questions": "NNS", "world": "NN", "profitable": "JJ", "retirement": "NN", "$": "$", "over-the-counter": "JJ", "workers": "NNS", "source": "NN", "Germany": "NNP", "...": ":", "customers": "NNS", "Last": "JJ", "emergency": "NN", "Of": "IN", "Air": "NNP", "game": "NN", "necessary": "JJ", "projects": "NNS", "follows": "VBZ", "individuals": "NNS", "popular": "JJ", "often": "RB", "Gulf": "NNP", "some": "DT", "3\\/4": "CD", "economic": "JJ", "3\\/8": "CD", "Frank": "NNP", "decision": "NN", "transactions": "NNS", "quickly": "RB", "Massachusetts": "NNP", "be": "VB", "Brady": "NNP", "300": "CD", "agreement": "NN", "David": "NNP", "output": "NN", "abroad": "RB", "pipeline": "NN", "goods": "NNS", "anything": "NN", "Pont": "NNP", "Roy": "NNP", "ounce": "NN", "Committee": "NNP", "into": "IN", "within": "IN", "NEC": "NNP", "nothing": "NN", "primarily": "RB", "Quebecor": "NNP", "bankruptcy": "NN", ":": ":", "himself": "PRP", "vehicle": "NN", "Ms.": "NNP", "Ltd.": "NNP", "Switzerland": "NNP", "subsidiary": "NN", "line": "NN", "Bell": "NNP", "Africa": "NNP", "us": "PRP", "Thatcher": "NNP", "maturity": "NN", "'re": "VBP", "exploration": "NN", "Those": "DT", "similar": "JJ", "Perhaps": "RB", "Hampshire": "NNP", "Westinghouse": "NNP", "single": "JJ", "Edward": "NNP", "International": "NNP", "Manhattan": "NNP", "%": "NN", "May": "NNP", "politicians": "NNS", "Mae": "NNP", "income": "NN", "department": "NN", "AG": "NNP", "problems": "NNS", "helping": "VBG", "allowing": "VBG", "reinsurance": "NN", "sides": "NNS", "structure": "NN", "vice": "NN", "age": "NN", "vehicles": "NNS", "bankers": "NNS", "An": "DT", "At": "IN", "requires": "VBZ", "having": "VBG", "results": "NNS", "Department": "NNP", "issues": "NNS", "young": "JJ", "suits": "NNS", "citing": "VBG", "UAL": "NNP", "Not": "RB", "Now": "RB", "resources": "NNS", "P&G": "NNP", "automotive": "JJ", "continues": "VBZ", "Mrs.": "NNP", "putting": "VBG", "entire": "JJ", "positions": "NNS", "race": "NN", "smaller": "JJR", "crop": "NN", "Hutton": "NNP", "makers": "NNS", "index": "NN", "business": "NN", "giving": "VBG", "Alan": "NNP", "access": "NN", "volatile": "JJ", "firms": "NNS", "America": "NNP", "pushing": "VBG", "jointly": "RB", "others": "NNS", "great": "JJ", "38": "CD", "technical": "JJ", "Energy": "NNP", "larger": "JJR", "37": "CD", "35": "CD", "CBS": "NNP", "survey": "NN", "Motor": "NNP", "opinion": "NN", "residents": "NNS", "gene": "NN", "makes": "VBZ", "maker": "NN", "apple": "NN", "Robert": "NNP", "private": "JJ", "privately": "RB", "scandal": "NN", "from": "IN", "&": "CC", "few": "JJ", "Fe": "NNP", "year-ago": "JJ", "themselves": "PRP", "chip": "NN", "reflects": "VBZ", "Wednesday": "NNP", "sharply": "RB", "women": "NNS", "customer": "NN", "this": "DT", "clients": "NNS", "recession": "NN", "industrial": "JJ", "F.": "NNP", "Northern": "NNP", "tax": "NN", "Mr.": "NNP", "reserves": "NNS", "something": "NN", "Party": "NNP", "BellSouth": "NNP", "holds": "VBZ", "traders": "NNS", "instead": "RB", "stock": "NN", "ABC": "NNP", "Nissan": "NNP", "Terms": "NNS", "engineering": "NN", "lines": "NNS", "Community": "NNP", "Oct.": "NNP", "software": "NN", "six": "CD", "producer": "NN", "institutional": "JJ", "Smith": "NNP", "including": "VBG", "year-earlier": "JJ", "industries": "NNS", "Exchange": "NNP", "Brooks": "NNP", "labor": "NN", "willing": "JJ", "greater": "JJR", "auto": "NN", "practice": "NN", "investor": "NN", "day": "NN", "Supreme": "NNP", "San": "NNP", "bills": "NNS", "Corry": "NNP", "doing": "VBG", "books": "NNS", "Treasury": "NNP", "our": "PRP$", "80": "CD", "Unisys": "NNP", "entertainment": "NN", "critics": "NNS", "China": "NNP", "disclose": "VB", "This": "DT", "regulatory": "JJ", "could": "MD", "Lawson": "NNP", "succeeds": "VBZ", "powerful": "JJ", "strategic": "JJ", "owner": "NN", "management": "NN", "system": "NN", "relations": "NNS", "Coast": "NNP", "their": "PRP$", "Pilson": "NNP", "final": "JJ", "Association": "NNP", "interests": "NNS", "acquire": "VB", "environmental": "JJ", "chemicals": "NNS", "reflecting": "VBG", "steel": "NN", "colleagues": "NNS", "patients": "NNS", "Peter": "NNP", "creditors": "NNS", "1.4": "CD", "1.5": "CD", "1.6": "CD", "1.1": "CD", "Richard": "NNP", "1.3": "CD", "unchanged": "JJ", "partnership": "NN", "Other": "JJ", ";": ":", "apparently": "RB", "clearly": "RB", "Development": "NNP", "documents": "NNS", "Goldman": "NNP", "After": "IN", "able": "JJ", "instance": "NN", "which": "WDT", "unless": "IN", "who": "WP", "eight": "CD", "segment": "NN", "payment": "NN", "Reserve": "NNP", "so-called": "JJ", "Some": "DT", "MCA": "NNP", "1,000": "CD", "}": ")", "Saturday": "NNP", "fact": "NN", "Paris": "NNP", "Sansui": "NNP", "Chemical": "NNP", "Under": "IN", "portfolio": "NN", "economist": "NN", "decade": "NN", "staff": "NN", "partners": "NNS", "based": "VBN", "Meanwhile": "RB", "(": "(", "should": "MD", "candidates": "NNS", "York": "NNP", "employee": "NN", "local": "JJ", "bonds": "NNS", "familiar": "JJ", "120": "CD", "ones": "NNS", "words": "NNS", "exchanges": "NNS", "buyer": "NN", "chips": "NNS", "areas": "NNS", "Because": "IN", "trucks": "NNS", "course": "NN", "taxes": "NNS", "calling": "VBG", "Wall": "NNP", "she": "PRP", "Burnham": "NNP", "temporarily": "RB", "national": "JJ", "computer": "NN", "nuclear": "JJ", "state": "NN", "July": "NNP", "Sen.": "NNP", "ability": "NN", "agencies": "NNS", "job": "NN", "takeover": "NN", "approval": "NN", "problem": "NN", "declining": "VBG", "restrictions": "NNS", "drug": "NN", "1\\/2": "CD", "1\\/4": "CD", "1\\/8": "CD", "ca": "MD", "Los": "NNP", "addition": "NN", "genetic": "JJ", "agreements": "NNS", "proposal": "NN", "Toronto": "NNP", "Center": "NNP", "Navigation": "NNP", "And": "CC", "homes": "NNS", "unlike": "IN", "value": "NN", "will": "MD", "PLC": "NNP", "Delmed": "NNP", "owns": "VBZ", "almost": "RB", "thus": "RB", "site": "NN", "partner": "NN", "You": "PRP", "perhaps": "RB", "began": "VBD", "administration": "NN", "Bear": "NNP", "member": "NN", "when": "WRB", "parts": "NNS", "largest": "JJS", "units": "NNS", "party": "NN", "gets": "VBZ", "difficult": "JJ", "effect": "NN", "Mitchell": "NNP", "Houston": "NNP", "transaction": "NN", "Senate": "NNP", "wants": "VBZ", "350": "CD", "position": "NN", "Shearson": "NNP", "latest": "JJS", "stores": "NNS", "heavily": "RB", "increasingly": "RB", "domestic": "JJ", "obtain": "VB", "sources": "NNS", "Sunday": "NNP", "rooms": "NNS", "ads": "NNS", "Friday": "NNP", "book": "NN", "cosmetics": "NNS", "Despite": "IN", "By": "IN", "provisions": "NNS", "government": "NN", "five": "CD", "immediately": "RB", "loss": "NN", "England": "NNP", "Aug.": "NNP", "success": "NN", "B.": "NNP", "payments": "NNS", "amendment": "NN", "arbitrage": "NN", "Sun": "NNP", "Kidder": "NNP", "February": "NNP", "growth": "NN", "employment": "NN", "Singapore": "NNP", "broad": "JJ", "However": "RB", "does": "VBZ", "leader": "NN", "?": ".", "Baker": "NNP", "Rothschild": "NNP", "monetary": "JJ", "expansion": "NN", "Fujitsu": "NNP", "although": "IN", "loans": "NNS", "panel": "NN", "actual": "JJ", "debentures": "NNS", "December": "NNP", "Peabody": "NNP", "holdings": "NNS", "carries": "VBZ", "carrier": "NN", "Japan": "NNP", "executives": "NNS", "letters": "NNS", "previously": "RB", "warrants": "NNS", "Two": "CD", "getting": "VBG", "strategy": "NN", "utility": "NN", "1986": "CD", "1987": "CD", "1984": "CD", "1985": "CD", "1982": "CD", "additional": "JJ", "1980": "CD", "Lawrence": "NNP", "housing": "NN", "1988": "CD", "1989": "CD", "biggest": "JJS", "November": "NNP", "funds": "NNS", "brand": "NN", "but": "CC", "delivery": "NN", "construction": "NN", "highest": "JJS", "he": "PRP", "also": "RB", "Industrial": "NNP", "whether": "IN", "cells": "NNS", "Britain": "NNP", "distribution": "NN", "minutes": "NNS", "flight": "NN", "margins": "NNS", "mutual": "JJ", "compared": "VBN", "'ll": "MD", "48": "CD", "49": "CD", "46": "CD", "Jr.": "NNP", "44": "CD", "45": "CD", "42": "CD", "Yesterday": "NN", "40": "CD", "Volume": "NN", "other": "JJ", "details": "NNS", "Corp.": "NNP", "junk": "NN", "Like": "IN", "class": "NN", "March": "NNP", "April": "NNP", "chance": "NN", "Morgan": "NNP", "Act": "NNP", "factors": "NNS", "portion": "NN", "pension": "NN"}

pyproject.toml

@@ -0,0 +1,14 @@
+[project]
+name = "diffusion-speech-360h"
+version = "0.1.0"
+description = "A simple diffusion-based text to speech model"
+readme = "README.md"
+requires-python = ">=3.12"
+dependencies = [
+    "g2p-en>=2.1.0",
+    "gradio>=5.9.1",
+    "nltk>=3.9.1",
+    "soundfile>=0.12.1",
+    "torch>=2.5.1",
+    "vocos>=0.1.0",
+]

sample.py

@@ -0,0 +1,233 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+import torch
+
+torch.backends.cuda.matmul.allow_tf32 = True
+torch.backends.cudnn.allow_tf32 = True
+import argparse
+import os
+
+import numpy as np
+import torch
+import yaml
+from tqdm import tqdm
+
+from diffusion import create_diffusion
+from models import DiT_models
+
+
+def find_model(model_name):
+    assert os.path.isfile(model_name), f"Could not find DiT checkpoint at {model_name}"
+    checkpoint = torch.load(model_name, map_location=lambda storage, loc: storage)
+    if "ema" in checkpoint:  # supports checkpoints from train.py
+        print("Using EMA model")
+        checkpoint = checkpoint["ema"]
+    else:
+        print("Using model")
+        checkpoint = checkpoint["model"]
+    return checkpoint
+
+
+def get_batch(
+    step, batch_size, seq_len, DEVICE, data_file, data_dim, data_mean, data_std
+):
+    # Load dataset from memmap file
+    arr = np.memmap(data_file, dtype=np.float16, mode="r")
+    arr = np.memmap(
+        data_file,
+        dtype=np.float16,
+        mode="r",
+        shape=(arr.shape[0] // (data_dim + 3), data_dim + 3),
+    )
+
+    # Create random number generator
+    rng = np.random.Generator(np.random.PCG64(seed=step))
+
+    # Generate start indices and convert to integer array
+    start_indices = rng.choice(
+        arr.shape[0] - seq_len, size=batch_size, replace=False
+    ).astype(np.int64)
+
+    # Create batch data array
+    batch_data = np.zeros((batch_size, seq_len, data_dim + 3), dtype=np.float16)
+    # Fill batch data one sequence at a time
+    for i, start_idx in enumerate(start_indices):
+        batch_data[i] = arr[start_idx : start_idx + seq_len]
+
+    # Extract features
+    x = batch_data[:, :, :data_dim].astype(np.float16)
+    x = np.moveaxis(x, 1, 2)
+    phone = batch_data[:, :, data_dim].astype(np.int32)
+    speaker_id = batch_data[:, :, data_dim + 1].astype(np.int32)
+    phone_kind = batch_data[:, :, data_dim + 2].astype(np.int32)
+
+    # convert to torch tensors
+    x = torch.from_numpy(x).to(DEVICE)
+    x = (x - data_mean) / data_std
+    phone = torch.from_numpy(phone).to(DEVICE)
+    speaker_id = torch.from_numpy(speaker_id).to(DEVICE)
+    phone_kind = torch.from_numpy(phone_kind).to(DEVICE)
+
+    return x, speaker_id, phone, phone_kind
+
+
+def get_data(config_path, seed=0):
+    with open(config_path, "r") as f:
+        config = yaml.safe_load(f)
+
+    data_config = config["data"]
+    model_config = config["model"]
+    device = "cuda" if torch.cuda.is_available() else "cpu"
+
+    x, speaker_id, phone, phone_kind = get_batch(
+        seed,
+        1,
+        seq_len=model_config["input_size"],
+        DEVICE=device,
+        data_file=data_config["data_path"],
+        data_dim=data_config["data_dim"],
+        data_mean=data_config["data_mean"],
+        data_std=data_config["data_std"],
+    )
+
+    return x, speaker_id, phone, phone_kind
+
+
+def plot_samples(samples, x):
+    # Create figure and axis
+    fig, ax = plt.subplots(figsize=(20, 4))
+    plt.tight_layout()
+
+    # Function to update frame
+    def update(frame):
+        ax.clear()
+        ax.text(
+            0.02,
+            0.98,
+            f"{frame+1} / 1000",
+            transform=ax.transAxes,
+            verticalalignment="top",
+            color="black",
+        )
+        if samples[frame].shape[1] > 1:
+            im = ax.imshow(
+                samples[frame].cpu().numpy()[0],
+                origin="lower",
+                aspect="auto",
+                interpolation="none",
+                vmin=-5,
+                vmax=5,
+            )
+            return [im]
+        elif samples[frame].shape[1] == 1:
+            line1 = ax.plot(samples[frame].cpu().numpy()[0, 0])[0]
+            line2 = ax.plot(x.cpu().numpy()[0, 0])[0]
+            plt.ylim(-10, 10)
+            return [line1, line2]
+
+    # Create animation with progress bar
+    anim = animation.FuncAnimation(
+        fig,
+        update,
+        frames=tqdm(range(len(samples)), desc="Generating animation"),
+        interval=1000 / 60,
+        blit=True,  # 24 fps
+    )
+
+    # Save as MP4
+    anim.save("animation.mp4", fps=60, extra_args=["-vcodec", "libx264"])
+    plt.close()
+
+
+def sample(
+    config_path,
+    ckpt_path,
+    cfg_scale=4.0,
+    num_sampling_steps=1000,
+    seed=0,
+    speaker_id=None,
+    phone=None,
+    phone_kind=None,
+):
+    torch.manual_seed(seed)
+    torch.set_grad_enabled(False)
+    device = "cuda" if torch.cuda.is_available() else "cpu"
+
+    with open(config_path, "r") as f:
+        config = yaml.safe_load(f)
+
+    data_config = config["data"]
+    model_config = config["model"]
+
+    # Load model:
+    model = DiT_models[model_config["name"]](
+        input_size=model_config["input_size"],
+        embedding_vocab_size=model_config["embedding_vocab_size"],
+        learn_sigma=model_config["learn_sigma"],
+        in_channels=data_config["data_dim"],
+    ).to(device)
+
+    state_dict = find_model(ckpt_path)
+    model.load_state_dict(state_dict)
+    model.eval()  # important!
+    diffusion = create_diffusion(str(num_sampling_steps))
+    n = 1
+    z = torch.randn(n, data_config["data_dim"], speaker_id.shape[1], device=device)
+
+    attn_mask = speaker_id[:, None, :] == speaker_id[:, :, None]
+    attn_mask = attn_mask.unsqueeze(1)
+    attn_mask = torch.cat([attn_mask, attn_mask], 0)
+    # Setup classifier-free guidance:
+    z = torch.cat([z, z], 0)
+    unconditional_value = model.y_embedder.unconditional_value
+    phone_null = torch.full_like(phone, unconditional_value)
+    speaker_id_null = torch.full_like(speaker_id, unconditional_value)
+    phone = torch.cat([phone, phone_null], 0)
+    speaker_id = torch.cat([speaker_id, speaker_id_null], 0)
+    phone_kind_null = torch.full_like(phone_kind, unconditional_value)
+    phone_kind = torch.cat([phone_kind, phone_kind_null], 0)
+    model_kwargs = dict(
+        phone=phone,
+        speaker_id=speaker_id,
+        phone_kind=phone_kind,
+        cfg_scale=cfg_scale,
+        attn_mask=attn_mask,
+    )
+
+    samples = diffusion.p_sample_loop(
+        model.forward_with_cfg,
+        z.shape,
+        z,
+        clip_denoised=False,
+        model_kwargs=model_kwargs,
+        progress=True,
+        device=device,
+    )
+    samples = [s.chunk(2, dim=0)[0] for s in samples]  # Remove null class samples
+    return samples
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--config", type=str, required=True)
+    parser.add_argument("--ckpt", type=str, required=True)
+    parser.add_argument("--cfg-scale", type=float, default=4.0)
+    parser.add_argument("--num-sampling-steps", type=int, default=1000)
+    parser.add_argument("--seed", type=int, default=0)
+    args = parser.parse_args()
+    x, speaker_id, phone, phone_kind = get_data(args.config, args.seed)
+    samples = sample(
+        args.config,
+        args.ckpt,
+        args.cfg_scale,
+        args.num_sampling_steps,
+        args.seed,
+        speaker_id,
+        phone,
+        phone_kind,
+    )
+    plot_samples(samples, x)

synthesize.py

@@ -0,0 +1,200 @@
+"""
+Synthesize a given text using the trained DiT models.
+"""
+
+import json
+import os
+
+os.environ["NLTK_DATA"] = "nltk_data"
+import torch
+import yaml
+from g2p_en import G2p
+import soundfile as sf
+from vocos import Vocos
+from sample import sample
+
+
+def synthesize(
+    text,
+    duration_model_config,
+    duration_model_checkpoint,
+    acoustic_model_config,
+    acoustic_model_checkpoint,
+    speaker_id,
+    cfg_scale=4.0,
+    num_sampling_steps=1000,
+):
+    """
+    Synthesize speech from text using trained DiT models.
+
+    Args:
+        text (str): Input text to synthesize
+        duration_model_config (str): Path to duration model config file
+        duration_model_checkpoint (str): Path to duration model checkpoint
+        acoustic_model_config (str): Path to acoustic model config file
+        acoustic_model_checkpoint (str): Path to acoustic model checkpoint
+        speaker_id (str): Speaker ID to use for synthesis
+        cfg_scale (float): Classifier-free guidance scale (default: 4.0)
+        num_sampling_steps (int): Number of sampling steps for diffusion (default: 1000)
+
+    Returns:
+        numpy.ndarray: Audio waveform array
+        int: Sample rate (24000)
+    """
+
+    print("Text:", text)
+
+    # Read duration model config
+    with open(duration_model_config, "r") as f:
+        duration_config = yaml.safe_load(f)
+
+    # Get data directory from data_path
+    data_dir = os.path.dirname(duration_config["data"]["data_path"])
+
+    # Read maps.json from same directory
+    with open(os.path.join(data_dir, "maps.json"), "r") as f:
+        maps = json.load(f)
+    phone_to_idx = maps["phone_to_idx"]
+    phone_kind_to_idx = maps["phone_kind_to_idx"]
+    speaker_id_to_idx = maps["speaker_id_to_idx"]
+
+    # Step 1: Text to phonemes
+    def text_to_phonemes(text, insert_empty=True):
+        g2p = G2p()
+        phonemes = g2p(text)
+        words = []
+        word = []
+        for p in phonemes:
+            if p == " ":
+                if len(word) > 0:
+                    words.append(word)
+                word = []
+            else:
+                word.append(p)
+        if len(word) > 0:
+            words.append(word)
+
+        phones = []
+        phone_kinds = []
+        for word in words:
+            for i, p in enumerate(word):
+                if p in [",", ".", "!", "?", ";", ":"]:
+                    p = "EMPTY"
+                elif p in phone_to_idx:
+                    pass
+                else:
+                    continue
+
+                if p == "EMPTY":
+                    phone_kind = "EMPTY"
+                elif len(word) == 1:
+                    phone_kind = "WORD"
+                elif i == 0:
+                    phone_kind = "START"
+                elif i == len(word) - 1:
+                    phone_kind = "END"
+                else:
+                    phone_kind = "MIDDLE"
+
+                phones.append(p)
+                phone_kinds.append(phone_kind)
+
+        if insert_empty:
+            if phones[0] != "EMPTY":
+                phones.insert(0, "EMPTY")
+                phone_kinds.insert(0, "EMPTY")
+            if phones[-1] != "EMPTY":
+                phones.append("EMPTY")
+                phone_kinds.append("EMPTY")
+
+        return phones, phone_kinds
+
+    phonemes, phone_kinds = text_to_phonemes(text)
+    # Convert phonemes to indices
+    phoneme_indices = [phone_to_idx[p] for p in phonemes]
+    phone_kind_indices = [phone_kind_to_idx[p] for p in phone_kinds]
+    print("Phonemes:", phonemes)
+
+    # Step 2: Duration prediction
+    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+    torch_phoneme_indices = torch.tensor(phoneme_indices)[None, :].long().to(device)
+    torch_speaker_id = torch.full_like(torch_phoneme_indices, int(speaker_id))
+    torch_phone_kind_indices = (
+        torch.tensor(phone_kind_indices)[None, :].long().to(device)
+    )
+
+    samples = sample(
+        duration_model_config,
+        duration_model_checkpoint,
+        cfg_scale=cfg_scale,
+        num_sampling_steps=num_sampling_steps,
+        seed=0,
+        speaker_id=torch_speaker_id,
+        phone=torch_phoneme_indices,
+        phone_kind=torch_phone_kind_indices,
+    )
+    phoneme_durations = samples[-1][0, 0]
+
+    # Step 3: Acoustic prediction
+    # First, we need to convert phoneme durations to number of frames per phoneme (min 1 frame)
+    SAMPLE_RATE = 24000
+    HOP_LENGTH = 256
+    N_FFT = 1024
+    N_MELS = 100
+    time_per_frame = HOP_LENGTH / SAMPLE_RATE
+    # convert predicted durations to raw durations using data mean and std in the config
+    if duration_config["data"]["normalize"]:
+        mean = duration_config["data"]["data_mean"]
+        std = duration_config["data"]["data_std"]
+        raw_durations = phoneme_durations * std + mean
+    else:
+        raw_durations = phoneme_durations
+
+    raw_durations = raw_durations.clamp(min=time_per_frame, max=1.0)
+    end_time = torch.cumsum(raw_durations, dim=0)
+    end_frame = end_time / time_per_frame
+    int_end_frame = end_frame.floor().int()
+    repeated_phoneme_indices = []
+    repeated_phone_kind_indices = []
+    for i in range(len(phonemes)):
+        repeated_phoneme_indices.extend(
+            [phoneme_indices[i]] * (int_end_frame[i] - len(repeated_phoneme_indices))
+        )
+        repeated_phone_kind_indices.extend(
+            [phone_kind_indices[i]]
+            * (int_end_frame[i] - len(repeated_phone_kind_indices))
+        )
+
+    torch_phoneme_indices = (
+        torch.tensor(repeated_phoneme_indices)[None, :].long().to(device)
+    )
+    torch_speaker_id = torch.full_like(torch_phoneme_indices, int(speaker_id))
+    torch_phone_kind_indices = (
+        torch.tensor(repeated_phone_kind_indices)[None, :].long().to(device)
+    )
+
+    samples = sample(
+        acoustic_model_config,
+        acoustic_model_checkpoint,
+        cfg_scale=cfg_scale,
+        num_sampling_steps=num_sampling_steps,
+        seed=0,
+        speaker_id=torch_speaker_id,
+        phone=torch_phoneme_indices,
+        phone_kind=torch_phone_kind_indices,
+    )
+    mel = samples[-1][0]
+    # compute raw mel if acoustic model normalize is true
+    acoustic_config = yaml.safe_load(open(acoustic_model_config, "r"))
+    if acoustic_config["data"]["normalize"]:
+        mean = acoustic_config["data"]["data_mean"]
+        std = acoustic_config["data"]["data_std"]
+        raw_mel = mel * std + mean
+    else:
+        raw_mel = mel
+
+    # Step 4: Vocoder
+    vocos = Vocos.from_pretrained("charactr/vocos-mel-24khz")
+    audio = vocos.decode(raw_mel.cpu()[None, :, :]).squeeze().cpu().numpy()
+
+    return audio, SAMPLE_RATE

train_acoustic_dit_b.yaml

@@ -0,0 +1,40 @@
+model:
+  name: DiT-B
+  input_size: 2048
+  embedding_vocab_size: 1024
+  learn_sigma: true
+optimization:
+  constant_memory: false
+  epochs: 1400
+  global_batch_size: 32
+  initial_input_size: 32
+  learning_rate: 1.0e-4
+  min_lr: 1.0e-5
+  warmup_iters: 10000
+  lr_decay_iters: 100000
+  decay_lr: true
+  weight_decay: 0.0
+  max_grad_norm: 20.0
+  betas:
+    beta1: 0.9
+    beta2: 0.999
+  loss:
+    num_timesteps: 1000
+data:
+  data_path: acoustic.npy
+  data_dim: 100
+  data_std: 2.0
+  data_mean: -1.0
+  normalize: true
+training:
+  enable_compile: true
+  use_bfloat16: true
+  use_block_mask: false
+  seed: 42
+  ckpt_every: 10_000
+  log_every: 100
+  results_dir: results/acoustic
+  resume_from_ckpt: null
+  wandb:
+    enable: true
+    project: diffusion-speech

train_duration_dit_s.yaml

@@ -0,0 +1,40 @@
+model:
+  name: DiT-S
+  input_size: 512
+  embedding_vocab_size: 1024
+  learn_sigma: true
+optimization:
+  constant_memory: false
+  epochs: 1400
+  global_batch_size: 512
+  initial_input_size: 32
+  learning_rate: 1.0e-4
+  min_lr: 1.0e-5
+  warmup_iters: 10000
+  lr_decay_iters: 100000
+  decay_lr: true
+  weight_decay: 0.0
+  max_grad_norm: 20.0
+  betas:
+    beta1: 0.9
+    beta2: 0.999
+  loss:
+    num_timesteps: 1000
+data:
+  data_path: duration.npy
+  data_dim: 1
+  data_std: 0.067776896
+  data_mean: 0.08663661
+  normalize: true
+training:
+  enable_compile: true
+  use_bfloat16: true
+  use_block_mask: false
+  seed: 42
+  ckpt_every: 10_000
+  log_every: 100
+  results_dir: results/duration
+  resume_from_ckpt: null
+  wandb:
+    enable: true
+    project: diffusion-speech